From 092dc86e6146b98f408a7691011da11a4abde435 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 12 Dec 2024 19:59:18 +1030 Subject: [PATCH] mimecast: add cloud integrated logs data stream (#11910) Tested against a real endpoint. Pipeline test cases obtained from a test instance. Up to 10 examples of each available type are included. Not all types are represented. --- packages/mimecast/_dev/build/docs/README.md | 11 + .../_dev/deploy/docker/files/config.yml | 133 ++ .../mimecast/_dev/deploy/docker/files/ent.gz | Bin 0 -> 629 bytes .../_dev/deploy/docker/files/mail0.gz | Bin 0 -> 361 bytes .../_dev/deploy/docker/files/mail1.gz | Bin 0 -> 492 bytes packages/mimecast/changelog.yml | 5 + .../pipeline/test-cloud-integrated-logs.log | 20 + ...st-cloud-integrated-logs.log-expected.json | 1209 +++++++++++++++++ .../_dev/test/pipeline/test-common-config.yml | 3 + .../_dev/test/system/test-default-config.yml | 17 + .../agent/stream/cel.yml.hbs | 177 +++ .../elasticsearch/ingest_pipeline/default.yml | 278 ++++ .../cloud_integrated_logs/fields/agent.yml | 39 + .../fields/base-fields.yml | 20 + .../cloud_integrated_logs/fields/field.yml | 123 ++ .../cloud_integrated_logs/manifest.yml | 77 ++ .../cloud_integrated_logs/sample_event.json | 66 + .../siem_logs/agent/stream/cel.yml.hbs | 4 + packages/mimecast/docs/README.md | 125 ++ packages/mimecast/manifest.yml | 2 +- 20 files changed, 2308 insertions(+), 1 deletion(-) create mode 100644 packages/mimecast/_dev/deploy/docker/files/ent.gz create mode 100644 packages/mimecast/_dev/deploy/docker/files/mail0.gz create mode 100644 packages/mimecast/_dev/deploy/docker/files/mail1.gz create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log-expected.json create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-common-config.yml create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/system/test-default-config.yml create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/agent/stream/cel.yml.hbs create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/fields/agent.yml create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/fields/base-fields.yml create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/fields/field.yml create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/manifest.yml create mode 100644 packages/mimecast/data_stream/cloud_integrated_logs/sample_event.json diff --git a/packages/mimecast/_dev/build/docs/README.md b/packages/mimecast/_dev/build/docs/README.md index cb7aaf3dc99..3bc36800c61 100644 --- a/packages/mimecast/_dev/build/docs/README.md +++ b/packages/mimecast/_dev/build/docs/README.md @@ -49,6 +49,17 @@ https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-stat {{fields "audit_events"}} +### Cloud Integrated Logs + +This is the `mimecast.cloud_integrated_logs` dataset. These logs contain Mimecast +threats and security events with the following details: entities, mail flows and URL +protected events. More information about [these logs]( +https://developer.services.mimecast.com/docs/threatssecurityeventsanddataforci/1/routes/siem/v1/batch/events/ci/get). + +{{event "cloud_integrated_logs"}} + +{{fields "cloud_integrated_logs"}} + ### DLP Logs This is the `mimecast.dlp_logs` dataset. These logs contain information about diff --git a/packages/mimecast/_dev/deploy/docker/files/config.yml b/packages/mimecast/_dev/deploy/docker/files/config.yml index dc217f31b5f..a643dade2f1 100644 --- a/packages/mimecast/_dev/deploy/docker/files/config.yml +++ b/packages/mimecast/_dev/deploy/docker/files/config.yml @@ -510,6 +510,139 @@ rules: "isCaughtUp": true } + - path: /siem/v1/batch/events/ci + methods: ["GET"] + query_params: + type: "entities" + nextPage: null + request_headers: + authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/octet-stream" + body: | + { + "value": [ + { + "url": "http://svc-mimecast:8080/siemblob/ent", + "expiry": "2024-11-19T02:14:04.839Z", + "size": 629 + } + ], + "@nextPage": "nexttoken", + "isCaughtUp": false + } + - path: /siemblob/ent + methods: ["GET"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/octet-stream" + body: '{{file "/files/ent.gz"}}' + - path: /siem/v1/batch/events/ci + methods: ["GET"] + query_params: + type: "entities" + nextPage: "nexttoken" + request_headers: + authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/octet-stream" + body: | + { + "value": [], + "@nextPage": "String", + "isCaughtUp": true + } + - path: /siem/v1/batch/events/ci + methods: ["GET"] + query_params: + type: "mailflow" + nextPage: null + request_headers: + authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/octet-stream" + body: | + { + "value": [ + { + "url": "http://svc-mimecast:8080/siemblob/mail0", + "expiry": "2024-11-19T02:14:04.839Z", + "size": 361 + } + ], + "@nextPage": "nexttoken", + "isCaughtUp": false + } + - path: /siemblob/mail0 + methods: ["GET"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/octet-stream" + body: '{{file "/files/mail0.gz"}}' + - path: /siem/v1/batch/events/ci + methods: ["GET"] + query_params: + type: "mailflow" + nextPage: "nexttoken" + request_headers: + authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/octet-stream" + body: | + { + "value": [ + { + "url": "http://svc-mimecast:8080/siemblob/mail1", + "expiry": "2024-11-19T02:14:04.839Z", + "size": 492 + } + ], + "@nextPage": "lasttoken", + "isCaughtUp": false + } + - path: /siemblob/mail1 + methods: ["GET"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/octet-stream" + body: '{{file "/files/mail1.gz"}}' + - path: /siem/v1/batch/events/ci + methods: ["GET"] + query_params: + type: "mailflow" + nextPage: "lasttoken" + request_headers: + authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"] + responses: + - status_code: 200 + headers: + Content-Type: + - "application/octet-stream" + body: | + { + "value": [], + "@nextPage": "String", + "isCaughtUp": true + } + - path: /api/ttp/threat-intel/get-feed methods: ["POST"] request_body: /"feedType":"malware_customer","fileType":"stix","start":/ diff --git a/packages/mimecast/_dev/deploy/docker/files/ent.gz b/packages/mimecast/_dev/deploy/docker/files/ent.gz new file mode 100644 index 0000000000000000000000000000000000000000..972435b53c83c2c1bf3c884870c040a8f3ea0765 GIT binary patch literal 629 zcmV-*0*d_~iwFpW$VX=Y17&V>0CiJcZ<{a_{VS251HxdwqH1c>(3YtQQIbxbstFnU zLc9T{;8bm@{`XzdcG@2HV%hiWoO|x|E6H_P4q(WPMy?;Sa7yxy_eL3|TQc*#Dd9rs zLu*n=X5`{->#~p$v{qFQ6*q9Qv->04-23q+I7X~)EY*1fKR)WJXhY%eIq#3&i&Pw{ zoQ9qox{SI0Sj8AF_6@XVAhQQTt>xU>!C>Os6yBI#FN&ks4VdLIKd_kRP^)mu*mB(>5@F;w~Ga{$&f@d_l}k*GecDF1oH(fDjL_sL%(sMV7C! z-A%f>#>hrX7$v*hrQZ^KW7+Y&fRZ0-Gs(QbpNzaGr;;z^4f{kyf$UXVo!C=1EL?~r z^*mOxh{{+>?vy|S-*W_&5O9zXM-KQN^N~k*ZskP*NuUorP zc%DmW^b;Suu!OP;Ms3}~DEt7egg$AXpw`_;pM2VGfa8glr4qd!bZL@jvFB|Co*P3* zK|~P;7Er$^gctB45WpyPVmNQa-zc92({Ak8VKxhG-?v#bdi0?eBW!kcm0l(}YL+iA z(%aE>D1PEjpzjW|YxM*@FKU=g<~Xg%1|AQ%NAGdw0H zYrOA!;AU%Zp~o2|w>Y!S_dH$Y>rHmONw%kiQ_$`|)fDcX!PUX9^hb3zexMI{6dr#S P4CL)MxF($>b^`zaxd%HN literal 0 HcmV?d00001 diff --git a/packages/mimecast/_dev/deploy/docker/files/mail0.gz b/packages/mimecast/_dev/deploy/docker/files/mail0.gz new file mode 100644 index 0000000000000000000000000000000000000000..0897e0686a8fab76f8dcc29b481697351f9f7cd9 GIT binary patch literal 361 zcmV-v0hazBiwFo2$wy}Z18rexY%l^I?l%pD+u~u2cF`GaWa-ZF zs;=M?0Cbnex8MyocZA&zKHH6w=T8CrF=AIq_98#EU% zL)QCtYR0R8=IH#>9pj3lTrG+|7IuvLI{&=6tr%Q4e5-l2!RzK;q+&9SduodHcP;CC zsvkPe-HIw;5{NM5MP(Y-%Q#HcIH38LXJo6gXku0z`zg*)zrV`*ogIdZkDD&$)7|YQ z+RcvWWUJvAZ`vmQ#4oO|Zch1Ts;b|rU)-E8KcL;$8LHzj*y7OPTNbGl&KvA)G+0Lx zFgR;50Rb%Kd>jA*n$v#`z~hoGc&!%O%Z2`kAHE6W_^|B@yYZ#9h2MIkRKGn0fKkSX{Lcbl zBPamCprgXtq=bV9iM3QHZ iSQC^YO5|4s50sov@y`J`U`CQ(1>gr$!)zx41ONa6YV%nD literal 0 HcmV?d00001 diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml index 7396c678b05..a54472851c5 100644 --- a/packages/mimecast/changelog.yml +++ b/packages/mimecast/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.3.0" + changes: + - description: Add `cloud_integrated_logs` data stream. + type: enhancement + link: https://github.com/elastic/integrations/pull/11910 - version: "2.2.0" changes: - description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error". diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log new file mode 100644 index 00000000000..6748648eafe --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log @@ -0,0 +1,20 @@ +{"_offset":1825747,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xz4RX59R6z84lm-ip3sazqogbn7c5haarz3ydiaeg_1732724425","attachments":["1-MB-Test_PCI-PII.xls"],"direction":"INBOUND","historicalMail":false,"messageId":"<38989264-3456-4301-b2f9-22b8c78289db@SJ0PR18MB4090.namprd18.prod.outlook.com>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"127b2e8d1334f481d9dda0fe16530c1de6aede980e5313957516c6adddefddbc_1732724425","recipients":["darron.giron@demovation-ci.b41.one"],"senderEnvelope":"","senderHeader":"MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Undeliverable: Book Request","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732724425453,"type":"entities"} +{"_offset":1803495,"_partition":53,"accountId":"AUS2474","aggregateId":"4XvP1x6xnGzFB8Y-djq8atkief95yfdwmtw8thpgtr_1732206838","attachments":[],"direction":"INBOUND","historicalMail":false,"messageId":"<2470e325-20f1-40fb-ae9a-adf6b3efaddb@BY1PR18MB5971.namprd18.prod.outlook.com>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"e6d23f36b8e86965d62f6d9d94115b5ddbee437f2f5e184a253e57f0b6ed8441_1732206838","recipients":["steven.kean@demovation-ci.b41.one"],"senderEnvelope":"","senderHeader":"MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Undeliverable: Hoecker letter","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732206838263,"type":"entities"} +{"_offset":1807856,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xw1kF1y6dzFyJB-hduwwwo67hmfwau3khzkdbfzo8_1732295253","attachments":[],"direction":"INBOUND","historicalMail":false,"messageId":"","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"e4caf5045cf8c4477d6a685eb9a6a2028ab84dc2f11b0b9ed175b6d4e4de124e_1732295253","recipients":["michael.brown@demovation-ci.b41.one"],"senderEnvelope":"sara.shackleton@demovation-ci.b41.one","senderHeader":"sara.shackleton@demovation-ci.b41.one","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"CONGRATULATIONS","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732295253577,"type":"entities"} +{"_offset":1807567,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xvzwn3R6ZzFB8T-1p4nzntpnrcageetaz5mfk9bm8_1732290393","attachments":["Mimecast Termination.docx"],"direction":"INBOUND","historicalMail":false,"messageId":"","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"793eee52c80fb40134eed640afa87eaaadba4ac194c6cbd2d63fa3e6b75decfc_1732290393","recipients":["tori.kuykendall@demovation-ci.b41.one"],"senderEnvelope":"randall.gay@demovation-ci.b41.one","senderHeader":"randall.gay@demovation-ci.b41.one","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"FW: San Juan Volumes","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732290393848,"type":"entities"} +{"_offset":1790506,"_partition":53,"accountId":"AUS2474","aggregateId":"4XsWdG63WDzFy33-kckoq8rx19ibc7iccjsjx6hsxd_1731943475","attachments":["Sandbox Test.xlsx"],"direction":"INBOUND","historicalMail":false,"messageId":"<0bb5c112-9013-584b-eb87-e5376f49c9ac@demovation-ci.b41.one>","policiesApplied":[{"action":"BLOCK","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"66e03b099e150698fc62f354796f2baa94c2f625a34ac92a0c7e8eb4a2afb11c_1731943475","recipients":["steve.january@demovation-ci.b41.one"],"senderEnvelope":"announcements@demovation-ci.b41.one","senderHeader":"","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Message from Node-RED","subtype":"MALWARE","tags":["MALWARE"],"threatState":"BLOCKED","threatType":"MALWARE","timestamp":1731943475250,"type":"entities"} +{"_offset":1808181,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xw3cr3g5BzFyJM-hw7nutbmioptjwcyjaxx6xiwi8_1732300380","attachments":[],"direction":"INBOUND","historicalMail":false,"messageId":"<871da736-1d6f-49c2-a746-e7bf53efde4f@mtasv.net>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"4632969e69e264e6387667fe5b16b227e500e3f139bad62803ef05dc3db6f7f0_1732300380","recipients":["devin@hamilton231.net"],"senderEnvelope":"pm_bounces@pm-bounces.onmimecast.com","senderHeader":"no-reply@onmimecast.com","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Mimecast quarantined a message detected as phishing from kbishop@summitpulse-enterprises.undefined","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732300380868,"type":"entities"} +{"_offset":1826382,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xz8Fg2f0Vz84mw-gii93d63mdqw55hyxwhokyohhg_1732734727","attachments":[],"direction":"INBOUND","historicalMail":false,"messageId":"<315ecb5a-c9da-4920-beb4-3f9bd929629d@PH0PR18MB3990.namprd18.prod.outlook.com>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"d5e62983f7eb0b7b5eda24df668ef5b77d20adf2801b376dc799d1e44fc2ac52_1732734727","recipients":["vince.kaminski@demovation-ci.b41.one"],"senderEnvelope":"","senderHeader":"MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Undeliverable: Message from Node-RED","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732734727836,"type":"entities"} +{"_offset":1815844,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xxhd273wrzFB8d-thdyqe6dgj83fsus95nwuzydwg_1732530231","attachments":["guess-what-day-it-is-hump-day-whoop-whoop-5af49483875db9003696b1cc.jpg"],"direction":"INBOUND","historicalMail":false,"messageId":"","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"1b9fa7007467b03bcb45824cb283cc3dbaedeca258dd76737e42e40c33fff573_1732530231","recipients":["rahil.jafry@demovation-ci.b41.one"],"senderEnvelope":"","senderHeader":"MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Undeliverable: FW: CMS Opportunity","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732530231824,"type":"entities"} +{"_offset":1826720,"_partition":53,"accountId":"AUS2474","aggregateId":"4XzBMQ1fTlz84nK-5p1346dgpwaugm3gkr13k56agb_1732740434","attachments":[],"direction":"INBOUND","historicalMail":false,"messageId":"<889fdc9f-6e0c-4714-956d-9b20a6e63ab3@PH0PR18MB4734.namprd18.prod.outlook.com>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"b52070e40b85ddc3d74991bb3b9b80de3aa7df0133d927a275a2ca70a9eb1f84_1732740434","recipients":["ruth.brown@demovation-ci.b41.one"],"senderEnvelope":"","senderHeader":"MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Undeliverable: P&D Pilots","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732740434855,"type":"entities"} +{"_offset":1790982,"_partition":53,"accountId":"AUS2474","aggregateId":"4XsZB263RjzFy2g-xjt37dsesi7fddbre96gprgpor_1731950379","attachments":["Screen Shot 2019-05-02 at 8.45.29 AM.png"],"direction":"INBOUND","historicalMail":false,"messageId":"<186ee007-089d-e657-a816-a08217d10b41@demovation-ci.b41.one>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"77f234462461122563d195bec0decddca5e6556bcdfc7c2797e62bda44979f63_1731950379","recipients":["darrell.schoolcraft@demovation-ci.b41.one","susan.scott@demovation-ci.b41.one","michelle.lokay@demovation-ci.b41.one","lee.huber@demovation-ci.b41.one"],"senderEnvelope":"bob.burleson@demovation-ci.b41.one","senderHeader":"bob.burleson@demovation-ci.b41.one","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Panhandle lateral operations","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1731950379429,"type":"entities"} +{"_offset":1822823,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xyccc0QWKz84PS-m1xto1ropcrh1cnp4tb7rzjsgn_1732660000","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"pass","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<1375b951-77d3-4231-9701-d0f7175f5cae@mtasv.net>","processingId":"a3783bd4c73d1d157a366b4d61451f72e559af9251115b3275d6f020526d34d5_1732660000","subtype":null,"timestamp":1732660000405,"type":"mailflow"} +{"_offset":1826870,"_partition":53,"accountId":"AUS2474","aggregateId":"4XzCL50vjVz84QW-1aohygaqrhj4oifhu6wz4xh36b_1732743069","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"pass","type":"DKIM"},{"aligned":null,"result":"fail","type":"DMARC"}],"messageId":"<774faf78-0457-4ac1-9987-fad251c0f445@LV3PR18MB6327.namprd18.prod.outlook.com>","processingId":"88f54ff7a57a7161477ae54bb2d3530f98b93f97ab17d745816bed5ec2475121_1732743069","subtype":null,"timestamp":1732743069761,"type":"mailflow"} +{"_offset":1790380,"_partition":53,"accountId":"AUS2474","aggregateId":"4XsVwP74NKzFyJL-enzpecqrdi9b61fxxktmow9rsd_1731941558","authResults":[{"aligned":false,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<637d63a1-62d7-cd45-f03e-95338cef3c16@demovation-ci.b41.one>","processingId":"b6b6de3fb92174026699081356f2d9af211c654a4e3841a1839bd24f7f8e90ad_1731941558","subtype":null,"timestamp":1731941558336,"type":"mailflow"} +{"_offset":1820809,"_partition":53,"accountId":"AUS2474","aggregateId":"4XyPGk5t8bz84mw-uonit3w88iwq13dqpqtzozhfir_1732629326","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<92b5b2a8-3687-87cd-9988-547b507ab314@demovation-ci.b41.one>","processingId":"46d8b9d3d194a6f9fff30255df9421cf46c2416568d7adeec4baf16a8980e696_1732629327","subtype":null,"timestamp":1732629327226,"type":"mailflow"} +{"_offset":1802955,"_partition":53,"accountId":"AUS2474","aggregateId":"4XvL0W0KRGzFB8k-fxoe7ics1f9uk7y6aiuba5ek8g_1732198651","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"","processingId":"7d5a611de68817a1aa0eac68cdf0fc8dba7878decbd639526ae862010331eb66_1732198651","subtype":null,"timestamp":1732198651453,"type":"mailflow"} +{"_offset":1808128,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xw3KV0jVqzFB8f-z3wa1ajy1jn6cftuepsu47xdob_1732299582","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"pass","type":"DKIM"},{"aligned":null,"result":"fail","type":"DMARC"}],"messageId":"<01c295b3-0110-4882-98a9-d60190c8e57b@CO1PR18MB4571.namprd18.prod.outlook.com>","processingId":"3e98a0a56f48c397d164d90e7129200c2b2e2ee801ee15a2de8e3dc556b99710_1732299582","subtype":null,"timestamp":1732299582518,"type":"mailflow"} +{"_offset":1826377,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xz88p751Yz84nC-a8cr4y85yq3hh6jjczgnn5sgj8_1732734475","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<297a30be-ed87-9385-04b1-ab18769d347f@demovation-ci.b41.one>","processingId":"6a3e4abe35879b7a35f42fb785ddd4e9f0564d3a2584e0902120976e21d4a9ba_1732734475","subtype":null,"timestamp":1732734475302,"type":"mailflow"} +{"_offset":1815702,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xxg1c71KpzFB8T-j9zj9y3f91byjes4ysqetgk19g_1732525893","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<642fa314-514d-2b9b-d4c1-c713421fb4f3@demovation-ci.b41.one>","processingId":"821aff47dea6b57c6cb6bd262738eeabd28e2659f2ac0cb3ee490828d3a143f4_1732525893","subtype":null,"timestamp":1732525893398,"type":"mailflow"} +{"_offset":1820908,"_partition":53,"accountId":"AUS2474","aggregateId":"4XyPl23Kn4z84ly-hnj5fgu9nbwnghx86mj18qp44b_1732630590","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<2e165bc9-cae9-8e0d-3baa-03532f32fbb3@demovation-ci.b41.one>","processingId":"3125eb6ff78c055eb7449a47286a453dcb66e176d2c751a6236bba4232c6fe31_1732630590","subtype":null,"timestamp":1732630590705,"type":"mailflow"} +{"_offset":1803841,"_partition":53,"accountId":"AUS2474","aggregateId":"4XvR1B4m7BzFB8L-qk59b4szrgayciaagczc977rzb_1732212206","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<2ae37333-38e7-89ff-dc36-c8d48c6e3df3@demovation-ci.b41.one>","processingId":"c40337e6860db0301575d8d09362bff214c0b010d6c4d41da9d770759ff54d10_1732212206","subtype":null,"timestamp":1732212206960,"type":"mailflow"} diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log-expected.json b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log-expected.json new file mode 100644 index 00000000000..8eb7938009e --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log-expected.json @@ -0,0 +1,1209 @@ +{ + "expected": [ + { + "@timestamp": "2024-11-27T16:20:25.453Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "attachments": [ + { + "file": { + "name": [ + "1-MB-Test_PCI-PII.xls" + ] + } + } + ], + "direction": "inbound", + "from": { + "address": [ + "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one" + ] + }, + "message_id": "<38989264-3456-4301-b2f9-22b8c78289db@SJ0PR18MB4090.namprd18.prod.outlook.com>", + "subject": "Undeliverable: Book Request", + "to": { + "address": [ + "darron.giron@demovation-ci.b41.one" + ] + } + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-27T16:20:25.453Z", + "original": "{\"_offset\":1825747,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4Xz4RX59R6z84lm-ip3sazqogbn7c5haarz3ydiaeg_1732724425\",\"attachments\":[\"1-MB-Test_PCI-PII.xls\"],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"<38989264-3456-4301-b2f9-22b8c78289db@SJ0PR18MB4090.namprd18.prod.outlook.com>\",\"policiesApplied\":[{\"action\":\"DO_NOTHING\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"127b2e8d1334f481d9dda0fe16530c1de6aede980e5313957516c6adddefddbc_1732724425\",\"recipients\":[\"darron.giron@demovation-ci.b41.one\"],\"senderEnvelope\":\"\",\"senderHeader\":\"MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"Undeliverable: Book Request\",\"subtype\":\"NO_DETECTIONS\",\"tags\":null,\"threatState\":\"DELIVERY_IN_PROGRESS\",\"threatType\":\"NO_DETECTIONS\",\"timestamp\":1732724425453,\"type\":\"entities\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4Xz4RX59R6z84lm-ip3sazqogbn7c5haarz3ydiaeg_1732724425", + "historicalMail": false, + "log_type": "entities", + "policiesApplied": [ + { + "action": "DO_NOTHING", + "mode": "ACTIVE", + "name": "Default O365 Mail policy" + } + ], + "processingId": "127b2e8d1334f481d9dda0fe16530c1de6aede980e5313957516c6adddefddbc_1732724425", + "senderHeader": "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one", + "source": "OFFICE_365_MAIL", + "subtype": "NO_DETECTIONS", + "threatState": "DELIVERY_IN_PROGRESS", + "threatType": "NO_DETECTIONS" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one", + "darron.giron@demovation-ci.b41.one" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-21T16:33:58.263Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "direction": "inbound", + "from": { + "address": [ + "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one" + ] + }, + "message_id": "<2470e325-20f1-40fb-ae9a-adf6b3efaddb@BY1PR18MB5971.namprd18.prod.outlook.com>", + "subject": "Undeliverable: Hoecker letter", + "to": { + "address": [ + "steven.kean@demovation-ci.b41.one" + ] + } + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-21T16:33:58.263Z", + "original": "{\"_offset\":1803495,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XvP1x6xnGzFB8Y-djq8atkief95yfdwmtw8thpgtr_1732206838\",\"attachments\":[],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"<2470e325-20f1-40fb-ae9a-adf6b3efaddb@BY1PR18MB5971.namprd18.prod.outlook.com>\",\"policiesApplied\":[{\"action\":\"DO_NOTHING\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"e6d23f36b8e86965d62f6d9d94115b5ddbee437f2f5e184a253e57f0b6ed8441_1732206838\",\"recipients\":[\"steven.kean@demovation-ci.b41.one\"],\"senderEnvelope\":\"\",\"senderHeader\":\"MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"Undeliverable: Hoecker letter\",\"subtype\":\"NO_DETECTIONS\",\"tags\":null,\"threatState\":\"DELIVERY_IN_PROGRESS\",\"threatType\":\"NO_DETECTIONS\",\"timestamp\":1732206838263,\"type\":\"entities\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XvP1x6xnGzFB8Y-djq8atkief95yfdwmtw8thpgtr_1732206838", + "historicalMail": false, + "log_type": "entities", + "policiesApplied": [ + { + "action": "DO_NOTHING", + "mode": "ACTIVE", + "name": "Default O365 Mail policy" + } + ], + "processingId": "e6d23f36b8e86965d62f6d9d94115b5ddbee437f2f5e184a253e57f0b6ed8441_1732206838", + "senderHeader": "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one", + "source": "OFFICE_365_MAIL", + "subtype": "NO_DETECTIONS", + "threatState": "DELIVERY_IN_PROGRESS", + "threatType": "NO_DETECTIONS" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one", + "steven.kean@demovation-ci.b41.one" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-22T17:07:33.577Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "direction": "inbound", + "from": { + "address": [ + "sara.shackleton@demovation-ci.b41.one" + ] + }, + "message_id": "", + "subject": "CONGRATULATIONS", + "to": { + "address": [ + "michael.brown@demovation-ci.b41.one" + ] + } + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-22T17:07:33.577Z", + "original": "{\"_offset\":1807856,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4Xw1kF1y6dzFyJB-hduwwwo67hmfwau3khzkdbfzo8_1732295253\",\"attachments\":[],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"\",\"policiesApplied\":[{\"action\":\"DO_NOTHING\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"e4caf5045cf8c4477d6a685eb9a6a2028ab84dc2f11b0b9ed175b6d4e4de124e_1732295253\",\"recipients\":[\"michael.brown@demovation-ci.b41.one\"],\"senderEnvelope\":\"sara.shackleton@demovation-ci.b41.one\",\"senderHeader\":\"sara.shackleton@demovation-ci.b41.one\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"CONGRATULATIONS\",\"subtype\":\"NO_DETECTIONS\",\"tags\":null,\"threatState\":\"DELIVERY_IN_PROGRESS\",\"threatType\":\"NO_DETECTIONS\",\"timestamp\":1732295253577,\"type\":\"entities\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4Xw1kF1y6dzFyJB-hduwwwo67hmfwau3khzkdbfzo8_1732295253", + "historicalMail": false, + "log_type": "entities", + "policiesApplied": [ + { + "action": "DO_NOTHING", + "mode": "ACTIVE", + "name": "Default O365 Mail policy" + } + ], + "processingId": "e4caf5045cf8c4477d6a685eb9a6a2028ab84dc2f11b0b9ed175b6d4e4de124e_1732295253", + "senderHeader": "sara.shackleton@demovation-ci.b41.one", + "source": "OFFICE_365_MAIL", + "subtype": "NO_DETECTIONS", + "threatState": "DELIVERY_IN_PROGRESS", + "threatType": "NO_DETECTIONS" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "sara.shackleton@demovation-ci.b41.one", + "michael.brown@demovation-ci.b41.one" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-22T15:46:33.848Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "attachments": [ + { + "file": { + "name": [ + "Mimecast Termination.docx" + ] + } + } + ], + "direction": "inbound", + "from": { + "address": [ + "randall.gay@demovation-ci.b41.one" + ] + }, + "message_id": "", + "subject": "FW: San Juan Volumes", + "to": { + "address": [ + "tori.kuykendall@demovation-ci.b41.one" + ] + } + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-22T15:46:33.848Z", + "original": "{\"_offset\":1807567,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4Xvzwn3R6ZzFB8T-1p4nzntpnrcageetaz5mfk9bm8_1732290393\",\"attachments\":[\"Mimecast Termination.docx\"],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"\",\"policiesApplied\":[{\"action\":\"DO_NOTHING\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"793eee52c80fb40134eed640afa87eaaadba4ac194c6cbd2d63fa3e6b75decfc_1732290393\",\"recipients\":[\"tori.kuykendall@demovation-ci.b41.one\"],\"senderEnvelope\":\"randall.gay@demovation-ci.b41.one\",\"senderHeader\":\"randall.gay@demovation-ci.b41.one\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"FW: San Juan Volumes\",\"subtype\":\"NO_DETECTIONS\",\"tags\":null,\"threatState\":\"DELIVERY_IN_PROGRESS\",\"threatType\":\"NO_DETECTIONS\",\"timestamp\":1732290393848,\"type\":\"entities\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4Xvzwn3R6ZzFB8T-1p4nzntpnrcageetaz5mfk9bm8_1732290393", + "historicalMail": false, + "log_type": "entities", + "policiesApplied": [ + { + "action": "DO_NOTHING", + "mode": "ACTIVE", + "name": "Default O365 Mail policy" + } + ], + "processingId": "793eee52c80fb40134eed640afa87eaaadba4ac194c6cbd2d63fa3e6b75decfc_1732290393", + "senderHeader": "randall.gay@demovation-ci.b41.one", + "source": "OFFICE_365_MAIL", + "subtype": "NO_DETECTIONS", + "threatState": "DELIVERY_IN_PROGRESS", + "threatType": "NO_DETECTIONS" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "randall.gay@demovation-ci.b41.one", + "tori.kuykendall@demovation-ci.b41.one" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-18T15:24:35.250Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "attachments": [ + { + "file": { + "name": [ + "Sandbox Test.xlsx" + ] + } + } + ], + "direction": "inbound", + "from": { + "address": [ + "announcements@demovation-ci.b41.one" + ] + }, + "message_id": "<0bb5c112-9013-584b-eb87-e5376f49c9ac@demovation-ci.b41.one>", + "subject": "Message from Node-RED", + "to": { + "address": [ + "steve.january@demovation-ci.b41.one" + ] + } + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-18T15:24:35.250Z", + "original": "{\"_offset\":1790506,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XsWdG63WDzFy33-kckoq8rx19ibc7iccjsjx6hsxd_1731943475\",\"attachments\":[\"Sandbox Test.xlsx\"],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"<0bb5c112-9013-584b-eb87-e5376f49c9ac@demovation-ci.b41.one>\",\"policiesApplied\":[{\"action\":\"BLOCK\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"66e03b099e150698fc62f354796f2baa94c2f625a34ac92a0c7e8eb4a2afb11c_1731943475\",\"recipients\":[\"steve.january@demovation-ci.b41.one\"],\"senderEnvelope\":\"announcements@demovation-ci.b41.one\",\"senderHeader\":\"\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"Message from Node-RED\",\"subtype\":\"MALWARE\",\"tags\":[\"MALWARE\"],\"threatState\":\"BLOCKED\",\"threatType\":\"MALWARE\",\"timestamp\":1731943475250,\"type\":\"entities\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XsWdG63WDzFy33-kckoq8rx19ibc7iccjsjx6hsxd_1731943475", + "historicalMail": false, + "log_type": "entities", + "policiesApplied": [ + { + "action": "BLOCK", + "mode": "ACTIVE", + "name": "Default O365 Mail policy" + } + ], + "processingId": "66e03b099e150698fc62f354796f2baa94c2f625a34ac92a0c7e8eb4a2afb11c_1731943475", + "source": "OFFICE_365_MAIL", + "subtype": "MALWARE", + "tags": [ + "MALWARE" + ], + "threatState": "BLOCKED", + "threatType": "MALWARE" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "announcements@demovation-ci.b41.one", + "steve.january@demovation-ci.b41.one" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-22T18:33:00.868Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "direction": "inbound", + "from": { + "address": [ + "no-reply@onmimecast.com", + "pm_bounces@pm-bounces.onmimecast.com" + ] + }, + "message_id": "<871da736-1d6f-49c2-a746-e7bf53efde4f@mtasv.net>", + "subject": "Mimecast quarantined a message detected as phishing from kbishop@summitpulse-enterprises.undefined", + "to": { + "address": [ + "devin@hamilton231.net" + ] + } + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-22T18:33:00.868Z", + "original": "{\"_offset\":1808181,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4Xw3cr3g5BzFyJM-hw7nutbmioptjwcyjaxx6xiwi8_1732300380\",\"attachments\":[],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"<871da736-1d6f-49c2-a746-e7bf53efde4f@mtasv.net>\",\"policiesApplied\":[{\"action\":\"DO_NOTHING\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"4632969e69e264e6387667fe5b16b227e500e3f139bad62803ef05dc3db6f7f0_1732300380\",\"recipients\":[\"devin@hamilton231.net\"],\"senderEnvelope\":\"pm_bounces@pm-bounces.onmimecast.com\",\"senderHeader\":\"no-reply@onmimecast.com\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"Mimecast quarantined a message detected as phishing from kbishop@summitpulse-enterprises.undefined\",\"subtype\":\"NO_DETECTIONS\",\"tags\":null,\"threatState\":\"DELIVERY_IN_PROGRESS\",\"threatType\":\"NO_DETECTIONS\",\"timestamp\":1732300380868,\"type\":\"entities\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4Xw3cr3g5BzFyJM-hw7nutbmioptjwcyjaxx6xiwi8_1732300380", + "historicalMail": false, + "log_type": "entities", + "policiesApplied": [ + { + "action": "DO_NOTHING", + "mode": "ACTIVE", + "name": "Default O365 Mail policy" + } + ], + "processingId": "4632969e69e264e6387667fe5b16b227e500e3f139bad62803ef05dc3db6f7f0_1732300380", + "senderHeader": "no-reply@onmimecast.com", + "source": "OFFICE_365_MAIL", + "subtype": "NO_DETECTIONS", + "threatState": "DELIVERY_IN_PROGRESS", + "threatType": "NO_DETECTIONS" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "no-reply@onmimecast.com", + "pm_bounces@pm-bounces.onmimecast.com", + "devin@hamilton231.net" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-27T19:12:07.836Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "direction": "inbound", + "from": { + "address": [ + "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one" + ] + }, + "message_id": "<315ecb5a-c9da-4920-beb4-3f9bd929629d@PH0PR18MB3990.namprd18.prod.outlook.com>", + "subject": "Undeliverable: Message from Node-RED", + "to": { + "address": [ + "vince.kaminski@demovation-ci.b41.one" + ] + } + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-27T19:12:07.836Z", + "original": "{\"_offset\":1826382,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4Xz8Fg2f0Vz84mw-gii93d63mdqw55hyxwhokyohhg_1732734727\",\"attachments\":[],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"<315ecb5a-c9da-4920-beb4-3f9bd929629d@PH0PR18MB3990.namprd18.prod.outlook.com>\",\"policiesApplied\":[{\"action\":\"DO_NOTHING\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"d5e62983f7eb0b7b5eda24df668ef5b77d20adf2801b376dc799d1e44fc2ac52_1732734727\",\"recipients\":[\"vince.kaminski@demovation-ci.b41.one\"],\"senderEnvelope\":\"\",\"senderHeader\":\"MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"Undeliverable: Message from Node-RED\",\"subtype\":\"NO_DETECTIONS\",\"tags\":null,\"threatState\":\"DELIVERY_IN_PROGRESS\",\"threatType\":\"NO_DETECTIONS\",\"timestamp\":1732734727836,\"type\":\"entities\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4Xz8Fg2f0Vz84mw-gii93d63mdqw55hyxwhokyohhg_1732734727", + "historicalMail": false, + "log_type": "entities", + "policiesApplied": [ + { + "action": "DO_NOTHING", + "mode": "ACTIVE", + "name": "Default O365 Mail policy" + } + ], + "processingId": "d5e62983f7eb0b7b5eda24df668ef5b77d20adf2801b376dc799d1e44fc2ac52_1732734727", + "senderHeader": "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one", + "source": "OFFICE_365_MAIL", + "subtype": "NO_DETECTIONS", + "threatState": "DELIVERY_IN_PROGRESS", + "threatType": "NO_DETECTIONS" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one", + "vince.kaminski@demovation-ci.b41.one" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-25T10:23:51.824Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "attachments": [ + { + "file": { + "name": [ + "guess-what-day-it-is-hump-day-whoop-whoop-5af49483875db9003696b1cc.jpg" + ] + } + } + ], + "direction": "inbound", + "from": { + "address": [ + "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one" + ] + }, + "message_id": "", + "subject": "Undeliverable: FW: CMS Opportunity", + "to": { + "address": [ + "rahil.jafry@demovation-ci.b41.one" + ] + } + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-25T10:23:51.824Z", + "original": "{\"_offset\":1815844,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4Xxhd273wrzFB8d-thdyqe6dgj83fsus95nwuzydwg_1732530231\",\"attachments\":[\"guess-what-day-it-is-hump-day-whoop-whoop-5af49483875db9003696b1cc.jpg\"],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"\",\"policiesApplied\":[{\"action\":\"DO_NOTHING\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"1b9fa7007467b03bcb45824cb283cc3dbaedeca258dd76737e42e40c33fff573_1732530231\",\"recipients\":[\"rahil.jafry@demovation-ci.b41.one\"],\"senderEnvelope\":\"\",\"senderHeader\":\"MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"Undeliverable: FW: CMS Opportunity\",\"subtype\":\"NO_DETECTIONS\",\"tags\":null,\"threatState\":\"DELIVERY_IN_PROGRESS\",\"threatType\":\"NO_DETECTIONS\",\"timestamp\":1732530231824,\"type\":\"entities\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4Xxhd273wrzFB8d-thdyqe6dgj83fsus95nwuzydwg_1732530231", + "historicalMail": false, + "log_type": "entities", + "policiesApplied": [ + { + "action": "DO_NOTHING", + "mode": "ACTIVE", + "name": "Default O365 Mail policy" + } + ], + "processingId": "1b9fa7007467b03bcb45824cb283cc3dbaedeca258dd76737e42e40c33fff573_1732530231", + "senderHeader": "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one", + "source": "OFFICE_365_MAIL", + "subtype": "NO_DETECTIONS", + "threatState": "DELIVERY_IN_PROGRESS", + "threatType": "NO_DETECTIONS" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one", + "rahil.jafry@demovation-ci.b41.one" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-27T20:47:14.855Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "direction": "inbound", + "from": { + "address": [ + "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one" + ] + }, + "message_id": "<889fdc9f-6e0c-4714-956d-9b20a6e63ab3@PH0PR18MB4734.namprd18.prod.outlook.com>", + "subject": "Undeliverable: P&D Pilots", + "to": { + "address": [ + "ruth.brown@demovation-ci.b41.one" + ] + } + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-27T20:47:14.855Z", + "original": "{\"_offset\":1826720,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XzBMQ1fTlz84nK-5p1346dgpwaugm3gkr13k56agb_1732740434\",\"attachments\":[],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"<889fdc9f-6e0c-4714-956d-9b20a6e63ab3@PH0PR18MB4734.namprd18.prod.outlook.com>\",\"policiesApplied\":[{\"action\":\"DO_NOTHING\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"b52070e40b85ddc3d74991bb3b9b80de3aa7df0133d927a275a2ca70a9eb1f84_1732740434\",\"recipients\":[\"ruth.brown@demovation-ci.b41.one\"],\"senderEnvelope\":\"\",\"senderHeader\":\"MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"Undeliverable: P&D Pilots\",\"subtype\":\"NO_DETECTIONS\",\"tags\":null,\"threatState\":\"DELIVERY_IN_PROGRESS\",\"threatType\":\"NO_DETECTIONS\",\"timestamp\":1732740434855,\"type\":\"entities\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XzBMQ1fTlz84nK-5p1346dgpwaugm3gkr13k56agb_1732740434", + "historicalMail": false, + "log_type": "entities", + "policiesApplied": [ + { + "action": "DO_NOTHING", + "mode": "ACTIVE", + "name": "Default O365 Mail policy" + } + ], + "processingId": "b52070e40b85ddc3d74991bb3b9b80de3aa7df0133d927a275a2ca70a9eb1f84_1732740434", + "senderHeader": "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one", + "source": "OFFICE_365_MAIL", + "subtype": "NO_DETECTIONS", + "threatState": "DELIVERY_IN_PROGRESS", + "threatType": "NO_DETECTIONS" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "MicrosoftExchange329e71ec88ae4615bbc36ab6ce41109e@mc-ci.b41.one", + "ruth.brown@demovation-ci.b41.one" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-18T17:19:39.429Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "attachments": [ + { + "file": { + "name": [ + "Screen Shot 2019-05-02 at 8.45.29 AM.png" + ] + } + } + ], + "direction": "inbound", + "from": { + "address": [ + "bob.burleson@demovation-ci.b41.one" + ] + }, + "message_id": "<186ee007-089d-e657-a816-a08217d10b41@demovation-ci.b41.one>", + "subject": "Panhandle lateral operations", + "to": { + "address": [ + "darrell.schoolcraft@demovation-ci.b41.one", + "susan.scott@demovation-ci.b41.one", + "michelle.lokay@demovation-ci.b41.one", + "lee.huber@demovation-ci.b41.one" + ] + } + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-18T17:19:39.429Z", + "original": "{\"_offset\":1790982,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XsZB263RjzFy2g-xjt37dsesi7fddbre96gprgpor_1731950379\",\"attachments\":[\"Screen Shot 2019-05-02 at 8.45.29 AM.png\"],\"direction\":\"INBOUND\",\"historicalMail\":false,\"messageId\":\"<186ee007-089d-e657-a816-a08217d10b41@demovation-ci.b41.one>\",\"policiesApplied\":[{\"action\":\"DO_NOTHING\",\"mode\":\"ACTIVE\",\"name\":\"Default O365 Mail policy\"}],\"processingId\":\"77f234462461122563d195bec0decddca5e6556bcdfc7c2797e62bda44979f63_1731950379\",\"recipients\":[\"darrell.schoolcraft@demovation-ci.b41.one\",\"susan.scott@demovation-ci.b41.one\",\"michelle.lokay@demovation-ci.b41.one\",\"lee.huber@demovation-ci.b41.one\"],\"senderEnvelope\":\"bob.burleson@demovation-ci.b41.one\",\"senderHeader\":\"bob.burleson@demovation-ci.b41.one\",\"senderIp\":\"81.2.69.144\",\"source\":\"OFFICE_365_MAIL\",\"subject\":\"Panhandle lateral operations\",\"subtype\":\"NO_DETECTIONS\",\"tags\":null,\"threatState\":\"DELIVERY_IN_PROGRESS\",\"threatType\":\"NO_DETECTIONS\",\"timestamp\":1731950379429,\"type\":\"entities\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XsZB263RjzFy2g-xjt37dsesi7fddbre96gprgpor_1731950379", + "historicalMail": false, + "log_type": "entities", + "policiesApplied": [ + { + "action": "DO_NOTHING", + "mode": "ACTIVE", + "name": "Default O365 Mail policy" + } + ], + "processingId": "77f234462461122563d195bec0decddca5e6556bcdfc7c2797e62bda44979f63_1731950379", + "senderHeader": "bob.burleson@demovation-ci.b41.one", + "source": "OFFICE_365_MAIL", + "subtype": "NO_DETECTIONS", + "threatState": "DELIVERY_IN_PROGRESS", + "threatType": "NO_DETECTIONS" + }, + "related": { + "ip": [ + "81.2.69.144" + ], + "user": [ + "bob.burleson@demovation-ci.b41.one", + "darrell.schoolcraft@demovation-ci.b41.one", + "susan.scott@demovation-ci.b41.one", + "michelle.lokay@demovation-ci.b41.one", + "lee.huber@demovation-ci.b41.one" + ] + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-26T22:26:40.405Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "message_id": "<1375b951-77d3-4231-9701-d0f7175f5cae@mtasv.net>" + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-26T22:26:40.405Z", + "original": "{\"_offset\":1822823,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4Xyccc0QWKz84PS-m1xto1ropcrh1cnp4tb7rzjsgn_1732660000\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"pass\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"pass\",\"type\":\"DMARC\"}],\"messageId\":\"<1375b951-77d3-4231-9701-d0f7175f5cae@mtasv.net>\",\"processingId\":\"a3783bd4c73d1d157a366b4d61451f72e559af9251115b3275d6f020526d34d5_1732660000\",\"subtype\":null,\"timestamp\":1732660000405,\"type\":\"mailflow\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4Xyccc0QWKz84PS-m1xto1ropcrh1cnp4tb7rzjsgn_1732660000", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "pass", + "type": "DKIM" + }, + { + "result": "pass", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "a3783bd4c73d1d157a366b4d61451f72e559af9251115b3275d6f020526d34d5_1732660000" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-27T21:31:09.761Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "message_id": "<774faf78-0457-4ac1-9987-fad251c0f445@LV3PR18MB6327.namprd18.prod.outlook.com>" + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-27T21:31:09.761Z", + "original": "{\"_offset\":1826870,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XzCL50vjVz84QW-1aohygaqrhj4oifhu6wz4xh36b_1732743069\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"pass\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"fail\",\"type\":\"DMARC\"}],\"messageId\":\"<774faf78-0457-4ac1-9987-fad251c0f445@LV3PR18MB6327.namprd18.prod.outlook.com>\",\"processingId\":\"88f54ff7a57a7161477ae54bb2d3530f98b93f97ab17d745816bed5ec2475121_1732743069\",\"subtype\":null,\"timestamp\":1732743069761,\"type\":\"mailflow\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XzCL50vjVz84QW-1aohygaqrhj4oifhu6wz4xh36b_1732743069", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "pass", + "type": "DKIM" + }, + { + "result": "fail", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "88f54ff7a57a7161477ae54bb2d3530f98b93f97ab17d745816bed5ec2475121_1732743069" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-18T14:52:38.336Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "message_id": "<637d63a1-62d7-cd45-f03e-95338cef3c16@demovation-ci.b41.one>" + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-18T14:52:38.336Z", + "original": "{\"_offset\":1790380,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XsVwP74NKzFyJL-enzpecqrdi9b61fxxktmow9rsd_1731941558\",\"authResults\":[{\"aligned\":false,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"none\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"pass\",\"type\":\"DMARC\"}],\"messageId\":\"<637d63a1-62d7-cd45-f03e-95338cef3c16@demovation-ci.b41.one>\",\"processingId\":\"b6b6de3fb92174026699081356f2d9af211c654a4e3841a1839bd24f7f8e90ad_1731941558\",\"subtype\":null,\"timestamp\":1731941558336,\"type\":\"mailflow\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XsVwP74NKzFyJL-enzpecqrdi9b61fxxktmow9rsd_1731941558", + "authResults": [ + { + "aligned": false, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "none", + "type": "DKIM" + }, + { + "result": "pass", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "b6b6de3fb92174026699081356f2d9af211c654a4e3841a1839bd24f7f8e90ad_1731941558" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-26T13:55:27.226Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "message_id": "<92b5b2a8-3687-87cd-9988-547b507ab314@demovation-ci.b41.one>" + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-26T13:55:27.226Z", + "original": "{\"_offset\":1820809,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XyPGk5t8bz84mw-uonit3w88iwq13dqpqtzozhfir_1732629326\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"none\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"pass\",\"type\":\"DMARC\"}],\"messageId\":\"<92b5b2a8-3687-87cd-9988-547b507ab314@demovation-ci.b41.one>\",\"processingId\":\"46d8b9d3d194a6f9fff30255df9421cf46c2416568d7adeec4baf16a8980e696_1732629327\",\"subtype\":null,\"timestamp\":1732629327226,\"type\":\"mailflow\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XyPGk5t8bz84mw-uonit3w88iwq13dqpqtzozhfir_1732629326", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "none", + "type": "DKIM" + }, + { + "result": "pass", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "46d8b9d3d194a6f9fff30255df9421cf46c2416568d7adeec4baf16a8980e696_1732629327" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-21T14:17:31.453Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "message_id": "" + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-21T14:17:31.453Z", + "original": "{\"_offset\":1802955,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XvL0W0KRGzFB8k-fxoe7ics1f9uk7y6aiuba5ek8g_1732198651\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"none\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"pass\",\"type\":\"DMARC\"}],\"messageId\":\"\",\"processingId\":\"7d5a611de68817a1aa0eac68cdf0fc8dba7878decbd639526ae862010331eb66_1732198651\",\"subtype\":null,\"timestamp\":1732198651453,\"type\":\"mailflow\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XvL0W0KRGzFB8k-fxoe7ics1f9uk7y6aiuba5ek8g_1732198651", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "none", + "type": "DKIM" + }, + { + "result": "pass", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "7d5a611de68817a1aa0eac68cdf0fc8dba7878decbd639526ae862010331eb66_1732198651" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-22T18:19:42.518Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "message_id": "<01c295b3-0110-4882-98a9-d60190c8e57b@CO1PR18MB4571.namprd18.prod.outlook.com>" + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-22T18:19:42.518Z", + "original": "{\"_offset\":1808128,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4Xw3KV0jVqzFB8f-z3wa1ajy1jn6cftuepsu47xdob_1732299582\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"pass\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"fail\",\"type\":\"DMARC\"}],\"messageId\":\"<01c295b3-0110-4882-98a9-d60190c8e57b@CO1PR18MB4571.namprd18.prod.outlook.com>\",\"processingId\":\"3e98a0a56f48c397d164d90e7129200c2b2e2ee801ee15a2de8e3dc556b99710_1732299582\",\"subtype\":null,\"timestamp\":1732299582518,\"type\":\"mailflow\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4Xw3KV0jVqzFB8f-z3wa1ajy1jn6cftuepsu47xdob_1732299582", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "pass", + "type": "DKIM" + }, + { + "result": "fail", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "3e98a0a56f48c397d164d90e7129200c2b2e2ee801ee15a2de8e3dc556b99710_1732299582" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-27T19:07:55.302Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "message_id": "<297a30be-ed87-9385-04b1-ab18769d347f@demovation-ci.b41.one>" + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-27T19:07:55.302Z", + "original": "{\"_offset\":1826377,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4Xz88p751Yz84nC-a8cr4y85yq3hh6jjczgnn5sgj8_1732734475\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"none\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"pass\",\"type\":\"DMARC\"}],\"messageId\":\"<297a30be-ed87-9385-04b1-ab18769d347f@demovation-ci.b41.one>\",\"processingId\":\"6a3e4abe35879b7a35f42fb785ddd4e9f0564d3a2584e0902120976e21d4a9ba_1732734475\",\"subtype\":null,\"timestamp\":1732734475302,\"type\":\"mailflow\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4Xz88p751Yz84nC-a8cr4y85yq3hh6jjczgnn5sgj8_1732734475", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "none", + "type": "DKIM" + }, + { + "result": "pass", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "6a3e4abe35879b7a35f42fb785ddd4e9f0564d3a2584e0902120976e21d4a9ba_1732734475" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-25T09:11:33.398Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "message_id": "<642fa314-514d-2b9b-d4c1-c713421fb4f3@demovation-ci.b41.one>" + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-25T09:11:33.398Z", + "original": "{\"_offset\":1815702,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4Xxg1c71KpzFB8T-j9zj9y3f91byjes4ysqetgk19g_1732525893\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"none\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"pass\",\"type\":\"DMARC\"}],\"messageId\":\"<642fa314-514d-2b9b-d4c1-c713421fb4f3@demovation-ci.b41.one>\",\"processingId\":\"821aff47dea6b57c6cb6bd262738eeabd28e2659f2ac0cb3ee490828d3a143f4_1732525893\",\"subtype\":null,\"timestamp\":1732525893398,\"type\":\"mailflow\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4Xxg1c71KpzFB8T-j9zj9y3f91byjes4ysqetgk19g_1732525893", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "none", + "type": "DKIM" + }, + { + "result": "pass", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "821aff47dea6b57c6cb6bd262738eeabd28e2659f2ac0cb3ee490828d3a143f4_1732525893" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-26T14:16:30.705Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "message_id": "<2e165bc9-cae9-8e0d-3baa-03532f32fbb3@demovation-ci.b41.one>" + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-26T14:16:30.705Z", + "original": "{\"_offset\":1820908,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XyPl23Kn4z84ly-hnj5fgu9nbwnghx86mj18qp44b_1732630590\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"none\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"pass\",\"type\":\"DMARC\"}],\"messageId\":\"<2e165bc9-cae9-8e0d-3baa-03532f32fbb3@demovation-ci.b41.one>\",\"processingId\":\"3125eb6ff78c055eb7449a47286a453dcb66e176d2c751a6236bba4232c6fe31_1732630590\",\"subtype\":null,\"timestamp\":1732630590705,\"type\":\"mailflow\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XyPl23Kn4z84ly-hnj5fgu9nbwnghx86mj18qp44b_1732630590", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "none", + "type": "DKIM" + }, + { + "result": "pass", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "3125eb6ff78c055eb7449a47286a453dcb66e176d2c751a6236bba4232c6fe31_1732630590" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-11-21T18:03:26.960Z", + "ecs": { + "version": "8.11.0" + }, + "email": { + "message_id": "<2ae37333-38e7-89ff-dc36-c8d48c6e3df3@demovation-ci.b41.one>" + }, + "event": { + "category": [ + "email" + ], + "created": "2024-11-21T18:03:26.960Z", + "original": "{\"_offset\":1803841,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XvR1B4m7BzFB8L-qk59b4szrgayciaagczc977rzb_1732212206\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"none\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"pass\",\"type\":\"DMARC\"}],\"messageId\":\"<2ae37333-38e7-89ff-dc36-c8d48c6e3df3@demovation-ci.b41.one>\",\"processingId\":\"c40337e6860db0301575d8d09362bff214c0b010d6c4d41da9d770759ff54d10_1732212206\",\"subtype\":null,\"timestamp\":1732212206960,\"type\":\"mailflow\"}" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XvR1B4m7BzFB8L-qk59b4szrgayciaagczc977rzb_1732212206", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "none", + "type": "DKIM" + }, + { + "result": "pass", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "c40337e6860db0301575d8d09362bff214c0b010d6c4d41da9d770759ff54d10_1732212206" + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-common-config.yml b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..4da22641654 --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_original_event diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/system/test-default-config.yml b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..1ccac5b5368 --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/_dev/test/system/test-default-config.yml @@ -0,0 +1,17 @@ +input: cel +service: mimecast +vars: + api_url: http://{{Hostname}}:{{Port}} + client_id: slightlysecretclientid + client_secret: totallysecretlientsecret +data_stream: + vars: + initial_interval: 24h + batch_size: 2 + preserve_original_event: true + enable_request_tracer: true + types: + - "entities" + - "mailflow" +assert: + hit_count: 4 diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/cloud_integrated_logs/agent/stream/cel.yml.hbs new file mode 100644 index 00000000000..8f39fb32c8a --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/agent/stream/cel.yml.hbs @@ -0,0 +1,177 @@ +config_version: 2 +interval: {{interval}} +{{#if enable_request_tracer}} +resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson" +resource.tracer.maxbackups: 5 +{{/if}} +resource.url: {{api_url}} +fields_under_root: true +keep_null: true +state: + client_id: {{client_id}} + client_secret: {{client_secret}} + page_size: {{batch_size}} + look_back: {{initial_interval}} + path: /siem/v1/batch/events/ci + start_field: dateRangeStartsAt + end_field: dateRangeEndsAt +{{#if types}} + types: +{{#each types as |t|}} + - {{t}} +{{/each}} +{{/if}} +redact: + fields: + - client_id + - client_secret + - token.access_token +program: | + // This program is shared between cloud_integrated_logs and siem_logs + // If it is changed here changes should be reflected in the other data + // streams. Do not differentiate the logic between these data streams + // lightly; use the state variable for this unless absolutely required. + state.with( + ( + (has(state.?token.expires) && now() < timestamp(state.token.expires)) ? + // The token we have is still valid. + state.token + : + // Get a new token. + post_request(state.url.trim_right("/") + "/oauth/token", "application/x-www-form-urlencoded", + { + "client_id": [state.client_id], + "client_secret": [state.client_secret], + "grant_type": ["client_credentials"], + }.format_query() + ).do_request().as(auth, auth.StatusCode == 200 ? + bytes(auth.Body).decode_json().as(auth_body, auth_body.with({ + // Include 60s grace period to avoid attempting to make + // a request with a stale authentication token. + "expires": now()+duration(string(int(auth_body.expires_in)-60)+"s"), + })) + : + { + "events": { + "error": { + "code": string(auth.StatusCode), + "id": string(auth.Status), + "message": "POST /oauth/token: "+( + size(auth.Body) != 0 ? + string(auth.Body) + : + string(auth.Status) + ' (' + string(auth.StatusCode) + ')' + ), + }, + }, + "want_more": false, + } + ) + ).as(token, !has(token.access_token) ? token : + state.?cursor.work_list.orValue(state.types.map(t, {"type": t})).as(work_list, size(work_list) == 0 ? + state.types.map(t, {"type": t}) + : + work_list + ).as(work_list, + get_request( + state.url.trim_right("/") + state.path + "?" + { + "type": [work_list[0].type], + ?"nextPage": work_list[0].?next.optMap(next, [next]), + ?"dateRangeStartsAt": state.?start.optMap(start, [start.format("2006-01-02")]), + ?"dateRangeEndsAt": state.?end.optMap(end, [end.format("2006-01-02")]), + ?"pageSize": state.?page_size.optMap(size, [string(int(size))]), + }.format_query() + ).with({ + "Header": { + "Authorization": ["Bearer " + token.access_token], + "Accept": ["application/json"], + "Content-Type": ["application/json"], + } + }).do_request().as(resp, resp.StatusCode == 200 ? + bytes(resp.Body).decode_json().as(body, + { + "events": body.value.map(b, has(b.url), + get(b.url).as(batch, batch.StatusCode == 200 ? + bytes(batch.Body).mime("application/gzip").mime("application/x-ndjson").map(e, + { + "message": dyn(e.encode_json()), + } + ) + : + [{ + "error": { + "code": string(batch.StatusCode), + "id": string(batch.Status), + "message": "GET " + b.url + ": " + ( + size(batch.Body) != 0 ? + string(batch.Body) + : + string(batch.Status) + ' (' + string(batch.StatusCode) + ')' + ), + }, + }] + ) + ).flatten(), + "cursor": { + "work_list": ( + "@nextPage" in body && size(body.value) != 0 ? + [work_list[0].with({"next": body["@nextPage"]})] + : + [] + ) + tail(work_list), + }, + "token": { + "access_token": token.access_token, + "expires": token.expires, + }, + "want_more": "@nextPage" in body && size(body.value) != 0, + }.as(to_publish, to_publish.with({ + "want_more": to_publish.want_more || size(to_publish.cursor.work_list) != 0, + })) + ).as(state, + // Check whether we still need to get more, but have + // no event for this type. If we do, populate events + // with a place-holder to be discarded by the ingest + // pipeline. + state.want_more && size(state.events) == 0 ? + state.with({"events": [{"message": "want_more"}]}) + : + state + ) + : + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "GET " + state.path + ": " + ( + size(resp.Body) != 0 ? + string(resp.Body) + : + string(resp.Status) + ' (' + string(resp.StatusCode) + ')' + ), + }, + }, + "want_more": false, + } + ) + ) + ) + ) +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..5064b4ac127 --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,278 @@ +--- +description: Pipeline for processing Mimecast cloud_integrated_logs. +processors: + - drop: + description: Ignore want_more placeholder message. + if: ctx.message == "want_more" + + - set: + field: ecs.version + value: '8.11.0' + - rename: + field: message + target_field: event.original + ignore_missing: true + if: ctx.event?.original == null + - json: + description: Parse 'message' JSON contents into a 'mimecast' object. + field: event.original + target_field: mimecast + - drop: + if: ctx?.mimecast?.datetime == null && ctx?.mimecast?.timestamp == null + - set: + field: event.category + value: [email] + + - date: + description: Use 'mimecast.timestamp' as the '@timestamp' + field: mimecast.timestamp + if: ctx.mimecast?.timestamp != null + timezone: UTC + formats: + - UNIX_MS + - set: + field: event.created + copy_from: '@timestamp' + if: ctx['@timestamp'] != null + + ### NOTE LOG TYPE + - rename: + field: mimecast.type + target_field: mimecast.log_type + ignore_missing: true + + - set: + field: original + copy_from: mimecast + tag: preserve_custom_duplicate_fields + ignore_empty_value: true + if: ctx.tags?.contains('preserve_duplicate_custom_fields') == true + + - fingerprint: + fields: + - mimecast.messageId + - mimecast.processingId + - mimecast.aggregateId + - mimecast.accountId + - mimecast.timestamp + - mimecast.action + - mimecast.log_type + - mimecast.subtype + target_field: "_id" + ignore_missing: true + + ### Common + - foreach: + if: ctx.mimecast?.recipients instanceof List + field: mimecast.recipients + processor: + append: + field: email.to.address + value: '{{{_ingest._value}}}' + allow_duplicates: false + - append: + field: email.to.address + value: '{{{mimecast.recipients}}}' + allow_duplicates: false + if: 'ctx.mimecast?.recipients != null && !(ctx.mimecast.recipients instanceof List)' + + ### ENTITIES + - rename: + field: mimecast.senderIp + target_field: source.ip + ignore_missing: true + - rename: + field: mimecast.messageId + target_field: email.message_id + ignore_missing: true + - append: + field: email.from.address + value: '{{{mimecast.senderHeader}}}' + allow_duplicates: false + if: ctx.mimecast?.senderHeader != null + - append: + field: email.from.address + value: '{{{mimecast.senderEnvelope}}}' + allow_duplicates: false + if: ctx.mimecast?.senderEnvelope != null + - rename: + field: mimecast.subject + target_field: email.subject + ignore_missing: true + - rename: + field: mimecast.direction + target_field: email.direction + ignore_missing: true + - lowercase: + field: email.direction + ignore_missing: true + - rename: + field: mimecast.attachments + target_field: email.attachments.file.name + ignore_missing: true + - rename: + field: mimecast.action + target_field: mimecast.threatState + ignore_missing: true + + ### MAIL FLOW LOGS + # none + + ### URL LOGS + - rename: + field: mimecast.action + target_field: event.action + ignore_missing: true + - rename: + field: mimecast.sourceIp + target_field: source.ip + ignore_missing: true + + - convert: + field: source.ip + type: ip + ignore_missing: true + on_failure: + - append: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - remove: + field: source.ip + ignore_failure: true + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + + - foreach: + field: email.from.address + if: ctx.email?.from?.address instanceof List + ignore_failure: true + processor: + append: + field: related.user + tag: append_email_from_address_list_to_related_user + value: '{{{_ingest._value}}}' + allow_duplicates: false + - foreach: + field: email.to.address + if: ctx.email?.to?.address instanceof List + ignore_failure: true + processor: + append: + field: related.user + tag: append_email_from_address_list_to_related_user + value: '{{{_ingest._value}}}' + allow_duplicates: false + - append: + if: ctx.email?.from?.address != null && !(ctx.email.from.address instanceof List) && ctx.email.from.address != '' + field: related.user + tag: append_email_from_address_to_related_user + value: '{{{email.from.address}}}' + allow_duplicates: false + - append: + if: ctx.source?.ip != null + field: related.ip + tag: append_source_ip_to_related_ip + value: '{{{source.ip}}}' + allow_duplicates: false + + # Handle attachment details. + - foreach: + tag: collect_attachment_file_hashes + if: ctx.email?.attachments?.file?.hash instanceof Map + field: email.attachments.file.hash + processor: + append: + field: related.hash + value: '{{{_ingest._value}}}' + allow_duplicates: false + - script: + tag: promote_email_attachments_to_array + if: ctx.email?.attachments != null + lang: painless + source: | + def attachments = []; + attachments.add(ctx.email.attachments); + ctx.email.attachments = attachments; + + - remove: + tag: remove_duplicate_custom_fields + if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') + field: + - mimecast.recipients + - mimecast.senderEnvelope + ignore_missing: true + + # Cleanup + - rename: + field: original + target_field: mimecast + override: true + ignore_missing: true + - remove: + description: Cleanup of repeated/unwanted/temporary fields. + field: + - mimecast._offset + - mimecast._partition + - mimecast.timestamp + ignore_missing: true + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0)); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0)); + } + handleMap(ctx); + +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false + - append: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/fields/agent.yml b/packages/mimecast/data_stream/cloud_integrated_logs/fields/agent.yml new file mode 100644 index 00000000000..48f513b61aa --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/fields/agent.yml @@ -0,0 +1,39 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/fields/base-fields.yml b/packages/mimecast/data_stream/cloud_integrated_logs/fields/base-fields.yml new file mode 100644 index 00000000000..18e727db32a --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: mimecast +- name: event.dataset + type: constant_keyword + description: Event dataset + value: mimecast.cloud_integrated_logs +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/fields/field.yml b/packages/mimecast/data_stream/cloud_integrated_logs/fields/field.yml new file mode 100644 index 00000000000..ebea28ebae1 --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/fields/field.yml @@ -0,0 +1,123 @@ +- name: mimecast + type: group + fields: + # API https://developer.services.mimecast.com/siem-tutorial-ci + + - name: log_type + type: keyword + + ### Undocumented + - name: recipients + type: keyword + description: >- + The recipient of the email. + - name: source + type: keyword + - name: subtype + type: keyword + + ### Common fields + - name: processingId + type: keyword + description: >- + Unique identifier that allows you to correlate/group related events. + - name: aggregateId + type: keyword + description: >- + Unique identifier that allows you to correlate/group related events. + - name: timestamp + type: keyword + description: >- + The date and time of event. + - name: accountId + type: keyword + description: >- + The Mimecast account code for your account. + + ### Entities (Message Summary) Logs + - name: attachments + type: keyword + description: >- + The filenames of all attachments on the emai. + - name: tags + type: keyword + description: >- + The determination if the email was untrustworthy. + - name: policiesApplied + type: group + description: >- + The policy applied to derive outcome. + fields: + - name: action + type: keyword + - name: mode + type: keyword + - name: name + type: keyword + - name: historicalMail + type: boolean + description: >- + Identifies whether the scan was from historical email (prior to Mimecast). + - name: senderEnvelope + type: keyword + description: >- + The sender of the email. + - name: senderIp + type: keyword + description: >- + The source IP of the sending mail server. + - name: subject + type: keyword + description: >- + The subject of the email, limited to 150 characters. + - name: threatState + type: keyword + description: >- + The action taken. + - name: threatType + type: keyword + description: >- + The type of threat identified where applicable. no detections= allowed. + - name: direction + type: keyword + description: >- + The direction of the email based on the sending and receiving domains. + - name: senderHeader + type: keyword + description: >- + The sender address found in the from header of the email. + + ### Mail Flow logs + - name: messageId + type: keyword + description: >- + The internet message id of the email. + - name: authResults + type: group + description: >- + The auth result for an email, for just one of the auth types (like DKIM). + fields: + - name: aligned + type: boolean + - name: result + type: keyword + - name: type + type: keyword + + ### URL Protect Logs(blocked events only) + - name: action + type: keyword + description: >- + The action taken for this message. + - name: originalUrl + type: keyword + description: >- + The original URL Clicked. + - name: redirectUrl + type: keyword + description: >- + The redirect URL, following original URL click. + - name: sourceIp + type: keyword + description: >- + The source IP of the original message. \ No newline at end of file diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/manifest.yml b/packages/mimecast/data_stream/cloud_integrated_logs/manifest.yml new file mode 100644 index 00000000000..31e009aec9c --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/manifest.yml @@ -0,0 +1,77 @@ +title: "Cloud Integrated Logs" +type: logs +streams: + - input: cel + template_path: cel.yml.hbs + title: Cloud Integrated logs + enabled: false + description: Collect Cloud Integrated Logs + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the API. Supported units for this parameter are h/m/s. + multi: false + required: true + show_user: false + default: 5m + - name: initial_interval + type: text + title: Initial Interval + multi: false + required: true + show_user: false + description: Initial interval for the first API call. Defaults to 24 hours. Supported units for this parameter are h/m/s. + default: 24h + - name: types + type: text + title: Log Types + description: Log types to collect. + multi: true + required: true + show_user: true + default: + - "entities" + - "mailflow" + - "urlclick" + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - mimecast-cloud-integrated-logs + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: preserve_duplicate_custom_fields + required: false + show_user: false + title: Preserve duplicate custom fields + description: Preserve mimecast SIEM logs fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + - name: enable_request_tracer + type: bool + title: Enable request tracing + multi: false + required: false + show_user: false + description: > + The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details. + + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/sample_event.json b/packages/mimecast/data_stream/cloud_integrated_logs/sample_event.json new file mode 100644 index 00000000000..0d2431ef996 --- /dev/null +++ b/packages/mimecast/data_stream/cloud_integrated_logs/sample_event.json @@ -0,0 +1,66 @@ +{ + "@timestamp": "2024-11-21T18:03:26.960Z", + "agent": { + "ephemeral_id": "d0571978-95ee-4d4e-9fc1-2c0b57f47a64", + "id": "27a1a18b-9cc1-424a-8573-425423e11318", + "name": "elastic-agent-77334", + "type": "filebeat", + "version": "8.15.0" + }, + "data_stream": { + "dataset": "mimecast.cloud_integrated_logs", + "namespace": "67059", + "type": "logs" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "27a1a18b-9cc1-424a-8573-425423e11318", + "snapshot": false, + "version": "8.15.0" + }, + "email": { + "message_id": "<2ae37333-38e7-89ff-dc36-c8d48c6e3df3@demovation-ci.b41.one>" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "email" + ], + "created": "2024-11-21T18:03:26.960Z", + "dataset": "mimecast.cloud_integrated_logs", + "ingested": "2024-12-09T23:17:44Z", + "original": "{\"_offset\":1803841,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XvR1B4m7BzFB8L-qk59b4szrgayciaagczc977rzb_1732212206\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"none\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"pass\",\"type\":\"DMARC\"}],\"messageId\":\"\\u003c2ae37333-38e7-89ff-dc36-c8d48c6e3df3@demovation-ci.b41.one\\u003e\",\"processingId\":\"c40337e6860db0301575d8d09362bff214c0b010d6c4d41da9d770759ff54d10_1732212206\",\"subtype\":null,\"timestamp\":1732212206960,\"type\":\"mailflow\"}" + }, + "input": { + "type": "cel" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XvR1B4m7BzFB8L-qk59b4szrgayciaagczc977rzb_1732212206", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "none", + "type": "DKIM" + }, + { + "result": "pass", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "c40337e6860db0301575d8d09362bff214c0b010d6c4d41da9d770759ff54d10_1732212206" + }, + "tags": [ + "preserve_original_event", + "forwarded", + "mimecast-cloud-integrated-logs" + ] +} \ No newline at end of file diff --git a/packages/mimecast/data_stream/siem_logs/agent/stream/cel.yml.hbs b/packages/mimecast/data_stream/siem_logs/agent/stream/cel.yml.hbs index bdd92057e24..a47b159f0c9 100644 --- a/packages/mimecast/data_stream/siem_logs/agent/stream/cel.yml.hbs +++ b/packages/mimecast/data_stream/siem_logs/agent/stream/cel.yml.hbs @@ -27,6 +27,10 @@ redact: - client_secret - token.access_token program: | + // This program is shared between cloud_integrated_logs and siem_logs + // If it is changed here changes should be reflected in the other data + // streams. Do not differentiate the logic between these data streams + // lightly; use the state variable for this unless absolutely required. state.with( ( (has(state.?token.expires) && now() < timestamp(state.token.expires)) ? diff --git a/packages/mimecast/docs/README.md b/packages/mimecast/docs/README.md index 8d2949502ab..09f856d38ee 100644 --- a/packages/mimecast/docs/README.md +++ b/packages/mimecast/docs/README.md @@ -228,6 +228,131 @@ An example event for `audit_events` looks as following: | mimecast.timezone | Timezone reported in the event message. | keyword | +### Cloud Integrated Logs + +This is the `mimecast.cloud_integrated_logs` dataset. These logs contain Mimecast +threats and security events with the following details: entities, mail flows and URL +protected events. More information about [these logs]( +https://developer.services.mimecast.com/docs/threatssecurityeventsanddataforci/1/routes/siem/v1/batch/events/ci/get). + +An example event for `cloud_integrated` looks as following: + +```json +{ + "@timestamp": "2024-11-21T18:03:26.960Z", + "agent": { + "ephemeral_id": "d0571978-95ee-4d4e-9fc1-2c0b57f47a64", + "id": "27a1a18b-9cc1-424a-8573-425423e11318", + "name": "elastic-agent-77334", + "type": "filebeat", + "version": "8.15.0" + }, + "data_stream": { + "dataset": "mimecast.cloud_integrated_logs", + "namespace": "67059", + "type": "logs" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "27a1a18b-9cc1-424a-8573-425423e11318", + "snapshot": false, + "version": "8.15.0" + }, + "email": { + "message_id": "<2ae37333-38e7-89ff-dc36-c8d48c6e3df3@demovation-ci.b41.one>" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "email" + ], + "created": "2024-11-21T18:03:26.960Z", + "dataset": "mimecast.cloud_integrated_logs", + "ingested": "2024-12-09T23:17:44Z", + "original": "{\"_offset\":1803841,\"_partition\":53,\"accountId\":\"AUS2474\",\"aggregateId\":\"4XvR1B4m7BzFB8L-qk59b4szrgayciaagczc977rzb_1732212206\",\"authResults\":[{\"aligned\":true,\"result\":\"pass\",\"type\":\"SPF\"},{\"aligned\":false,\"result\":\"none\",\"type\":\"DKIM\"},{\"aligned\":null,\"result\":\"pass\",\"type\":\"DMARC\"}],\"messageId\":\"\\u003c2ae37333-38e7-89ff-dc36-c8d48c6e3df3@demovation-ci.b41.one\\u003e\",\"processingId\":\"c40337e6860db0301575d8d09362bff214c0b010d6c4d41da9d770759ff54d10_1732212206\",\"subtype\":null,\"timestamp\":1732212206960,\"type\":\"mailflow\"}" + }, + "input": { + "type": "cel" + }, + "mimecast": { + "accountId": "AUS2474", + "aggregateId": "4XvR1B4m7BzFB8L-qk59b4szrgayciaagczc977rzb_1732212206", + "authResults": [ + { + "aligned": true, + "result": "pass", + "type": "SPF" + }, + { + "aligned": false, + "result": "none", + "type": "DKIM" + }, + { + "result": "pass", + "type": "DMARC" + } + ], + "log_type": "mailflow", + "processingId": "c40337e6860db0301575d8d09362bff214c0b010d6c4d41da9d770759ff54d10_1732212206" + }, + "tags": [ + "preserve_original_event", + "forwarded", + "mimecast-cloud-integrated-logs" + ] +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| host.containerized | If the host is a container. | boolean | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| input.type | Input type | keyword | +| log.offset | Log offset | long | +| mimecast.accountId | The Mimecast account code for your account. | keyword | +| mimecast.action | The action taken for this message. | keyword | +| mimecast.aggregateId | Unique identifier that allows you to correlate/group related events. | keyword | +| mimecast.attachments | The filenames of all attachments on the emai. | keyword | +| mimecast.authResults.aligned | | boolean | +| mimecast.authResults.result | | keyword | +| mimecast.authResults.type | | keyword | +| mimecast.direction | The direction of the email based on the sending and receiving domains. | keyword | +| mimecast.historicalMail | Identifies whether the scan was from historical email (prior to Mimecast). | boolean | +| mimecast.log_type | | keyword | +| mimecast.messageId | The internet message id of the email. | keyword | +| mimecast.originalUrl | The original URL Clicked. | keyword | +| mimecast.policiesApplied.action | | keyword | +| mimecast.policiesApplied.mode | | keyword | +| mimecast.policiesApplied.name | | keyword | +| mimecast.processingId | Unique identifier that allows you to correlate/group related events. | keyword | +| mimecast.recipients | The recipient of the email. | keyword | +| mimecast.redirectUrl | The redirect URL, following original URL click. | keyword | +| mimecast.senderEnvelope | The sender of the email. | keyword | +| mimecast.senderHeader | The sender address found in the from header of the email. | keyword | +| mimecast.senderIp | The source IP of the sending mail server. | keyword | +| mimecast.source | | keyword | +| mimecast.sourceIp | The source IP of the original message. | keyword | +| mimecast.subject | The subject of the email, limited to 150 characters. | keyword | +| mimecast.subtype | | keyword | +| mimecast.tags | The determination if the email was untrustworthy. | keyword | +| mimecast.threatState | The action taken. | keyword | +| mimecast.threatType | The type of threat identified where applicable. no detections= allowed. | keyword | +| mimecast.timestamp | The date and time of event. | keyword | + + ### DLP Logs This is the `mimecast.dlp_logs` dataset. These logs contain information about diff --git a/packages/mimecast/manifest.yml b/packages/mimecast/manifest.yml index 0a382444e5f..117fd4a4882 100644 --- a/packages/mimecast/manifest.yml +++ b/packages/mimecast/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: mimecast title: "Mimecast" -version: "2.2.0" +version: "2.3.0" description: Collect logs from Mimecast with Elastic Agent. type: integration categories: ["security", "email_security"]