From baf74aca0daf727f2e9f4d6d604782ab1d1dce9b Mon Sep 17 00:00:00 2001 From: MakoWish <26614684+MakoWish@users.noreply.github.com> Date: Tue, 11 Jul 2023 19:45:45 +0930 Subject: [PATCH] Add missing ECS field mappings --- .../cisco_secure_email_gateway/changelog.yml | 5 + .../test-common-consolidated-event.log | 4 +- ...ommon-consolidated-event.log-expected.json | 291 ++++++++++++++++++ .../pipeline_consolidated_event.yml | 28 +- .../cisco_secure_email_gateway/manifest.yml | 2 +- .../nginx/data_stream/access/fields/ecs.yml | 4 + .../nginx/data_stream/error/fields/ecs.yml | 6 + .../data_stream/stubstatus/fields/ecs.yml | 6 + 8 files changed, 333 insertions(+), 13 deletions(-) diff --git a/packages/cisco_secure_email_gateway/changelog.yml b/packages/cisco_secure_email_gateway/changelog.yml index b3ced95d9b4..7de312413c6 100644 --- a/packages/cisco_secure_email_gateway/changelog.yml +++ b/packages/cisco_secure_email_gateway/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.10.1" + changes: + - description: Fix grok timeout on expensive consolidated events logs. + type: bugfix + link: https://github.com/elastic/integrations/pull/6879 - version: "1.10.0" changes: - description: Convert dashboard to lens. diff --git a/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log b/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log index f17c35dd1d0..d5bb8254c73 100644 --- a/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log +++ b/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log @@ -9,4 +9,6 @@ <166>Mar 17 18:24:37 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-023|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=423A4DF759243122B64F-7941F28E57A4 ESAMID=4086421 ESAICID=13956459 ESADCID=2522340 ESAAMPVerdict=UNKNOWN ESAASVerdict=NEGATIVE ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NO_MATCH endTime=Thu Nov 24 13:39:24 2022 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=1.128.3.4 ESAAttachmentDetails={'image002.png': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '30bf618599d8784ebcf38769f8b524b40dc20d2ba262a1e4052d24711abcd064'}, 'BodyScanner': {}}, 'image001.png': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '7de9d8514c142887d11821fd30faddc693d192efdd19dfb6459872a1be63dcfa'}, 'BodyScanner': {}}} ESAFriendlyFrom=example.com ESAGMVerdict=NEGATIVE startTime=Thu Nov 24 13:39:16 2022 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=United States ESAMFVerdict=MATCH act=DELIVERED cs4Label=ExternalMsgID cs4='' ESAMsgSize=716707 ESAOFVerdict=NEGATIVE duser=example.com ESAHeloDomain=example.cisco.com ESAHeloIP=1.128.3.4 cfp1Label=SBRSScore cfp1=3.5 ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Favorable sourceHostName=example.cisco.com ESASenderGroup=ACCEPTLIST sourceAddress=1.128.3.4 msg='RE: SR 312312 : consolidate event log' ESATLSInCipher=KWLDS-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESATLSOutCipher=HDKWA-RSA-AES256-JMB-SHA384 ESATLSOutConnStatus=Success ESATLSOutProtocol=TLSv1.2 ESADaneHost=testdomain.com ESADaneStatus=success ESADHASource=1.128.3.4 ESADMARCVerdict=TempFailure cs5Label=ESAMsgLanguage cs5=English ESAMARAction={'action':'<>';'succesful_rcpts'='<>';'failed_recipients'='<>';'filename'='<>'} ESAMsgTooBigFromSender=true ESARateLimitedIP=1.128.3.4 <166>Apr 03 12:20:40 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.0.2-020|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=422084EE64B1B0454D49-AAFBF6B55869 ESAMID=164229 ESAICID=62908 ESADCID=47845 ESAAMPVerdict=SKIPPED ESAASVerdict=NEGATIVE ESAAVVerdict=NEGATIVE ESACFVerdict=NO_MATCH endTime=Mon Nov 14 15:40:48 2022 ESADLPVerdict=NOT_EVALUATED dvc=1.128.3.4 ESAFriendlyFrom=example.com ESAGMVerdict=NEGATIVE startTime=Mon Nov 14 15:40:47 2022 deviceInboundInterface=IncList deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=United States ESAMFVerdict=NOT_EVALUATED act=DELIVERED cs4Label=ExternalMsgID cs4='' ESAMsgSize=1411 ESAOFVerdict=NEGATIVE duser=example.com ESAHeloDomain=example.cisco.com ESAHeloIP=1.128.3.4 ESAReplyTo=example.com cfp1Label=SBRSScore cfp1=5.2 ESASDRDomainAge=1 month cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral sourceHostName=example.cisco.com ESASenderGroup=UNKNOWNLIST sourceAddress=1.128.3.4 msg="Demande d'achat Econocom Products and Solutions, ref: SSAY-MEDECIN3" <14>Jun 12 14:01:33 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=4235746AE34E6DD44EB8-CE101C994AA5 ESAMID=5181473 ESAICID=17267358 ESADCID=3036000 endTime=Mon Jun 12 14:01:31 2023 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=1.128.3.4 ESAFriendlyFrom=jean sistin ESAGMVerdict=NEGATIVE startTime=Mon Jun 12 14:01:27 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=jsistin@gmail.com cs1Label=MailPolicy cs1=Test quarantaine utilisateur cs2Label=SenderCountry cs2=United States ESAMFVerdict=MATCH act=DELIVERED cs4Label=ExternalMsgID cs4='' ESAMsgSize=3078 ESAOFVerdict=NEGATIVE duser=fpenigaud@exaprobe.com ESAHeloDomain=mail-oi1-f182.google.com ESAHeloIP=1.128.3.4 cfp1Label=SBRSScore cfp1=3.4 ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'jsistin@gmail.com'}} sourceHostName=mail-oi1-f182.google.com ESASenderGroup=ACCEPTLIST sourceAddress=1.128.3.4 msg='test url' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESATLSOutCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSOutConnStatus=Success ESATLSOutProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=NEGATIVE ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NO_MATCH - +<14>Jul 04 06:21:54 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=E7DEF468022C4EB09683-9A331A42E1F7 ESAMID=54376810 ESAICID=43587623 endTime=Tue Jul 4 06:21:54 2023 ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.144 ESAAttachmentDetails={'meeting.ics': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '17ae79446b4ec3baf161704831970aac49457d32935c6383c2a45aed136a99df'}, 'BodyScanner': {}}} ESAFriendlyFrom=River ESAGMVerdict=NEGATIVE startTime=Tue Jul 4 16:12:44 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=river@this.example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=New Zealand ESAMFVerdict=MATCH act=DQ ESAFinalActionDetails=Message held temporarily in Delay Quarantine cs4Label=ExternalMsgID cs4='<2403354681.734500.1688449973515.mail.lion@example.com>' ESAMsgSize=18675 ESAOFVerdict=NEGATIVE duser=smith@example.com ESAHeloDomain=vm-lion.dmz ESAHeloIP=89.160.20.128 cfp1Label=SBRSScore cfp1=None ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'river@this.example.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@vm-lion.dmz'}} sourceHostName=company.example.com sourceAddress=89.160.20.128 msg='Accept: Cisco - SOLUTIONS' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=FA_PENDING ESAASVerdict=NEGATIVE ESAAVVerdict=NOT_EVALUATED ESACFVerdict=MATCH +<14>Jul 04 06:21:54 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=30543A3AB9E54FA8AAC1-FB812C95028D ESAMID=238746 ESAICID=435897324 ESADCID=34809573 endTime=Tue Jul 4 06:21:54 2023 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.142 ESAFriendlyFrom=Will ESAGMVerdict=NEGATIVE startTime=Tue Jul 4 15:14:29 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=irobot@example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=United States ESAMFVerdict=MATCH act=QUARANTINED ESAFinalActionDetails=To SPAM cs4Label=ExternalMsgID cs4='' ESAMsgSize=12312 ESAOFVerdict=NEGATIVE duser=alfombra@example.com ESAHeloDomain=mail-q6by9-a42.google.com ESAHeloIP=81.2.69.192 cfp1Label=SBRSScore cfp1=2.7 ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'irobot@example.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@mail-q6by9-a42.google.com'}} sourceHostName=mail-q6by9-a42.google.com ESASenderGroup=ACCEPTLIST sourceAddress=81.2.69.192 msg='IE : Crayons' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=MATCH +<14>Jul 04 06:21:54 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=4FEF3A4372664BCCB404-20EE1767D434 ESAMID=786324 ESAICID=35635425 ESADCID=970897 endTime=Tue Jul 4 06:21:54 2023 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.144 ESAFriendlyFrom=Beaches ESAGMVerdict=NEGATIVE startTime=Tue Jul 4 14:42:34 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=westinghouse-thoreau\\=example.com@example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=NZ ESAMFVerdict=MATCH act=QUARANTINED ESAFinalActionDetails=To SPAM cs4Label=ExternalMsgID cs4='<490b2a15fa4742331779cdaa4e@example.com>' ESAMsgSize=20668 ESAOFVerdict=NEGATIVE duser=thoreau@example.com ESAHeloDomain=example.com ESAHeloIP=89.160.20.112 ESAReplyTo=lane@example.com cfp1Label=SBRSScore cfp1=None ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'westinghouse-thoreau=example.com@example.com'}, 'helo': {'result': 'Pass', 'sender': 'postmaster@example.com'}} sourceHostName=example.com sourceAddress=89.160.20.112 msg=\"Totally not suspicious email subject\" ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NO_MATCH diff --git a/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log-expected.json b/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log-expected.json index fecac4e4163..59d03a4a4d3 100644 --- a/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log-expected.json +++ b/packages/cisco_secure_email_gateway/data_stream/log/_dev/test/pipeline/test-common-consolidated-event.log-expected.json @@ -1160,12 +1160,303 @@ ] }, { + "@timestamp": "2023-07-04T06:21:54.000Z", + "cisco_secure_email_gateway": { + "log": { + "act": "DQ", + "appliance": { + "product": "C100V Email Security Virtual Appliance", + "vendor": "Cisco", + "version": "14.3.0-032" + }, + "category": { + "name": "consolidated_event" + }, + "cef_format_version": "0", + "cfp1_label": "SBRSScore", + "cs1": "DEFAULT", + "cs1_label": "MailPolicy", + "cs2": "New Zealand", + "cs2_label": "SenderCountry", + "cs3": "N/A", + "cs3_label": "SDRThreatCategory", + "cs4": "2403354681.734500.1688449973515.mail.lion@example.com", + "cs4_label": "ExternalMsgID", + "cs6": "Neutral", + "cs6_label": "SDRRepScore", + "data": { + "ip": "81.2.69.144" + }, + "device_direction": "incoming", + "esa": { + "attachment_details": "{'meeting.ics': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '17ae79446b4ec3baf161704831970aac49457d32935c6383c2a45aed136a99df'}, 'BodyScanner': {}}}", + "dlp_verdict": "NOT_EVALUATED", + "final_action_details": "Message held temporarily in Delay Quarantine", + "friendly_from": "River \u003criver@this.example.com\u003e", + "graymail_verdict": "NEGATIVE", + "helo": { + "domain": "vm-lion.dmz", + "ip": "89.160.20.128" + }, + "injection_connection_id": "43587623", + "mail_flow_policy": "ACCEPT", + "mf_verdict": "MATCH", + "msg_size": 18675, + "outbreak_filter_verdict": "NEGATIVE", + "sdr_consolidated_domain_age": "30 days (or greater)", + "spf_verdict": "{'mailfrom': {'result': 'Pass', 'sender': 'river@this.example.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@vm-lion.dmz'}}" + }, + "event": { + "name": "Consolidated Log Event" + }, + "event_class_id": "ESA_CONSOLIDATED_LOG_EVENT", + "listener": { + "name": "IncomingMail" + }, + "message": "'Accept: Cisco - SOLUTIONS' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=FA_PENDING ESAASVerdict=NEGATIVE ESAAVVerdict=NOT_EVALUATED ESACFVerdict=MATCH" + } + }, "ecs": { "version": "8.8.0" }, + "email": { + "from": { + "address": "river@this.example.com" + }, + "message_id": "54376810", + "to": { + "address": "smith@example.com" + } + }, "event": { + "end": "Tue Jul 4 06:21:54 2023", + "kind": "event", + "original": "\u003c14\u003eJul 04 06:21:54 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=E7DEF468022C4EB09683-9A331A42E1F7 ESAMID=54376810 ESAICID=43587623 endTime=Tue Jul 4 06:21:54 2023 ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.144 ESAAttachmentDetails={'meeting.ics': {'AMP': {'Verdict': 'FILE UNKNOWN', 'fileHash': '17ae79446b4ec3baf161704831970aac49457d32935c6383c2a45aed136a99df'}, 'BodyScanner': {}}} ESAFriendlyFrom=River \u003criver@this.example.com\u003e ESAGMVerdict=NEGATIVE startTime=Tue Jul 4 16:12:44 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=river@this.example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=New Zealand ESAMFVerdict=MATCH act=DQ ESAFinalActionDetails=Message held temporarily in Delay Quarantine cs4Label=ExternalMsgID cs4='\u003c2403354681.734500.1688449973515.mail.lion@example.com\u003e' ESAMsgSize=18675 ESAOFVerdict=NEGATIVE duser=smith@example.com ESAHeloDomain=vm-lion.dmz ESAHeloIP=89.160.20.128 cfp1Label=SBRSScore cfp1=None ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'river@this.example.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@vm-lion.dmz'}} sourceHostName=company.example.com sourceAddress=89.160.20.128 msg='Accept: Cisco - SOLUTIONS' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=FA_PENDING ESAASVerdict=NEGATIVE ESAAVVerdict=NOT_EVALUATED ESACFVerdict=MATCH", + "severity": "5", + "start": "Tue Jul 4 16:12:44 2023", "timezone": "UTC" }, + "host": { + "id": "E7DEF468022C4EB09683-9A331A42E1F7" + }, + "log": { + "syslog": { + "priority": 14 + } + }, + "related": { + "ip": [ + "89.160.20.128", + "81.2.69.144" + ] + }, + "source": { + "domain": "company.example.com", + "ip": "89.160.20.128" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2023-07-04T06:21:54.000Z", + "cisco_secure_email_gateway": { + "log": { + "act": "QUARANTINED", + "appliance": { + "product": "C100V Email Security Virtual Appliance", + "vendor": "Cisco", + "version": "14.3.0-032" + }, + "category": { + "name": "consolidated_event" + }, + "cef_format_version": "0", + "cfp1": 2.7, + "cfp1_label": "SBRSScore", + "cs1": "DEFAULT", + "cs1_label": "MailPolicy", + "cs2": "United States", + "cs2_label": "SenderCountry", + "cs3": "N/A", + "cs3_label": "SDRThreatCategory", + "cs4": "MDlhMDg0MjY0NmE2OWFkNTZhMzA2NDA0MDVkZWNlZWVlYzI3MjMyYmI5YWJlNDMxM2UxOGVjZTBiNGZmOGZmYSAgLQo@hotmail.com", + "cs4_label": "ExternalMsgID", + "cs6": "Neutral", + "cs6_label": "SDRRepScore", + "data": { + "ip": "81.2.69.142" + }, + "device_direction": "incoming", + "esa": { + "delivery_connection_id": "34809573", + "dkim_verdict": "pass", + "dlp_verdict": "NOT_EVALUATED", + "final_action_details": "To SPAM", + "friendly_from": "Will \u003cirobot@example.com\u003e", + "graymail_verdict": "NEGATIVE", + "helo": { + "domain": "mail-q6by9-a42.google.com", + "ip": "81.2.69.192" + }, + "injection_connection_id": "435897324", + "mail_flow_policy": "ACCEPT", + "mf_verdict": "MATCH", + "msg_size": 12312, + "outbreak_filter_verdict": "NEGATIVE", + "sdr_consolidated_domain_age": "30 days (or greater)", + "sender_group": "ACCEPTLIST", + "spf_verdict": "{'mailfrom': {'result': 'Pass', 'sender': 'irobot@example.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@mail-q6by9-a42.google.com'}}" + }, + "event": { + "name": "Consolidated Log Event" + }, + "event_class_id": "ESA_CONSOLIDATED_LOG_EVENT", + "listener": { + "name": "IncomingMail" + }, + "message": "'IE : Crayons' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=MATCH" + } + }, + "ecs": { + "version": "8.8.0" + }, + "email": { + "from": { + "address": "irobot@example.com" + }, + "message_id": "238746", + "to": { + "address": "alfombra@example.com" + } + }, + "event": { + "end": "Tue Jul 4 06:21:54 2023", + "kind": "event", + "original": "\u003c14\u003eJul 04 06:21:54 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=30543A3AB9E54FA8AAC1-FB812C95028D ESAMID=238746 ESAICID=435897324 ESADCID=34809573 endTime=Tue Jul 4 06:21:54 2023 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.142 ESAFriendlyFrom=Will \u003cirobot@example.com\u003e ESAGMVerdict=NEGATIVE startTime=Tue Jul 4 15:14:29 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=irobot@example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=United States ESAMFVerdict=MATCH act=QUARANTINED ESAFinalActionDetails=To SPAM cs4Label=ExternalMsgID cs4='\u003cMDlhMDg0MjY0NmE2OWFkNTZhMzA2NDA0MDVkZWNlZWVlYzI3MjMyYmI5YWJlNDMxM2UxOGVjZTBiNGZmOGZmYSAgLQo@hotmail.com\u003e' ESAMsgSize=12312 ESAOFVerdict=NEGATIVE duser=alfombra@example.com ESAHeloDomain=mail-q6by9-a42.google.com ESAHeloIP=81.2.69.192 cfp1Label=SBRSScore cfp1=2.7 ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'irobot@example.com'}, 'helo': {'result': 'None', 'sender': 'postmaster@mail-q6by9-a42.google.com'}} sourceHostName=mail-q6by9-a42.google.com ESASenderGroup=ACCEPTLIST sourceAddress=81.2.69.192 msg='IE : Crayons' ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=MATCH", + "severity": "5", + "start": "Tue Jul 4 15:14:29 2023", + "timezone": "UTC" + }, + "host": { + "id": "30543A3AB9E54FA8AAC1-FB812C95028D" + }, + "log": { + "syslog": { + "priority": 14 + } + }, + "related": { + "ip": [ + "81.2.69.192", + "81.2.69.142" + ] + }, + "source": { + "domain": "mail-q6by9-a42.google.com", + "ip": "81.2.69.192" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2023-07-04T06:21:54.000Z", + "cisco_secure_email_gateway": { + "log": { + "act": "QUARANTINED", + "appliance": { + "product": "C100V Email Security Virtual Appliance", + "vendor": "Cisco", + "version": "14.3.0-032" + }, + "category": { + "name": "consolidated_event" + }, + "cef_format_version": "0", + "cfp1_label": "SBRSScore", + "cs1": "DEFAULT", + "cs1_label": "MailPolicy", + "cs2": "NZ", + "cs2_label": "SenderCountry", + "cs3": "N/A", + "cs3_label": "SDRThreatCategory", + "cs4": "490b2a15fa4742331779cdaa4e@example.com", + "cs4_label": "ExternalMsgID", + "cs6": "Neutral", + "cs6_label": "SDRRepScore", + "data": { + "ip": "81.2.69.144" + }, + "device_direction": "incoming", + "esa": { + "delivery_connection_id": "970897", + "dkim_verdict": "pass", + "dlp_verdict": "NOT_EVALUATED", + "final_action_details": "To SPAM", + "friendly_from": "Beaches \u003cplayas@example.com\u003e", + "graymail_verdict": "NEGATIVE", + "helo": { + "domain": "example.com", + "ip": "89.160.20.112" + }, + "injection_connection_id": "35635425", + "mail_flow_policy": "ACCEPT", + "mf_verdict": "MATCH", + "msg_size": 20668, + "outbreak_filter_verdict": "NEGATIVE", + "reply_to": "lane@example.com", + "sdr_consolidated_domain_age": "30 days (or greater)", + "spf_verdict": "{'mailfrom': {'result': 'Pass', 'sender': 'westinghouse-thoreau=example.com@example.com'}, 'helo': {'result': 'Pass', 'sender': 'postmaster@example.com'}}" + }, + "event": { + "name": "Consolidated Log Event" + }, + "event_class_id": "ESA_CONSOLIDATED_LOG_EVENT", + "listener": { + "name": "IncomingMail" + }, + "message": "\\\"Totally not suspicious email subject\\\" ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NO_MATCH" + } + }, + "ecs": { + "version": "8.8.0" + }, + "email": { + "from": { + "address": "westinghouse-thoreau\\\\=example.com@example.com" + }, + "message_id": "786324", + "to": { + "address": "thoreau@example.com" + } + }, + "event": { + "end": "Tue Jul 4 06:21:54 2023", + "kind": "event", + "original": "\u003c14\u003eJul 04 06:21:54 consolidated_event: CEF:0|Cisco|C100V Email Security Virtual Appliance|14.3.0-032|ESA_CONSOLIDATED_LOG_EVENT|Consolidated Log Event|5|deviceExternalId=4FEF3A4372664BCCB404-20EE1767D434 ESAMID=786324 ESAICID=35635425 ESADCID=970897 endTime=Tue Jul 4 06:21:54 2023 ESADKIMVerdict=pass ESADLPVerdict=NOT_EVALUATED dvc=81.2.69.144 ESAFriendlyFrom=Beaches \u003cplayas@example.com\u003e ESAGMVerdict=NEGATIVE startTime=Tue Jul 4 14:42:34 2023 deviceInboundInterface=IncomingMail deviceDirection=0 ESAMailFlowPolicy=ACCEPT suser=westinghouse-thoreau\\\\=example.com@example.com cs1Label=MailPolicy cs1=DEFAULT cs2Label=SenderCountry cs2=NZ ESAMFVerdict=MATCH act=QUARANTINED ESAFinalActionDetails=To SPAM cs4Label=ExternalMsgID cs4='\u003c490b2a15fa4742331779cdaa4e@example.com\u003e' ESAMsgSize=20668 ESAOFVerdict=NEGATIVE duser=thoreau@example.com ESAHeloDomain=example.com ESAHeloIP=89.160.20.112 ESAReplyTo=lane@example.com cfp1Label=SBRSScore cfp1=None ESASDRDomainAge=30 days (or greater) cs3Label=SDRThreatCategory cs3=N/A cs6Label=SDRRepScore cs6=Neutral ESASPFVerdict={'mailfrom': {'result': 'Pass', 'sender': 'westinghouse-thoreau=example.com@example.com'}, 'helo': {'result': 'Pass', 'sender': 'postmaster@example.com'}} sourceHostName=example.com sourceAddress=89.160.20.112 msg=\\\"Totally not suspicious email subject\\\" ESATLSInCipher=ECDHE-RSA-AES256-GCM-SHA384 ESATLSInConnStatus=Success ESATLSInProtocol=TLSv1.2 ESAAMPVerdict=SKIPPED ESAASVerdict=SUSPECT ESAAVVerdict=NOT_EVALUATED ESACFVerdict=NO_MATCH", + "severity": "5", + "start": "Tue Jul 4 14:42:34 2023", + "timezone": "UTC" + }, + "host": { + "id": "4FEF3A4372664BCCB404-20EE1767D434" + }, + "log": { + "syslog": { + "priority": 14 + } + }, + "related": { + "ip": [ + "89.160.20.112", + "81.2.69.144" + ] + }, + "source": { + "domain": "example.com", + "ip": "89.160.20.112" + }, "tags": [ "preserve_original_event" ] diff --git a/packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_consolidated_event.yml b/packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_consolidated_event.yml index ebcf2281af9..fc06195d95c 100644 --- a/packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_consolidated_event.yml +++ b/packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_consolidated_event.yml @@ -52,23 +52,29 @@ processors: (?:sourceHostName=%{DATA:source.domain} )?\ (?:ESASenderGroup=%{DATA:cisco_secure_email_gateway.log.esa.sender_group} )?\ (?:sourceAddress=%{IP:source.ip} )?\ - msg=('|\")%{DATA:email.subject}('|\")($|\\s)\ - (?:ESAURLDetails=%{DATA:cisco_secure_email_gateway.log.esa.url_details}($|\\s))?\ + msg=%{GREEDYDATA:_tmp.msg}$" + - ^%{GREEDYDATA:cisco_secure_email_gateway.log.message}$ + - grok: + field: _tmp.msg + if: ctx._tmp?.msg != null + patterns: + - "^['\"]%{DATA:email.subject}['\"](?:$|\\s)\ + (?:ESAURLDetails=%{DATA:cisco_secure_email_gateway.log.esa.url_details}(?:$|\\s))?\ (?:ESATLSInCipher=%{DATA:cisco_secure_email_gateway.log.esa.tls.in.cipher} )?\ (?:ESATLSInConnStatus=%{WORD:cisco_secure_email_gateway.log.esa.tls.in.connection_status} )?\ (?:ESATLSInProtocol=%{DATA:cisco_secure_email_gateway.log.esa.tls.in.protocol} )?\ (?:ESATLSOutCipher=%{DATA:cisco_secure_email_gateway.log.esa.tls.out.cipher} )?\ (?:ESATLSOutConnStatus=%{WORD:cisco_secure_email_gateway.log.esa.tls.out.connection_status} )?\ - (?:ESATLSOutProtocol=%{DATA:cisco_secure_email_gateway.log.esa.tls.out.protocol}($|\\s))?\ + (?:ESATLSOutProtocol=%{DATA:cisco_secure_email_gateway.log.esa.tls.out.protocol}(?:$|\\s))?\ (?:ESADaneHost=%{DATA:cisco_secure_email_gateway.log.esa.dane.host} )?\ - (?:ESADaneStatus=%{WORD:cisco_secure_email_gateway.log.esa.dane.status}($|\\s))?\ - (?:ESADHASource=%{IP:cisco_secure_email_gateway.log.esa.dha_source}($|\\s))?\ - (?:ESADMARCVerdict=%{WORD:cisco_secure_email_gateway.log.esa.dmarc_verdict}($|\\s))?\ - (?:cs5Label=%{DATA:cisco_secure_email_gateway.log.cs5_label}($|\\s))?\ - (?:cs5=%{DATA:cisco_secure_email_gateway.log.cs5}($|\\s))?\ - (?:ESAMARAction=%{DATA:cisco_secure_email_gateway.log.esa.mail_auto_remediation_action}($|\\s))?\ - (?:ESAMsgTooBigFromSender=%{WORD:cisco_secure_email_gateway.log.esa.msg_too_big_from_sender}($|\\s))?\ - (?:ESARateLimitedIP=%{GREEDYDATA:cisco_secure_email_gateway.log.esa.rate_limited_ip}($|\\s))?$" + (?:ESADaneStatus=%{WORD:cisco_secure_email_gateway.log.esa.dane.status}(?:$|\\s))?\ + (?:ESADHASource=%{IP:cisco_secure_email_gateway.log.esa.dha_source}(?:$|\\s))?\ + (?:ESADMARCVerdict=%{WORD:cisco_secure_email_gateway.log.esa.dmarc_verdict}(?:$|\\s))?\ + (?:cs5Label=%{DATA:cisco_secure_email_gateway.log.cs5_label}(?:$|\\s))?\ + (?:cs5=%{DATA:cisco_secure_email_gateway.log.cs5}(?:$|\\s))?\ + (?:ESAMARAction=%{DATA:cisco_secure_email_gateway.log.esa.mail_auto_remediation_action}(?:$|\\s))?\ + (?:ESAMsgTooBigFromSender=%{WORD:cisco_secure_email_gateway.log.esa.msg_too_big_from_sender}(?:$|\\s))?\ + (?:ESARateLimitedIP=%{GREEDYDATA:cisco_secure_email_gateway.log.esa.rate_limited_ip}(?:$|\\s))?$" - ^%{GREEDYDATA:cisco_secure_email_gateway.log.message}$ - kv: field: _tmp.details diff --git a/packages/cisco_secure_email_gateway/manifest.yml b/packages/cisco_secure_email_gateway/manifest.yml index b109f29960e..89348c208e8 100644 --- a/packages/cisco_secure_email_gateway/manifest.yml +++ b/packages/cisco_secure_email_gateway/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: cisco_secure_email_gateway title: Cisco Secure Email Gateway -version: "1.10.0" +version: "1.10.1" license: basic description: Collect logs from Cisco Secure Email Gateway with Elastic Agent. type: integration diff --git a/packages/nginx/data_stream/access/fields/ecs.yml b/packages/nginx/data_stream/access/fields/ecs.yml index 1888586bd9b..e6ce7e4f2c0 100644 --- a/packages/nginx/data_stream/access/fields/ecs.yml +++ b/packages/nginx/data_stream/access/fields/ecs.yml @@ -34,12 +34,16 @@ name: source.geo.continent_name - external: ecs name: source.geo.country_iso_code +- external: ecs + name: source.geo.dma_code - external: ecs name: source.geo.country_name - description: Longitude and latitude. level: core name: source.geo.location type: geo_point +- external: ecs + name: source.geo.country_code - external: ecs name: source.geo.region_iso_code - external: ecs diff --git a/packages/nginx/data_stream/error/fields/ecs.yml b/packages/nginx/data_stream/error/fields/ecs.yml index e26b3ec9bd2..30f5b0c85ed 100644 --- a/packages/nginx/data_stream/error/fields/ecs.yml +++ b/packages/nginx/data_stream/error/fields/ecs.yml @@ -12,5 +12,11 @@ name: process.pid - external: ecs name: process.thread.id +- external: ecs + name: related.ip +- external: ecs + name: source.geo.dma_code +- external: ecs + name: source.geo.postal_code - external: ecs name: tags diff --git a/packages/nginx/data_stream/stubstatus/fields/ecs.yml b/packages/nginx/data_stream/stubstatus/fields/ecs.yml index d7ea22de8d6..e5c42c36a29 100644 --- a/packages/nginx/data_stream/stubstatus/fields/ecs.yml +++ b/packages/nginx/data_stream/stubstatus/fields/ecs.yml @@ -5,5 +5,11 @@ - external: ecs name: service.address dimension: true +- external: ecs + name: related.ip - external: ecs name: service.type +- external: ecs + name: source.geo.dma_code +- external: ecs + name: source.geo.postal_code