From 165d36204b46a7947d5ae8511558c82e2a83e432 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Mon, 14 Aug 2023 07:30:28 +0930 Subject: [PATCH] x-pack/filebeat/module/juniper: fix handling of jrx structured data Previously, the leading word was ignored. I all our test cases this was in the form /junos@(\d+\.){5}\d+/. When this value is not present, we lose the first structured data value, so be more careful in assessing whether the first element should be discarded. --- CHANGELOG.next.asciidoc | 35 +----------- .../module/juniper/srx/ingest/pipeline.yml | 2 +- .../filebeat/module/juniper/srx/test/flow.log | 1 + .../juniper/srx/test/flow.log-expected.json | 57 +++++++++++++++++++ 4 files changed, 60 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ba94ca31247..5e63f64e980 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -113,6 +113,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Update mito CEL extension library to v1.5.0. {pull}36146[36146] - Filter out duplicate paths resolved from matching globs. {issue}36253[36253] {pull}36256[36256] - Fix handling of TCP/UDP address resolution during metric initialization. {issue}35064[35064] {pull}36287[36287] +- Fix handling of Juniper SRX structured data when there is no leading junos element. {issue}36270[36270] {pull}36308[36308] *Heartbeat* @@ -142,33 +143,6 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Enable heartbeat-wide publish timeout setting with run_once. {pull}35721[35721] - Added default timezone UTC to heartbeat docker images to fix synthetics journeys navigation errors. {pull}36193[36193] -*Heartbeat* - - -*Heartbeat* - - -*Heartbeat* - - -*Heartbeat* - - -*Auditbeat* - - -*Filebeat* - - -*Auditbeat* - - -*Filebeat* - - -*Heartbeat* - - *Metricbeat* - in module/windows/perfmon, changed collection method of the second counter value required to create a displayable value {pull}32305[32305] @@ -200,13 +174,6 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Fix powershell details regexp to prevent excessive backtracking when processing command invocations. {pull}36178[36178] -*Functionbeat* - - -*Functionbeat* - - - *Elastic Logging Plugin* diff --git a/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml b/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml index a7d4b22ee7e..062066ab7ab 100644 --- a/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml +++ b/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml @@ -5,7 +5,7 @@ processors: - grok: field: message patterns: - - '^<%{POSINT:syslog_pri}>(\d{1,3}\s)?(?:%{TIMESTAMP_ISO8601:_temp_.raw_date})\s%{SYSLOGHOST:syslog_hostname}\s%{PROG:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:log_type}\s\[.+?\s%{GREEDYDATA:log.original}\]$' + - '^<%{POSINT:syslog_pri}>(\d{1,3}\s)?(?:%{TIMESTAMP_ISO8601:_temp_.raw_date})\s%{SYSLOGHOST:syslog_hostname}\s%{PROG:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:log_type}\s\[(?:[^=]+\s)?%{GREEDYDATA:log.original}\]$' # split Juniper-SRX fields - kv: diff --git a/x-pack/filebeat/module/juniper/srx/test/flow.log b/x-pack/filebeat/module/juniper/srx/test/flow.log index 89c8b63e546..16ce15c66f9 100644 --- a/x-pack/filebeat/module/juniper/srx/test/flow.log +++ b/x-pack/filebeat/module/juniper/srx/test/flow.log @@ -23,3 +23,4 @@ <14>1 2020-07-13T16:43:05.041Z SRX100HM RT_FLOW - RT_FLOW_SESSION_CLOSE [junos@2636.81.2.69.144.129 reason="idle Timeout" source-address="10.1.1.100" source-port="64720" destination-address="91.228.167.172" destination-port="8883" connection-tag="0" service-name="None" nat-source-address="172.19.34.100" nat-source-port="24519" nat-destination-address="91.228.167.172" nat-destination-port="8883" nat-connection-tag="0" src-nat-rule-type="source rule" src-nat-rule-name="our-nat-rule" dst-nat-rule-type="N/A" dst-nat-rule-name="N/A" protocol-id="6" policy-name="default-permit" source-zone-name="trust" destination-zone-name="untrust" session-id-32="3851" packets-from-client="161" bytes-from-client="9530" packets-from-server="96" bytes-from-server="9670" elapsed-time="23755" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="ge-0/0/1.0" encrypted="UNKNOWN" application-category="N/A" application-sub-category="N/A" application-risk="1" application-characteristics="N/A" secure-web-proxy-session-type="NA" peer-session-id="0" peer-source-address="0.0.0.0" peer-source-port="0" peer-destination-address="0.0.0.0" peer-destination-port="0" hostname="NA NA" src-vrf-grp="N/A" dst-vrf-grp="N/A"] <14>1 2020-07-13T16:12:05.530Z SRX100HM RT_FLOW - RT_FLOW_SESSION_CREATE [junos@2636.81.2.69.144.129 source-address="10.1.1.100" source-port="49583" destination-address="175.16.199.1" destination-port="53" connection-tag="0" service-name="junos-dns-udp" nat-source-address="172.19.34.100" nat-source-port="30838" nat-destination-address="175.16.199.1" nat-destination-port="53" nat-connection-tag="0" src-nat-rule-type="source rule" src-nat-rule-name="our-nat-rule" dst-nat-rule-type="N/A" dst-nat-rule-name="N/A" protocol-id="17" policy-name="default-permit" source-zone-name="trust" destination-zone-name="untrust" session-id-32="15399" username="N/A" roles="N/A" packet-incoming-interface="ge-0/0/1.0" application="UNKNOWN" nested-application="UNKNOWN" encrypted="UNKNOWN" application-category="N/A" application-sub-category="N/A" application-risk="1" application-characteristics="N/A" src-vrf-grp="N/A" dst-vrf-grp="N/A"] <14>1 2020-07-13T16:12:05.530Z SRX100HM RT_FLOW - APPTRACK_SESSION_CLOSE [junos@2636.81.2.69.144.129 reason="Closed by junos-alg" source-address="10.1.1.100" source-port="63381" destination-address="175.16.199.1" destination-port="53" service-name="junos-dns-udp" application="UNKNOWN" nested-application="UNKNOWN" nat-source-address="172.19.34.100" nat-source-port="26764" nat-destination-address="175.16.199.1" nat-destination-port="53" src-nat-rule-name="our-nat-rule" dst-nat-rule-name="N/A" protocol-id="17" policy-name="default-permit" source-zone-name="trust" destination-zone-name="untrust" session-id-32="15361" packets-from-client="1" bytes-from-client="66" packets-from-server="1" bytes-from-server="82" elapsed-time="3" username="N/A" roles="N/A" encrypted="No" profile-name="N/A" rule-name="N/A" routing-instance="default" destination-interface-name="ge-0/0/0.0" uplink-incoming-interface-name="N/A" uplink-tx-bytes="0" uplink-rx-bytes="0" category="N/A" sub-category="N/A" apbr-policy-name="N/A" multipath-rule-name="N/A" src-vrf-grp="N/A" dst-vrf-grp="N/A"] +<14>1 2023-08-08T14:28:00.778-05:00 Route1- RT_FLOW - RT_FLOW_SESSION_DENY [source-address="192.168.1.1" source-port="39017" destination-address="8.8.4.4" destination-port="53" connection-tag="0" service-name="junos-dns-udp" protocol-id="17" icmp-type="0" policy-name="dns_deny_outbound" source-zone-name="trust" destination-zone-name="untrust" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="reth0.0" encrypted="No" reason="Denied by policy" session-id="85905209174" application-category="N/A" application-sub-category="N/A" application-risk="-1" application-characteristics="N/A" src-vrf-grp="N/A" dst-vrf-grp="N/A"] diff --git a/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json index 4f4f7275fbc..79bae3051f9 100644 --- a/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json +++ b/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json @@ -1867,5 +1867,62 @@ "forwarded", "juniper.srx" ] + }, + { + "@timestamp": "2023-08-08T17:28:00.778-02:00", + "client.ip": "192.168.1.1", + "client.port": 39017, + "destination.ip": "8.8.4.4", + "destination.port": 53, + "event.action": "flow_deny", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"192.168.1.1\" source-port=\"39017\" destination-address=\"8.8.4.4\" destination-port=\"53\" connection-tag=\"0\" service-name=\"junos-dns-udp\" protocol-id=\"17\" icmp-type=\"0\" policy-name=\"dns_deny_outbound\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth0.0\" encrypted=\"No\" reason=\"Denied by policy\" session-id=\"85905209174\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"-1\" application-characteristics=\"N/A\" src-vrf-grp=\"N/A\" dst-vrf-grp=\"N/A\"", + "event.outcome": "success", + "event.risk_score": -1.0, + "event.severity": 14, + "event.timezone": "-02:00", + "event.type": [ + "connection", + "denied" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.connection_tag": "0", + "juniper.srx.encrypted": "No", + "juniper.srx.icmp_type": "0", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "Denied by policy", + "juniper.srx.service_name": "junos-dns-udp", + "juniper.srx.session_id": "85905209174", + "juniper.srx.tag": "RT_FLOW_SESSION_DENY", + "log.level": "informational", + "log.offset": 19862, + "network.iana_number": "17", + "observer.egress.zone": "untrust", + "observer.ingress.interface.name": "reth0.0", + "observer.ingress.zone": "trust", + "observer.name": "Route1-", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "192.168.1.1", + "8.8.4.4" + ], + "rule.name": "dns_deny_outbound", + "server.ip": "8.8.4.4", + "server.port": 53, + "service.type": "juniper", + "source.ip": "192.168.1.1", + "source.port": 39017, + "tags": [ + "forwarded", + "juniper.srx" + ] } ] \ No newline at end of file