x-pack/filebeat/module/juniper: fix handling of jrx structured data (e…

…lastic#36308) Previously, the leading word was ignored. I all our test cases this was in the form /junos@(\d+\.){5}\d+/. When this value is not present, we lose the first structured data value, so be more careful in assessing whether the first element should be discarded.
Scholar-Li · Feb 5, 2024 · d2a0cea · d2a0cea
1 parent 97f0d84
commit d2a0cea
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 35 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -113,6 +113,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Update mito CEL extension library to v1.5.0. {pull}36146[36146]
 - Filter out duplicate paths resolved from matching globs. {issue}36253[36253] {pull}36256[36256]
 - Fix handling of TCP/UDP address resolution during metric initialization. {issue}35064[35064] {pull}36287[36287]
+- Fix handling of Juniper SRX structured data when there is no leading junos element. {issue}36270[36270] {pull}36308[36308]
 
 *Heartbeat*
 
@@ -142,33 +143,6 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Enable heartbeat-wide publish timeout setting with run_once. {pull}35721[35721]
 - Added default timezone UTC to heartbeat docker images to fix synthetics journeys navigation errors. {pull}36193[36193]
 
-*Heartbeat*
-
-
-*Heartbeat*
-
-
-*Heartbeat*
-
-
-*Heartbeat*
-
-
-*Auditbeat*
-
-
-*Filebeat*
-
-
-*Auditbeat*
-
-
-*Filebeat*
-
-
-*Heartbeat*
-
-
 *Metricbeat*
 
 - in module/windows/perfmon, changed collection method of the second counter value required to create a displayable value {pull}32305[32305]
@@ -200,13 +174,6 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 
 - Fix powershell details regexp to prevent excessive backtracking when processing command invocations. {pull}36178[36178]
 
-*Functionbeat*
-
-
-*Functionbeat*
-
-
-
 *Elastic Logging Plugin*
 
 

diff --git a/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml b/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml
@@ -5,7 +5,7 @@ processors:
 - grok:
     field: message
     patterns:
-    - '^<%{POSINT:syslog_pri}>(\d{1,3}\s)?(?:%{TIMESTAMP_ISO8601:_temp_.raw_date})\s%{SYSLOGHOST:syslog_hostname}\s%{PROG:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:log_type}\s\[.+?\s%{GREEDYDATA:log.original}\]$'
+    - '^<%{POSINT:syslog_pri}>(\d{1,3}\s)?(?:%{TIMESTAMP_ISO8601:_temp_.raw_date})\s%{SYSLOGHOST:syslog_hostname}\s%{PROG:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:log_type}\s\[(?:[^=]+\s)?%{GREEDYDATA:log.original}\]$'
 
 # split Juniper-SRX fields
 - kv:

diff --git a/x-pack/filebeat/module/juniper/srx/test/flow.log b/x-pack/filebeat/module/juniper/srx/test/flow.log
@@ -23,3 +23,4 @@
 <14>1 2020-07-13T16:43:05.041Z SRX100HM RT_FLOW - RT_FLOW_SESSION_CLOSE [[email protected] reason="idle Timeout" source-address="10.1.1.100" source-port="64720" destination-address="91.228.167.172" destination-port="8883" connection-tag="0" service-name="None" nat-source-address="172.19.34.100" nat-source-port="24519" nat-destination-address="91.228.167.172" nat-destination-port="8883" nat-connection-tag="0" src-nat-rule-type="source rule" src-nat-rule-name="our-nat-rule" dst-nat-rule-type="N/A" dst-nat-rule-name="N/A" protocol-id="6" policy-name="default-permit" source-zone-name="trust" destination-zone-name="untrust" session-id-32="3851" packets-from-client="161" bytes-from-client="9530" packets-from-server="96" bytes-from-server="9670" elapsed-time="23755" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="ge-0/0/1.0" encrypted="UNKNOWN" application-category="N/A" application-sub-category="N/A" application-risk="1" application-characteristics="N/A" secure-web-proxy-session-type="NA" peer-session-id="0" peer-source-address="0.0.0.0" peer-source-port="0" peer-destination-address="0.0.0.0" peer-destination-port="0" hostname="NA NA" src-vrf-grp="N/A" dst-vrf-grp="N/A"]
 <14>1 2020-07-13T16:12:05.530Z SRX100HM RT_FLOW - RT_FLOW_SESSION_CREATE [[email protected] source-address="10.1.1.100" source-port="49583" destination-address="175.16.199.1" destination-port="53" connection-tag="0" service-name="junos-dns-udp" nat-source-address="172.19.34.100" nat-source-port="30838" nat-destination-address="175.16.199.1" nat-destination-port="53" nat-connection-tag="0" src-nat-rule-type="source rule" src-nat-rule-name="our-nat-rule" dst-nat-rule-type="N/A" dst-nat-rule-name="N/A" protocol-id="17" policy-name="default-permit" source-zone-name="trust" destination-zone-name="untrust" session-id-32="15399" username="N/A" roles="N/A" packet-incoming-interface="ge-0/0/1.0" application="UNKNOWN" nested-application="UNKNOWN" encrypted="UNKNOWN" application-category="N/A" application-sub-category="N/A" application-risk="1" application-characteristics="N/A" src-vrf-grp="N/A" dst-vrf-grp="N/A"]
 <14>1 2020-07-13T16:12:05.530Z SRX100HM RT_FLOW - APPTRACK_SESSION_CLOSE [[email protected] reason="Closed by junos-alg" source-address="10.1.1.100" source-port="63381" destination-address="175.16.199.1" destination-port="53" service-name="junos-dns-udp" application="UNKNOWN" nested-application="UNKNOWN" nat-source-address="172.19.34.100" nat-source-port="26764" nat-destination-address="175.16.199.1" nat-destination-port="53" src-nat-rule-name="our-nat-rule" dst-nat-rule-name="N/A" protocol-id="17" policy-name="default-permit" source-zone-name="trust" destination-zone-name="untrust" session-id-32="15361" packets-from-client="1" bytes-from-client="66" packets-from-server="1" bytes-from-server="82" elapsed-time="3" username="N/A" roles="N/A" encrypted="No" profile-name="N/A" rule-name="N/A" routing-instance="default" destination-interface-name="ge-0/0/0.0" uplink-incoming-interface-name="N/A" uplink-tx-bytes="0" uplink-rx-bytes="0" category="N/A" sub-category="N/A" apbr-policy-name="N/A" multipath-rule-name="N/A" src-vrf-grp="N/A" dst-vrf-grp="N/A"]
+<14>1 2023-08-08T14:28:00.778-05:00 Route1- RT_FLOW - RT_FLOW_SESSION_DENY [source-address="192.168.1.1" source-port="39017" destination-address="8.8.4.4" destination-port="53" connection-tag="0" service-name="junos-dns-udp" protocol-id="17" icmp-type="0" policy-name="dns_deny_outbound" source-zone-name="trust" destination-zone-name="untrust" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="reth0.0" encrypted="No" reason="Denied by policy" session-id="85905209174" application-category="N/A" application-sub-category="N/A" application-risk="-1" application-characteristics="N/A" src-vrf-grp="N/A" dst-vrf-grp="N/A"]
diff --git a/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json
@@ -1867,5 +1867,62 @@
             "forwarded",
             "juniper.srx"
         ]
+    },
+    {
+        "@timestamp": "2023-08-08T17:28:00.778-02:00",
+        "client.ip": "192.168.1.1",
+        "client.port": 39017,
+        "destination.ip": "8.8.4.4",
+        "destination.port": 53,
+        "event.action": "flow_deny",
+        "event.category": [
+            "network"
+        ],
+        "event.dataset": "juniper.srx",
+        "event.kind": "event",
+        "event.module": "juniper",
+        "event.original": "source-address=\"192.168.1.1\" source-port=\"39017\" destination-address=\"8.8.4.4\" destination-port=\"53\" connection-tag=\"0\" service-name=\"junos-dns-udp\" protocol-id=\"17\" icmp-type=\"0\" policy-name=\"dns_deny_outbound\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth0.0\" encrypted=\"No\" reason=\"Denied by policy\" session-id=\"85905209174\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"-1\" application-characteristics=\"N/A\" src-vrf-grp=\"N/A\" dst-vrf-grp=\"N/A\"",
+        "event.outcome": "success",
+        "event.risk_score": -1.0,
+        "event.severity": 14,
+        "event.timezone": "-02:00",
+        "event.type": [
+            "connection",
+            "denied"
+        ],
+        "fileset.name": "srx",
+        "input.type": "log",
+        "juniper.srx.connection_tag": "0",
+        "juniper.srx.encrypted": "No",
+        "juniper.srx.icmp_type": "0",
+        "juniper.srx.process": "RT_FLOW",
+        "juniper.srx.reason": "Denied by policy",
+        "juniper.srx.service_name": "junos-dns-udp",
+        "juniper.srx.session_id": "85905209174",
+        "juniper.srx.tag": "RT_FLOW_SESSION_DENY",
+        "log.level": "informational",
+        "log.offset": 19862,
+        "network.iana_number": "17",
+        "observer.egress.zone": "untrust",
+        "observer.ingress.interface.name": "reth0.0",
+        "observer.ingress.zone": "trust",
+        "observer.name": "Route1-",
+        "observer.product": "SRX",
+        "observer.type": "firewall",
+        "observer.vendor": "Juniper",
+        "related.ip": [
+            "192.168.1.1",
+            "8.8.4.4"
+        ],
+        "rule.name": "dns_deny_outbound",
+        "server.ip": "8.8.4.4",
+        "server.port": 53,
+        "service.type": "juniper",
+        "source.ip": "192.168.1.1",
+        "source.port": 39017,
+        "tags": [
+            "forwarded",
+            "juniper.srx"
+        ]
     }
 ]