Fix url scheme grok pattern

elastic · Jun 13, 2023 · f47347b · f47347b
1 parent 4af3e60
commit f47347b
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 44 deletions.
diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -82,6 +82,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Fix the ingest pipeline for mysql slowlog to parse schema name with dash {pull}34371[34372]
 - Fix the multiple host support for mongodb module {pull}34624[34624]
 - Skip HTTPJSON flakey test. {issue}34929[34929] {pull}35138[35138]
+- Fix ingest pipeline for panw module to parse url scheme correctly {pull}35757[35757]
 
 ==== Added
 

diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml
@@ -437,7 +437,7 @@ processors:
   - grok:
       field: url.original
       patterns:
-        - '(%{ANY:url.scheme}\:\/\/)?(%{USERNAME:url.username}(\:%{PASSWORD:url.password})?\@)?%{DOMAIN:url.domain}(\:%{POSINT:url.port})?(%{PATH:url.path})?(\?%{QUERY:url.query})?(\#%{ANY:url.fragment})?'
+        - '(%{URIPROTO:url.scheme}\:\/\/)?(%{USERNAME:url.username}(\:%{PASSWORD:url.password})?\@)?%{DOMAIN:url.domain}(\:%{POSINT:url.port})?(%{PATH:url.path})?(\?%{QUERY:url.query})?(\#%{ANY:url.fragment})?'
       ignore_missing: true
       pattern_definitions:
         USERNAME: '[^\:]*'

diff --git a/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log b/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log
@@ -62,7 +62,7 @@ Mar 25 23:59:12 1,2013/03/25 23:59:12,01606001116,THREAT,url,1,2012/04/10 04:42:
 Mar 25 23:59:12 1,2013/03/25 23:59:12,01606001116,THREAT,url,1,2012/04/10 04:42:42,192.168.0.2,213.180.199.61,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:42:42,28932,1,59721,80,0,0,0x200000,tcp,block-url,"edw-melon.narod.ru/config.txt",(9999),malware-sites,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,Russian Federation,0,
 Mar 25 23:59:12 1,2013/03/25 23:59:12,01606001116,THREAT,url,1,2012/04/10 04:42:51,192.168.0.2,213.180.199.61,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:42:51,28953,1,59752,80,0,0,0x200000,tcp,block-url,"maximtushin.narod.ru/config.txt",(9999),malware-sites,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,Russian Federation,0,
 Mar 25 23:59:17 1,2013/03/25 23:59:17,01606001116,THREAT,file,1,2012/04/10 04:19:59,89.160.20.112,192.168.0.2,0.0.0.0,0.0.0.0,rule1,,crusher,web-browsing,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:20:05,64856,1,80,54431,0,0,0x200000,tcp,deny,"uLLGRaXP.exe",Windows Executable (EXE)(52020),any,low,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0,
-Mar 25 23:59:22 1,2013/03/25 23:59:22,01606001116,THREAT,url,1,2012/04/10 04:09:01,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:09:01,55402,1,63183,80,0,0,0x200000,tcp,block-url,"marketingsoluchion.biz/fkn/config.bin",(9999),unknown,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,
+Mar 25 23:59:22 1,2013/03/25 23:59:22,01606001116,THREAT,url,1,2012/04/10 04:09:01,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:09:01,55402,1,63183,80,0,0,0x200000,tcp,block-url,"www.sportspar.de/widgets/index/refreshStatistic?requestPage=/&requestController=index&referer=https://www.google.com/",(9999),unknown,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,
 Mar 25 23:59:32 1,2013/03/25 23:59:32,01606001116,THREAT,data,1,2012/04/09 08:18:27,192.168.0.6,207.46.140.46,0.0.0.0,0.0.0.0,rule1,jordy,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 08:18:32,25217,1,1047,80,0,0,0x200000,tcp,alert,"default.aspx",PII(60000),any,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,
 Mar 25 23:59:32 1,2013/03/25 23:59:32,01606001116,THREAT,data,1,2012/04/09 08:18:29,81.2.69.143,192.168.0.6,0.0.0.0,0.0.0.0,rule1,,jordy,web-browsing,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 08:18:34,25653,1,80,1039,0,0,0x200000,tcp,alert,"sck.aspx",PII(60000),any,informational,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0,
 Mar 25 23:59:32 1,2013/03/25 23:59:32,01606001116,THREAT,data,1,2012/04/09 08:18:32,81.2.69.143,192.168.0.6,0.0.0.0,0.0.0.0,rule1,,jordy,web-browsing,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 08:18:37,25717,3,80,1064,0,0,0x200000,tcp,alert,"ADSAdClient31.dll",PII(60000),any,informational,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0,

diff --git a/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log-expected.json b/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log-expected.json
@@ -6010,7 +6010,7 @@
         "event.dataset": "panw.panos",
         "event.kind": "alert",
         "event.module": "panw",
-        "event.original": "Mar 25 23:59:22 1,2013/03/25 23:59:22,01606001116,THREAT,url,1,2012/04/10 04:09:01,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:09:01,55402,1,63183,80,0,0,0x200000,tcp,block-url,\"marketingsoluchion.biz/fkn/config.bin\",(9999),unknown,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,",
+        "event.original": "Mar 25 23:59:22 1,2013/03/25 23:59:22,01606001116,THREAT,url,1,2012/04/10 04:09:01,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:09:01,55402,1,63183,80,0,0,0x200000,tcp,block-url,\"www.sportspar.de/widgets/index/refreshStatistic?requestPage=/&requestController=index&referer=https://www.google.com/\",(9999),unknown,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,",
         "event.outcome": "success",
         "event.severity": 5,
         "event.timezone": "-02:00",
@@ -6048,12 +6048,12 @@
         "panw.panos.sub_type": "url",
         "panw.panos.threat.id": "9999",
         "panw.panos.threat.name": "URL-filtering",
-        "panw.panos.threat.resource": "marketingsoluchion.biz/fkn/config.bin",
+        "panw.panos.threat.resource": "www.sportspar.de/widgets/index/refreshStatistic?requestPage=/&requestController=index&referer=https://www.google.com/",
         "panw.panos.type": "THREAT",
         "panw.panos.url.category": "unknown",
         "panw.panos.virtual_sys": "vsys1",
         "related.hosts": [
-            "marketingsoluchion.biz"
+            "www.sportspar.de"
         ],
         "related.ip": [
             "0.0.0.0",
@@ -6076,10 +6076,10 @@
             "forwarded",
             "pan-os"
         ],
-        "url.domain": "marketingsoluchion.biz",
-        "url.extension": "bin",
-        "url.original": "marketingsoluchion.biz/fkn/config.bin",
-        "url.path": "/fkn/config.bin",
+        "url.domain": "www.sportspar.de",
+        "url.original": "www.sportspar.de/widgets/index/refreshStatistic?requestPage=/&requestController=index&referer=https://www.google.com/",
+        "url.path": "/widgets/index/refreshStatistic",
+        "url.query": "requestPage=/&requestController=index&referer=https://www.google.com/",
         "user.name": "crusher"
     },
     {
@@ -6113,7 +6113,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 26586,
+        "log.offset": 26666,
         "network.application": "web-browsing",
         "network.community_id": "1:KC3xpBK9CdouZqamG9S6Mjl6LIo=",
         "network.direction": "inbound",
@@ -6198,7 +6198,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 26964,
+        "log.offset": 27044,
         "network.application": "web-browsing",
         "network.community_id": "1:oZUSrEMVr54enE9TsNjtdpJu0L8=",
         "network.direction": "outbound",
@@ -6290,7 +6290,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 27336,
+        "log.offset": 27416,
         "network.application": "web-browsing",
         "network.community_id": "1:vpvx2rrEII2Wtti+NqSoe98K6s4=",
         "network.direction": "outbound",
@@ -6382,7 +6382,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 27717,
+        "log.offset": 27797,
         "network.application": "web-browsing",
         "network.community_id": "1:MeB0cefg5kMN7f+LW+cirwH2nA8=",
         "network.direction": "inbound",
@@ -6466,7 +6466,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 28086,
+        "log.offset": 28166,
         "network.application": "web-browsing",
         "network.community_id": "1:lI0hgoESF7/v82QAbsIMoPxInGQ=",
         "network.direction": "outbound",
@@ -6560,7 +6560,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 28455,
+        "log.offset": 28535,
         "network.application": "pandora",
         "network.community_id": "1:c67I85z1uJV7VW6M9MR5Q8fjHQM=",
         "network.direction": "inbound",
@@ -6644,7 +6644,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 28843,
+        "log.offset": 28923,
         "network.application": "google-maps",
         "network.community_id": "1:tsjbpnOPfE5+wHs/9MImDTjVjp8=",
         "network.direction": "outbound",
@@ -6736,7 +6736,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "low",
-        "log.offset": 29215,
+        "log.offset": 29295,
         "network.application": "web-browsing",
         "network.community_id": "1:a/X3iTqQa+TxkHJgrAy4Npfe+ZM=",
         "network.direction": "outbound",
@@ -6821,7 +6821,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 29590,
+        "log.offset": 29670,
         "network.application": "google-maps",
         "network.community_id": "1:Tc4KEUPBViPeku88f+PNN9tpeuc=",
         "network.direction": "outbound",
@@ -6912,7 +6912,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 29962,
+        "log.offset": 30042,
         "network.application": "google-maps",
         "network.community_id": "1:OjvHxM13sIYbWzkV4RtvyxXDyVM=",
         "network.direction": "outbound",
@@ -7004,7 +7004,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 30336,
+        "log.offset": 30416,
         "network.application": "google-maps",
         "network.community_id": "1:kYzGF0Llye+Lln7ejrGG5SI6mW8=",
         "network.direction": "outbound",
@@ -7096,7 +7096,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 30710,
+        "log.offset": 30790,
         "network.application": "google-maps",
         "network.community_id": "1:AwfQlEV4j9qZjH7WG4q1qExon/o=",
         "network.direction": "outbound",
@@ -7188,7 +7188,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 31082,
+        "log.offset": 31162,
         "network.application": "google-analytics",
         "network.community_id": "1:pRuFj5DzdmtFceU+OTawbYPhbJg=",
         "network.direction": "inbound",
@@ -7272,7 +7272,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 31462,
+        "log.offset": 31542,
         "network.application": "google-maps",
         "network.community_id": "1:PFB0Gj5/utCZj8v3vJPCiBrGY3Y=",
         "network.direction": "outbound",
@@ -7365,7 +7365,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 31836,
+        "log.offset": 31916,
         "network.application": "web-browsing",
         "network.community_id": "1:N/Bc1RgG30q1Owz0DWHR2yEwN44=",
         "network.direction": "outbound",
@@ -7450,7 +7450,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 32215,
+        "log.offset": 32295,
         "network.application": "web-browsing",
         "network.community_id": "1:mSmmKo9krpIsh+2qFAZoA8nMDhg=",
         "network.direction": "outbound",
@@ -7540,7 +7540,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 32600,
+        "log.offset": 32680,
         "network.application": "web-browsing",
         "network.community_id": "1:03rrdI/L+dbrLea/vrQULMTFqvU=",
         "network.direction": "outbound",
@@ -7632,7 +7632,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 32974,
+        "log.offset": 33054,
         "network.application": "web-browsing",
         "network.community_id": "1:bJxw0tI76mNYOiv1ZJjBXdDpnTU=",
         "network.direction": "outbound",
@@ -7721,7 +7721,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 33378,
+        "log.offset": 33458,
         "network.application": "google-maps",
         "network.community_id": "1:h4FhwHd9ztu4jpl3xgOaiB011a4=",
         "network.direction": "outbound",
@@ -7812,7 +7812,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 33749,
+        "log.offset": 33829,
         "network.application": "google-maps",
         "network.community_id": "1:dULQBKOE61wtZ1QM6GKohdrM1GE=",
         "network.direction": "outbound",
@@ -7904,7 +7904,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 34119,
+        "log.offset": 34199,
         "network.application": "rss",
         "network.community_id": "1:DLYH0WNYoXQ93i3rnp9QFsh63iM=",
         "network.direction": "outbound",
@@ -7993,7 +7993,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 34486,
+        "log.offset": 34566,
         "network.application": "google-maps",
         "network.community_id": "1:jorKmgA/OY669gtX62Fasc1iKGc=",
         "network.direction": "outbound",
@@ -8084,7 +8084,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 34858,
+        "log.offset": 34938,
         "network.application": "web-browsing",
         "network.community_id": "1:v/xhtv/qhJVgrOjMPvPqMWlrHXA=",
         "network.direction": "outbound",
@@ -8170,7 +8170,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 35225,
+        "log.offset": 35305,
         "network.application": "web-browsing",
         "network.community_id": "1:lM6ErOc/Uj5ui7hk5LvnxpCB/K0=",
         "network.direction": "outbound",
@@ -8261,7 +8261,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 35600,
+        "log.offset": 35680,
         "network.application": "google-maps",
         "network.community_id": "1:AFqpyz1JYwEsC+Bm2Q7fspI+r8Y=",
         "network.direction": "outbound",
@@ -8363,7 +8363,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 35972,
+        "log.offset": 36052,
         "network.application": "google-analytics",
         "network.community_id": "1:8xEo6/LvOntD+xMHdXzKIXv9JxE=",
         "network.direction": "inbound",
@@ -8447,7 +8447,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 36353,
+        "log.offset": 36433,
         "network.application": "google-maps",
         "network.community_id": "1:diAtdns9tWiH2bS++Pup9kMV+AI=",
         "network.direction": "outbound",
@@ -8538,7 +8538,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 36725,
+        "log.offset": 36805,
         "network.application": "google-maps",
         "network.community_id": "1:cs7mutkQqIorGFAbWD2/09AnYXk=",
         "network.direction": "outbound",
@@ -8630,7 +8630,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 37097,
+        "log.offset": 37177,
         "network.application": "pandora",
         "network.community_id": "1:PzMJQoALQDxnDaqwOEEz4zxyhHU=",
         "network.direction": "inbound",
@@ -8714,7 +8714,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 37484,
+        "log.offset": 37564,
         "network.application": "google-maps",
         "network.community_id": "1:8xnlPG6iTh0CwnSMVwmWkniCAeM=",
         "network.direction": "outbound",
@@ -8807,7 +8807,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 37857,
+        "log.offset": 37937,
         "network.application": "google-maps",
         "network.community_id": "1:SQGgi8ETBszNJv+EzlSRiGB/m5A=",
         "network.direction": "outbound",
@@ -8900,7 +8900,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 38228,
+        "log.offset": 38308,
         "network.application": "google-maps",
         "network.community_id": "1:21uyYLV+/XbEeb+gCdBr5K1MWLU=",
         "network.direction": "outbound",
@@ -8991,7 +8991,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 38597,
+        "log.offset": 38677,
         "network.application": "google-maps",
         "network.community_id": "1:QEEd+0of3hSmO6x9aRpIaHXdaUI=",
         "network.direction": "outbound",
@@ -9083,7 +9083,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 38967,
+        "log.offset": 39047,
         "network.application": "google-analytics",
         "network.community_id": "1:BnyjuRL2HOxT/uRoNE3ra3neRSY=",
         "network.direction": "outbound",
@@ -9174,7 +9174,7 @@
         "input.type": "log",
         "labels.captive_portal": true,
         "log.level": "informational",
-        "log.offset": 39339,
+        "log.offset": 39419,
         "network.application": "google-maps",
         "network.community_id": "1:eGnclJrBulAHa+EiT+kLvValbJE=",
         "network.direction": "outbound",