[citrix_adc] Handle time zone parsing in sslvpn_and_aaatm_feature pip…

…eline (elastic#10846) This has a few pipeline improvements * Fail if sslvpn_and_aaatm_feature message data cannot be parsed. If this data is not parsed, most data provided by this pipeline is silently not populated. So I think overall its better to fail, so that users and developers are more aware that there is an error. * Improve parsing of the message to handle optional space between username and group. Both formats have been observed. * Handle the presence of time zone in the message timestamp.
ali786XI · Aug 28, 2024 · 91399a8 · 91399a8
1 parent 930a301
commit 91399a8
Show file tree

Hide file tree

Showing 5 changed files with 207 additions and 5 deletions.
diff --git a/packages/citrix_adc/changelog.yml b/packages/citrix_adc/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.7.2"
+  changes:
+    - description: Parse timezone when present in sslvpn_and_aaatm_feature pipeline
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10846
 - version: "1.7.1"
   changes:
     - description: Timezone field made optional for the citrix_adc log messages

diff --git a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log
@@ -105,3 +105,5 @@ Jun 22 19:14:37 <local0.info> 81.2.69.144 06/22/2015:19:14:37 GMT ns 0-PPE-1 : S
 Oct 6 14:03:23 <local0.info> 81.2.69.144 10/06/2014:14:03:23 GMT ns1 0-PPE-0 : UI CMD_EXECUTED 4471 0 : User jane.doe - ADM_User john - Remote_ip 192.168.1.105 - Command "scp file.txt" - Status "Success"
 Oct 6 14:03:23 <local0.info> 81.2.69.144 10/06/2014:14:03:23 GMT ns1 0-PPE-0 : APPFW APPFW_REST_VALIDATION 4471 0 : Rest Validation relaxation rule: Allow hit at url: https://service.example.org/query?id=1234
 Oct 6 14:03:23 <local0.info> 81.2.69.144 10/06/2014:14:03:23 GMT ns1 0-PPE-0 : APPFW APPFW_REST_VALIDATION 4471 0 : gRPC Validation relaxation rule: Allow hit at url: https://service.example.org/query?id=1234
+Jun 22 19:14:37 <local0.info> 81.2.69.144 06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context [email protected] - SessionId: 1756710 - citrix.example.com User user.name: Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -
+Jun 22 19:14:37 <local0.info> 81.2.69.144 06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context [email protected] - SessionId: 1756710 - citrix.example.com User user.name : Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -
diff --git a/...es/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log-expected.json b/...es/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log-expected.json
@@ -8511,6 +8511,200 @@
                 "query": "id=1234",
                 "scheme": "https"
             }
+        },
+        {
+            "@timestamp": "2015-06-22T19:14:37.000Z",
+            "citrix": {
+                "cef_format": false,
+                "detail": "06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context [email protected] - SessionId: 1756710 - citrix.example.com User user.name: Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -",
+                "device_event_class_id": "SSLVPN",
+                "extended": {
+                    "message": "Context [email protected] - SessionId: 1756710 - citrix.example.com User user.name: Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -"
+                },
+                "facility": "local0",
+                "host": "ns",
+                "name": "HTTPREQUEST",
+                "priority": "info"
+            },
+            "citrix_adc": {
+                "log": {
+                    "client_ip": "81.2.69.145",
+                    "groups": "N/A",
+                    "hostname": "citrix.example.com",
+                    "message": "Context [email protected] - SessionId: 1756710 - citrix.example.com User user.name: Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -",
+                    "method": "POST",
+                    "request": {
+                        "path": "/Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre-"
+                    },
+                    "session_id": "1756710",
+                    "sso_status": "ON",
+                    "timestamp": "2024-07-12T06:54:39.000Z",
+                    "user": "user.name",
+                    "username": "user.name",
+                    "vserver": {
+                        "ip": "81.2.69.143",
+                        "port": 443
+                    }
+                }
+            },
+            "client": {
+                "geo": {
+                    "city_name": "London",
+                    "continent_name": "Europe",
+                    "country_iso_code": "GB",
+                    "country_name": "United Kingdom",
+                    "location": {
+                        "lat": 51.5142,
+                        "lon": -0.0931
+                    },
+                    "region_iso_code": "GB-ENG",
+                    "region_name": "England"
+                },
+                "ip": "81.2.69.145"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "authentication"
+                ],
+                "id": "152923587",
+                "original": "Jun 22 19:14:37 <local0.info> 81.2.69.144 06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context [email protected] - SessionId: 1756710 - citrix.example.com User user.name: Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -",
+                "severity": 0,
+                "timezone": "GMT",
+                "type": [
+                    "info"
+                ]
+            },
+            "group": {
+                "name": "N/A"
+            },
+            "observer": {
+                "product": "Netscaler",
+                "type": "firewall",
+                "vendor": "Citrix"
+            },
+            "related": {
+                "ip": [
+                    "81.2.69.143",
+                    "81.2.69.145"
+                ],
+                "user": [
+                    "user.name"
+                ]
+            },
+            "server": {
+                "ip": "81.2.69.143",
+                "port": 443
+            },
+            "tags": [
+                "preserve_original_event",
+                "preserve_duplicate_custom_fields"
+            ],
+            "url": {
+                "domain": "citrix.example.com"
+            },
+            "user": {
+                "name": "user.name"
+            }
+        },
+        {
+            "@timestamp": "2015-06-22T19:14:37.000Z",
+            "citrix": {
+                "cef_format": false,
+                "detail": "06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context [email protected] - SessionId: 1756710 - citrix.example.com User user.name : Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -",
+                "device_event_class_id": "SSLVPN",
+                "extended": {
+                    "message": "Context [email protected] - SessionId: 1756710 - citrix.example.com User user.name : Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -"
+                },
+                "facility": "local0",
+                "host": "ns",
+                "name": "HTTPREQUEST",
+                "priority": "info"
+            },
+            "citrix_adc": {
+                "log": {
+                    "client_ip": "81.2.69.145",
+                    "groups": "N/A",
+                    "hostname": "citrix.example.com",
+                    "message": "Context [email protected] - SessionId: 1756710 - citrix.example.com User user.name : Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -",
+                    "method": "POST",
+                    "request": {
+                        "path": "/Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre-"
+                    },
+                    "session_id": "1756710",
+                    "sso_status": "ON",
+                    "timestamp": "2024-07-12T06:54:39.000Z",
+                    "user": "user.name",
+                    "username": "user.name",
+                    "vserver": {
+                        "ip": "81.2.69.143",
+                        "port": 443
+                    }
+                }
+            },
+            "client": {
+                "geo": {
+                    "city_name": "London",
+                    "continent_name": "Europe",
+                    "country_iso_code": "GB",
+                    "country_name": "United Kingdom",
+                    "location": {
+                        "lat": 51.5142,
+                        "lon": -0.0931
+                    },
+                    "region_iso_code": "GB-ENG",
+                    "region_name": "England"
+                },
+                "ip": "81.2.69.145"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "authentication"
+                ],
+                "id": "152923587",
+                "original": "Jun 22 19:14:37 <local0.info> 81.2.69.144 06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context [email protected] - SessionId: 1756710 - citrix.example.com User user.name : Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -",
+                "severity": 0,
+                "timezone": "GMT",
+                "type": [
+                    "info"
+                ]
+            },
+            "group": {
+                "name": "N/A"
+            },
+            "observer": {
+                "product": "Netscaler",
+                "type": "firewall",
+                "vendor": "Citrix"
+            },
+            "related": {
+                "ip": [
+                    "81.2.69.143",
+                    "81.2.69.145"
+                ],
+                "user": [
+                    "user.name"
+                ]
+            },
+            "server": {
+                "ip": "81.2.69.143",
+                "port": 443
+            },
+            "tags": [
+                "preserve_original_event",
+                "preserve_duplicate_custom_fields"
+            ],
+            "url": {
+                "domain": "citrix.example.com"
+            },
+            "user": {
+                "name": "user.name"
+            }
         }
     ]
 }
diff --git a/...ges/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/sslvpn_and_aaatm_feature.yml b/...ges/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/sslvpn_and_aaatm_feature.yml
@@ -21,18 +21,19 @@ processors:
         - '^Session%{SPACE}id %{NUMBER:citrix_adc.log.session_id:int} - User %{USER:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver_ip %{IP:citrix_adc.log.vserver.ip} - Errmsg \"%{DATA:citrix_adc.log.errmsg}\"$'
         - '^User %{USER:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Browser_type \"%{DATA:citrix_adc.log.browser_type}\" - Group\(s\) \"%{DATA:citrix_adc.log.groups}\"$'
         - '^User %{USER:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Start_time \"%{DATA:citrix_adc.log.start_time}\" - End_time \"%{DATA:citrix_adc.log.end_time}\" - Duration %{DATA:citrix_adc.log.duration} - Http_resources_accessed %{INT:citrix_adc.log.http_resources_accessed} - Total_TCP_connections %{INT:citrix_adc.log.total_tcp_connections} - Total_policies_allowed %{INT:citrix_adc.log.total_policies_allowed} - Total_policies_denied %{INT:citrix_adc.log.total_policies_denied} - Total_bytes_send %{DATA:citrix_adc.log.total_bytes_send} - Total_bytes_recv %{DATA:citrix_adc.log.total_bytes_received} - Total_compressedbytes_send %{DATA:citrix_adc.log.total_compressed_bytes_send} - Total_compressedbytes_recv %{DATA:citrix_adc.log.total_compressed_bytes_recieved} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved}% - LogoutMethod \"%{DATA:citrix_adc.log.logout_method}\" - Group\(s\) \"%{DATA:citrix_adc.log.groups}\"$'
-        - '^Context %{USERNAME:citrix_adc.log.username}@%{IP:citrix_adc.log.client_ip} - SessionId: %{NUMBER:citrix_adc.log.session_id} - %{HOSTNAME:citrix_adc.log.hostname} User %{USERNAME:citrix_adc.log.user} : Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:citrix_adc.log.timestamp} : SSO is %{WORD:citrix_adc.log.sso_status} : %{WORD:citrix_adc.log.method} %{URIPATHPARAM:citrix_adc.log.request.path} - -$'
-        - '^Context %{DATA:citrix_adc.log.username}@%{IP:citrix_adc.log.client_ip}%{SPACE}- SessionId: %{NUMBER:citrix_adc.log.session_id}%{SPACE}- %{HOSTNAME:citrix_adc.log.hostname} User %{DATA:citrix_adc.log.user} : Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:citrix_adc.log.timestamp} %{DATA:citrix_adc.log.timezone} %{WORD:citrix_adc.log.method} %{DATA:citrix_adc.log.request.path} - -$'
-        - '^%{GREEDYDATA:citrix_adc.log.message}$'
-      ignore_failure: true
+        - '^Context %{USERNAME:citrix_adc.log.username}@%{IP:citrix_adc.log.client_ip} - SessionId: %{NUMBER:citrix_adc.log.session_id} - %{HOSTNAME:citrix_adc.log.hostname} User %{USERNAME:citrix_adc.log.user}%{SPACE}?: Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:citrix_adc.log.timestamp} : SSO is %{WORD:citrix_adc.log.sso_status} : %{WORD:citrix_adc.log.method} %{URIPATHPARAM:citrix_adc.log.request.path} - -$'
+        - '^Context %{DATA:citrix_adc.log.username}@%{IP:citrix_adc.log.client_ip}%{SPACE}- SessionId: %{NUMBER:citrix_adc.log.session_id}%{SPACE}?- %{HOSTNAME:citrix_adc.log.hostname} User %{DATA:citrix_adc.log.user} : Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:citrix_adc.log.timestamp} %{DATA:citrix_adc.log.timezone} %{WORD:citrix_adc.log.method} %{DATA:citrix_adc.log.request.path} - -$'
+      ignore_failure: false
   - date:
       field: citrix_adc.log.timestamp
       tag: date_timestamp
       target_field: citrix_adc.log.timestamp
       formats:
         - ISO8601
         - MM/dd/yyyy:HH:mm:ss
+        - MM/dd/yyyy:HH:mm:ss z
         - yyyy/MM/dd:HH:mm:ss
+        - yyyy/MM/dd:HH:mm:ss z
       if: ctx.citrix_adc?.log?.timestamp != null && ctx.citrix_adc.log.timestamp != ''
       on_failure:
         - append:

diff --git a/packages/citrix_adc/manifest.yml b/packages/citrix_adc/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: citrix_adc
 title: Citrix ADC
-version: "1.7.1"
+version: "1.7.2"
 description: This Elastic integration collects logs and metrics from Citrix ADC product.
 type: integration
 categories: