From 91399a85a7f273496c9604bc898467b6ba0fc651 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Wed, 28 Aug 2024 13:09:36 -0700 Subject: [PATCH] [citrix_adc] Handle time zone parsing in sslvpn_and_aaatm_feature pipeline (#10846) This has a few pipeline improvements * Fail if sslvpn_and_aaatm_feature message data cannot be parsed. If this data is not parsed, most data provided by this pipeline is silently not populated. So I think overall its better to fail, so that users and developers are more aware that there is an error. * Improve parsing of the message to handle optional space between username and group. Both formats have been observed. * Handle the presence of time zone in the message timestamp. --- packages/citrix_adc/changelog.yml | 5 + .../test/pipeline/test-citrix-waf-native.log | 2 + .../test-citrix-waf-native.log-expected.json | 194 ++++++++++++++++++ .../sslvpn_and_aaatm_feature.yml | 9 +- packages/citrix_adc/manifest.yml | 2 +- 5 files changed, 207 insertions(+), 5 deletions(-) diff --git a/packages/citrix_adc/changelog.yml b/packages/citrix_adc/changelog.yml index 86893f434ae..fe6c02357e5 100644 --- a/packages/citrix_adc/changelog.yml +++ b/packages/citrix_adc/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.7.2" + changes: + - description: Parse timezone when present in sslvpn_and_aaatm_feature pipeline + type: bugfix + link: https://github.com/elastic/integrations/pull/10846 - version: "1.7.1" changes: - description: Timezone field made optional for the citrix_adc log messages diff --git a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log index 652093ff421..636b3fc957e 100644 --- a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log +++ b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log @@ -105,3 +105,5 @@ Jun 22 19:14:37 81.2.69.144 06/22/2015:19:14:37 GMT ns 0-PPE-1 : S Oct 6 14:03:23 81.2.69.144 10/06/2014:14:03:23 GMT ns1 0-PPE-0 : UI CMD_EXECUTED 4471 0 : User jane.doe - ADM_User john - Remote_ip 192.168.1.105 - Command "scp file.txt" - Status "Success" Oct 6 14:03:23 81.2.69.144 10/06/2014:14:03:23 GMT ns1 0-PPE-0 : APPFW APPFW_REST_VALIDATION 4471 0 : Rest Validation relaxation rule: Allow hit at url: https://service.example.org/query?id=1234 Oct 6 14:03:23 81.2.69.144 10/06/2014:14:03:23 GMT ns1 0-PPE-0 : APPFW APPFW_REST_VALIDATION 4471 0 : gRPC Validation relaxation rule: Allow hit at url: https://service.example.org/query?id=1234 +Jun 22 19:14:37 81.2.69.144 06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context user.name@81.2.69.145 - SessionId: 1756710 - citrix.example.com User user.name: Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - - +Jun 22 19:14:37 81.2.69.144 06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context user.name@81.2.69.145 - SessionId: 1756710 - citrix.example.com User user.name : Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - - diff --git a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log-expected.json b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log-expected.json index a859cb02264..5b48927e240 100644 --- a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log-expected.json +++ b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-waf-native.log-expected.json @@ -8511,6 +8511,200 @@ "query": "id=1234", "scheme": "https" } + }, + { + "@timestamp": "2015-06-22T19:14:37.000Z", + "citrix": { + "cef_format": false, + "detail": "06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context user.name@81.2.69.145 - SessionId: 1756710 - citrix.example.com User user.name: Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -", + "device_event_class_id": "SSLVPN", + "extended": { + "message": "Context user.name@81.2.69.145 - SessionId: 1756710 - citrix.example.com User user.name: Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -" + }, + "facility": "local0", + "host": "ns", + "name": "HTTPREQUEST", + "priority": "info" + }, + "citrix_adc": { + "log": { + "client_ip": "81.2.69.145", + "groups": "N/A", + "hostname": "citrix.example.com", + "message": "Context user.name@81.2.69.145 - SessionId: 1756710 - citrix.example.com User user.name: Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -", + "method": "POST", + "request": { + "path": "/Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre-" + }, + "session_id": "1756710", + "sso_status": "ON", + "timestamp": "2024-07-12T06:54:39.000Z", + "user": "user.name", + "username": "user.name", + "vserver": { + "ip": "81.2.69.143", + "port": 443 + } + } + }, + "client": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.145" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "authentication" + ], + "id": "152923587", + "original": "Jun 22 19:14:37 81.2.69.144 06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context user.name@81.2.69.145 - SessionId: 1756710 - citrix.example.com User user.name: Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -", + "severity": 0, + "timezone": "GMT", + "type": [ + "info" + ] + }, + "group": { + "name": "N/A" + }, + "observer": { + "product": "Netscaler", + "type": "firewall", + "vendor": "Citrix" + }, + "related": { + "ip": [ + "81.2.69.143", + "81.2.69.145" + ], + "user": [ + "user.name" + ] + }, + "server": { + "ip": "81.2.69.143", + "port": 443 + }, + "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields" + ], + "url": { + "domain": "citrix.example.com" + }, + "user": { + "name": "user.name" + } + }, + { + "@timestamp": "2015-06-22T19:14:37.000Z", + "citrix": { + "cef_format": false, + "detail": "06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context user.name@81.2.69.145 - SessionId: 1756710 - citrix.example.com User user.name : Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -", + "device_event_class_id": "SSLVPN", + "extended": { + "message": "Context user.name@81.2.69.145 - SessionId: 1756710 - citrix.example.com User user.name : Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -" + }, + "facility": "local0", + "host": "ns", + "name": "HTTPREQUEST", + "priority": "info" + }, + "citrix_adc": { + "log": { + "client_ip": "81.2.69.145", + "groups": "N/A", + "hostname": "citrix.example.com", + "message": "Context user.name@81.2.69.145 - SessionId: 1756710 - citrix.example.com User user.name : Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -", + "method": "POST", + "request": { + "path": "/Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre-" + }, + "session_id": "1756710", + "sso_status": "ON", + "timestamp": "2024-07-12T06:54:39.000Z", + "user": "user.name", + "username": "user.name", + "vserver": { + "ip": "81.2.69.143", + "port": 443 + } + } + }, + "client": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.145" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "authentication" + ], + "id": "152923587", + "original": "Jun 22 19:14:37 81.2.69.144 06/22/2015:19:14:37 GMT ns 0-PPE-1 : SSLVPN HTTPREQUEST 152923587 0 : Context user.name@81.2.69.145 - SessionId: 1756710 - citrix.example.com User user.name : Group(s) N/A : Vserver 81.2.69.143:443 - 07/12/2024:06:54:39 GMT : SSO is ON : POST /Citrix/RIM-StoreWeb/Resources/GetLaunchStatus/Q29udHJvbGx89fsj3k9Sf23mk0d9823uRDGre- - -", + "severity": 0, + "timezone": "GMT", + "type": [ + "info" + ] + }, + "group": { + "name": "N/A" + }, + "observer": { + "product": "Netscaler", + "type": "firewall", + "vendor": "Citrix" + }, + "related": { + "ip": [ + "81.2.69.143", + "81.2.69.145" + ], + "user": [ + "user.name" + ] + }, + "server": { + "ip": "81.2.69.143", + "port": 443 + }, + "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields" + ], + "url": { + "domain": "citrix.example.com" + }, + "user": { + "name": "user.name" + } } ] } \ No newline at end of file diff --git a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/sslvpn_and_aaatm_feature.yml b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/sslvpn_and_aaatm_feature.yml index 97a08b44c8e..56526fdb67c 100644 --- a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/sslvpn_and_aaatm_feature.yml +++ b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/sslvpn_and_aaatm_feature.yml @@ -21,10 +21,9 @@ processors: - '^Session%{SPACE}id %{NUMBER:citrix_adc.log.session_id:int} - User %{USER:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver_ip %{IP:citrix_adc.log.vserver.ip} - Errmsg \"%{DATA:citrix_adc.log.errmsg}\"$' - '^User %{USER:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Browser_type \"%{DATA:citrix_adc.log.browser_type}\" - Group\(s\) \"%{DATA:citrix_adc.log.groups}\"$' - '^User %{USER:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Start_time \"%{DATA:citrix_adc.log.start_time}\" - End_time \"%{DATA:citrix_adc.log.end_time}\" - Duration %{DATA:citrix_adc.log.duration} - Http_resources_accessed %{INT:citrix_adc.log.http_resources_accessed} - Total_TCP_connections %{INT:citrix_adc.log.total_tcp_connections} - Total_policies_allowed %{INT:citrix_adc.log.total_policies_allowed} - Total_policies_denied %{INT:citrix_adc.log.total_policies_denied} - Total_bytes_send %{DATA:citrix_adc.log.total_bytes_send} - Total_bytes_recv %{DATA:citrix_adc.log.total_bytes_received} - Total_compressedbytes_send %{DATA:citrix_adc.log.total_compressed_bytes_send} - Total_compressedbytes_recv %{DATA:citrix_adc.log.total_compressed_bytes_recieved} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved}% - LogoutMethod \"%{DATA:citrix_adc.log.logout_method}\" - Group\(s\) \"%{DATA:citrix_adc.log.groups}\"$' - - '^Context %{USERNAME:citrix_adc.log.username}@%{IP:citrix_adc.log.client_ip} - SessionId: %{NUMBER:citrix_adc.log.session_id} - %{HOSTNAME:citrix_adc.log.hostname} User %{USERNAME:citrix_adc.log.user} : Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:citrix_adc.log.timestamp} : SSO is %{WORD:citrix_adc.log.sso_status} : %{WORD:citrix_adc.log.method} %{URIPATHPARAM:citrix_adc.log.request.path} - -$' - - '^Context %{DATA:citrix_adc.log.username}@%{IP:citrix_adc.log.client_ip}%{SPACE}- SessionId: %{NUMBER:citrix_adc.log.session_id}%{SPACE}- %{HOSTNAME:citrix_adc.log.hostname} User %{DATA:citrix_adc.log.user} : Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:citrix_adc.log.timestamp} %{DATA:citrix_adc.log.timezone} %{WORD:citrix_adc.log.method} %{DATA:citrix_adc.log.request.path} - -$' - - '^%{GREEDYDATA:citrix_adc.log.message}$' - ignore_failure: true + - '^Context %{USERNAME:citrix_adc.log.username}@%{IP:citrix_adc.log.client_ip} - SessionId: %{NUMBER:citrix_adc.log.session_id} - %{HOSTNAME:citrix_adc.log.hostname} User %{USERNAME:citrix_adc.log.user}%{SPACE}?: Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:citrix_adc.log.timestamp} : SSO is %{WORD:citrix_adc.log.sso_status} : %{WORD:citrix_adc.log.method} %{URIPATHPARAM:citrix_adc.log.request.path} - -$' + - '^Context %{DATA:citrix_adc.log.username}@%{IP:citrix_adc.log.client_ip}%{SPACE}- SessionId: %{NUMBER:citrix_adc.log.session_id}%{SPACE}?- %{HOSTNAME:citrix_adc.log.hostname} User %{DATA:citrix_adc.log.user} : Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:citrix_adc.log.timestamp} %{DATA:citrix_adc.log.timezone} %{WORD:citrix_adc.log.method} %{DATA:citrix_adc.log.request.path} - -$' + ignore_failure: false - date: field: citrix_adc.log.timestamp tag: date_timestamp @@ -32,7 +31,9 @@ processors: formats: - ISO8601 - MM/dd/yyyy:HH:mm:ss + - MM/dd/yyyy:HH:mm:ss z - yyyy/MM/dd:HH:mm:ss + - yyyy/MM/dd:HH:mm:ss z if: ctx.citrix_adc?.log?.timestamp != null && ctx.citrix_adc.log.timestamp != '' on_failure: - append: diff --git a/packages/citrix_adc/manifest.yml b/packages/citrix_adc/manifest.yml index 16edf1eb97e..e52020118e7 100644 --- a/packages/citrix_adc/manifest.yml +++ b/packages/citrix_adc/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: citrix_adc title: Citrix ADC -version: "1.7.1" +version: "1.7.2" description: This Elastic integration collects logs and metrics from Citrix ADC product. type: integration categories: