Updated fields rebase 2 (#344)

* Update regex to parse IPv6 (#334) Update default listener log path * Add HAProxy Plugin (#335) * Add haproxy plugin * Add supported platforms and min stanza version * PR Feedback fixes * Rename frontend_name to frontend_name_transport in regex * for all move operations, check if field is nil before moving. "set log type to haproxy and haproxy.error (not .http / .tcp)" * typoe: nill --> nil * typo, log_format: http --> default Co-authored-by: jsirianni <[email protected]> * Allow DBID to be empty & Correct case matching (#331) * Allow DBID to be empty & Correct case matching The DBID field is able to be empty on some versions of Oracle DB The multiline regex was looking for `Audit File`, but logs have `Audit file` * Switch to line end for multiline with double newline pattern * Fix plugin failure when using inline truncate check * Switch back to a regex parse for record splitting Co-authored-by: jsirianni <[email protected]> * Release 0.0.79 (#336) * 0.0.79 changelog * dbid oracle pr * fix release date * move frontend port to resources (#338) * Add more checks to reduce errors (#337) * Add more checks to reduce errors * Add ac_lite_ap_parser change to changelog for ubiquiti * 0.0.80 changelog Co-authored-by: jsirianni <[email protected]> * rebase the stanza-plugins changes * fix haproxy * fix ubiquiti * fix labels rather than attributes on operator field * oracledb attributes * fix haproxy * update regex to handle {} brackets before http request info (#342) * update regex to handle {} brackets before http request info * haproxy http default log format fix * make change backwards compatible * Adjust parsing further based on more detailed oracle db audit logs (#343) * release 0.0.82 Co-authored-by: Dylan Myers <[email protected]> Co-authored-by: EricWHolt <[email protected]> Co-authored-by: jsirianni <[email protected]> Co-authored-by: jsirianni <[email protected]>
observIQ · Sep 28, 2021 · cdb9666 · cdb9666
1 parent 7c1a014
commit cdb9666
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.0.82] 2021-09-28
+
+### Fixed
+
+- OracleDB: Resolved parsing issue related to multiple audit log formats [PR341](https://github.com/observIQ/stanza-plugins/pull/343)
+
+## [0.0.81] 2021-09-28
+
+### Fixed
+- HAProxy: Resolved an issue where http logs using default format can fail to parse ([PR342](https://github.com/observIQ/stanza-plugins/pull/342))
+
 ## [0.0.80] 2021-09-23
 
 ### Fixed

diff --git a/plugins/haproxy.yaml b/plugins/haproxy.yaml
@@ -72,7 +72,7 @@ pipeline:
   - id: httplog_parser
     type: regex_parser
     parse_from: $body.message
-    regex: '^(\s)?(?P<frontend_ip>[^:]+):(?P<frontend_port>[^\s]+)\s+\[(?P<accept_date>[^\]]+)\]\s+(?P<frontend_name_transport>[^\s]+)\s+(?P<backend_name>[^/]+)/(?P<server_name>[^\s]+)\s+(?P<client_request_send_time>[^/]+)/(?P<queue_wait_time>[^/]+)/(?P<response_time>[^/]+)/(?P<response_send_time>[^/]+)/(?P<client_request_active_time>[^\s]+)\s+(?P<status>[^\s]+)\s+(?P<bytes_read>[^\s]+)\s+(?P<captured_request_cookie>[^\s]+)\s+(?P<captured_response_cookie>[^\s]+)\s+(?P<termination_state>[\w-]{4})\s+(?P<process_concurrent_connections>[^/]+)/(?P<frontend_concurrent_connections>[^/]+)/(?P<backend_concurrent_connections>[^/]+)/(?P<server_concurrent_connections>[^/]+)/(?P<retries>[^\s]+)\s+(?P<server_queue>[^/]+)/(?P<backend_queue>[^\s]+)\s+"(?P<method>\S+) +(?P<uri>[^ ]*)( (?P<protocol>[^/]*)/(?P<protocol_version>[^\"]*)|[^\"]*)?"'
+    regex: '^(\s)?(?P<frontend_ip>[^:]+):(?P<frontend_port>[^\s]+)\s+\[(?P<accept_date>[^\]]+)\]\s+(?P<frontend_name_transport>[^\s]+)\s+(?P<backend_name>[^/]+)/(?P<server_name>[^\s]+)\s+(?P<client_request_send_time>[^/]+)/(?P<queue_wait_time>[^/]+)/(?P<response_time>[^/]+)/(?P<response_send_time>[^/]+)/(?P<client_request_active_time>[^\s]+)\s+(?P<status>[^\s]+)\s+(?P<bytes_read>[^\s]+)\s+(?P<captured_request_cookie>[^\s]+)\s+(?P<captured_response_cookie>[^\s]+)\s+(?P<termination_state>[\w-]{4})\s+(?P<process_concurrent_connections>[^/]+)/(?P<frontend_concurrent_connections>[^/]+)/(?P<backend_concurrent_connections>[^/]+)/(?P<server_concurrent_connections>[^/]+)/(?P<retries>[^\s]+)\s+(?P<server_queue>[^/]+)/(?P<backend_queue>[^\s]+) ({[\w\d[:ascii:]]+?}\s)?"(?P<method>\S+) +(?P<uri>[^ ]*)( (?P<protocol>[^/]*)/(?P<protocol_version>[^\"]*)|[^\"]*)?"'
     output: frontend_type_http_add
 
   - id: frontend_type_http_add

diff --git a/plugins/oracledb.yaml b/plugins/oracledb.yaml
@@ -83,14 +83,28 @@ pipeline:
   - id: audit_router
     type: router
     routes:
-      - output: audit_regex_parser
-        expr: $ matches '\\w+ \\w+\\s{1,2}\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\d{4} [-+]\\d{2}:\\d{2}'
+      - output: audit_regex_parser_action
+        expr: $ matches '\\w+ \\w+\\s{1,2}\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\d{4} [-+]\\d{2}:\\d{2}\\nLENGTH\\s:\\s\\D\\d+\\D\\nACTION'
+      - output: audit_regex_parser_session
+        expr: $ matches '\\w+ \\w+\\s{1,2}\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\d{4} [-+]\\d{2}:\\d{2}\\nLENGTH:\\s\\D\\d+\\D\\nSESSION'
       - output: server_start_regex_parser
         expr: $ startsWith 'Audit file '
 
-  - id: audit_regex_parser
+  - id: audit_regex_parser_action
     type: regex_parser
-    regex: '(?P<timestamp>\w+ \w+\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4} [-+]\d{2}:\d{2})\nLENGTH : \D(?P<length>\d*)\D\nACTION :\[\d+\]\s+\D(?P<action>[\d\w[:ascii:]]+)\D\nDATABASE USER:\[\d+\]\s+\D(?P<database_user>[^\s]+)\D\n(PRIVILEGE :\[\d+\]\s+\D(?P<privilege>[^\s]+)\D\n)?(CLIENT USER:\[\d+\]\s+\D(?P<client_user>[^\s]+|)\D\n)?(CLIENT TERMINAL:\[\d+\]\s+\D(?P<client_terminal>[^\s]+|)\D\n)?(STATUS:\[\d+\]\s+\D(?P<status_code>[^\s]+|)\D\n)?(DBID:\[\d+\]\s\D(?P<dbid>\d+|)\D\n)?(SESSIONID:\[\d+\]\s+\D(?P<sessionid>[^\s]+|)\D\n)?(USERHOST:\[\d+\]\s+\D(?P<userhost>[^\s]+|)\D\n)?(CLIENT ADDRESS:\[\d+\]\s+\D(?P<client_address>[^\s]+|)\D\n)?(ACTION NUMBER:\[\d+\]\s+\D(?P<action_number>[^\s]+|)\D)?'
+    regex: '^(?P<timestamp>\w+ \w+\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4} [-+]\d{2}:\d{2})\nLENGTH : \D(?P<length>\d*)\D\nACTION :\[\d+\]\s+\D(?P<action>[\d\w[:ascii:]]+?)\D\nDATABASE USER:\[\d+\]\s+\D(?P<database_user>[^\s]+)\D\n(PRIVILEGE :\[\d+\]\s+\D(?P<privilege>[^\s]+)\D\n)?(CLIENT USER:\[\d+\]\s+\D(?P<client_user>[^\s]+|)\D\n)?(CLIENT TERMINAL:\[\d+\]\s+\D(?P<client_terminal>[^\s]+|)\D\n)?(STATUS:\[\d+\]\s+\D(?P<status_code>[^\s]+|)\D\n)?(DBID:\[\d+\]\s\D(?P<dbid>[^\s]+|)\D\n)?(SESSIONID:\[\d+\]\s+\D(?P<sessionid>[^\s]+|)\D\n)?(USERHOST:\[\d+\]\s+\D(?P<userhost>[^\s]+|)\D\n)?(CLIENT ADDRESS:\[\d+\]\s+\D(?P<client_address>[^\s]+|)\D\n)?(ACTION NUMBER:\[\d+\]\s+\D(?P<action_number>[^\s]+|)\D\n)?'
+    timestamp:
+      parse_from: timestamp
+      layout: '%a %h %g %H:%M:%S %Y %j'
+    # {{ if $enable_truncate_audit_action }}
+    output: audit_action_restructurer
+    # {{ else }}
+    output: {{ .output }}
+    # {{ end }}
+
+  - id: audit_regex_parser_session
+    type: regex_parser
+    regex: '^(?P<timestamp>\w+ \w+\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4} [-+]\d{2}:\d{2})\nLENGTH: \D(?P<length>\d*)\D\n(SESSIONID:\[\d+\]\s+\D(?P<sessionid>[^\s]+|)\D[\n\s]+)?(ENTRYID:\[\d+\]\s+\D(?P<entryid>[^\s]+|)\D[\n\s]+)?(STATEMENT:\[\d+\]\s+\D(?P<statement>[^\s]+|)\D[\n\s]+)?(USERID:\[\d+\]\s+\D(?P<userid>[^\s]+|)\D[\n\s]+)?(USERHOST:\[\d+\]\s+\D(?P<userhost>[^\s]+|)\D[\n\s]+)?(TERMINAL:\[\d+\]\s+\D(?P<terminal>[\d\w[:ascii:]]+?|)\D[\n\s]+)?ACTION:\[\d+\]\s+\D(?P<action>[\d\w[:ascii:]]+?|)\D[\n\s]+(RETURNCODE:\[\d+\]\s+\D(?P<returncode>[^\s]+|)\D[\n\s]+)?(COMMENT\$TEXT:\[\d+\]\s+\D(?P<comment_text>[^"]+|)\D[\n\s]+)?(LOGOFF\$PREAD:\[\d+\]\s+\D(?P<logoff_pread>[^"]+|)\D[\n\s]+)?(LOGOFF\$LREAD:\[\d+\]\s+\D(?P<logoff_lread>[^"]+|)\D[\n\s]+)?(LOGOFF\$LWRITE:\[\d+\]\s+\D(?P<logoff_lwrite>[^"]+|)\D[\n\s]+)?(LOGOFF\$DEAD:\[\d+\]\s+\D(?P<logoff_dead>[^"]+|)\D[\n\s]+)?(OBJ\$CREATOR:\[\d+\]\s+\D(?P<obj_creator>[^"]+|)\D[\n\s]+)?(OBJ\$NAME:\[\d+\]\s+\D(?P<obj_name>[^"]+|)\D[\n\s]+)?(OBJ\$PRIVILEGES:\[\d+\]\s+\D(?P<obj_privileges>[^"]+|)\D[\n\s]+)?(AUTH\$GRANTEE:\[\d+\]\s+\D(?P<auth_grantee>[^"]+|)\D[\n\s]+)?(OS\$USERID:\[\d+\]\s+\D(?P<os_userid>[^\s]+|)\D[\n\s]+)?(DBID:\[\d+\]\s+\D(?P<dbid>[^\s]+|)\D[\n\s]+)?(SESSIONCPU:\[\d+\]\s+\D(?P<sessioncpu>\d+|)\D[\n\s]+)?(PRIV\$USED:\[\d+\]\s+\D(?P<priv_user>[^\s]+|)\D[\n\s]+)?'
     timestamp:
       parse_from: timestamp
       layout: '%a %h %g %H:%M:%S %Y %j'