From a74b7c48ce6cd91b5349c9231a6e2dc4adc42005 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 15 Apr 2024 10:56:07 -0700 Subject: [PATCH 1/4] Add draft queries for haproxy Signed-off-by: Simeon Widdis --- .../haproxy/assets/create_mv-1.0.0.sql | 38 +++++++++++++++++++ .../haproxy/assets/create_table-1.0.0.sql | 7 ++++ 2 files changed, 45 insertions(+) create mode 100644 server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql diff --git a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql new file mode 100644 index 0000000000..f3960e33eb --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql @@ -0,0 +1,38 @@ +CREATE MATERIALIZED VIEW {table_name}_mview AS +SELECT + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 1) AS client_ip, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 2) AS client_port, + to_timestamp(regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 3), "dd/MMM/yyyy:HH:mm:ss.SSS") AS `@timestamp`, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 4) AS frontend_name, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 5) AS backend_name, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 6) AS server_name, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 7) AS time_total_ms, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 8) AS time_queue_ms, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 9) AS time_connection_ms, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 10) AS time_server_ms, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 11) AS time_active_ms, + cast(regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 12) as int) AS `http.response.status_code`, + cast(regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 13) as int) AS `http.response.bytes`, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 14) AS captured_request_cookie, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 15) AS captured_response_cookie, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 16) AS termination_state, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 17) AS active_connections, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 18) AS frontend_connections, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 19) AS backend_connections, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 20) AS server_connections, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 21) AS retries, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 22) AS server_queue, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 23) AS backend_queue, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 24) AS captured_request_headers, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 25) AS captured_response_headers, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 26) AS request_method, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 27) AS `http.url`, + regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 28) AS request_protocol, + 'haproxy_access' AS `event.domain` +FROM {table_name} +WITH ( + auto_refresh = true, + checkpoint_location = '{s3_checkpoint_location}', + watermark_delay = '1 Minute', + extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' +); diff --git a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql new file mode 100644 index 0000000000..5e749e0998 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql @@ -0,0 +1,7 @@ +CREATE EXTERNAL TABLE {table_name} ( + record STRING +) USING csv +LOCATION '{s3_bucket_location}' +OPTIONS ( + sep='\0x1E' +); From 951fc65b84bedfe9a1bd3fed7a9036b0fc92142b Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 15 Apr 2024 11:26:30 -0700 Subject: [PATCH 2/4] OTEL-ify more fields Signed-off-by: Simeon Widdis --- .../haproxy/assets/create_mv-1.0.0.sql | 168 +++++++++++++++--- 1 file changed, 140 insertions(+), 28 deletions(-) diff --git a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql index f3960e33eb..4e8c4d1029 100644 --- a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql @@ -1,33 +1,145 @@ CREATE MATERIALIZED VIEW {table_name}_mview AS SELECT - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 1) AS client_ip, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 2) AS client_port, - to_timestamp(regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 3), "dd/MMM/yyyy:HH:mm:ss.SSS") AS `@timestamp`, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 4) AS frontend_name, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 5) AS backend_name, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 6) AS server_name, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 7) AS time_total_ms, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 8) AS time_queue_ms, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 9) AS time_connection_ms, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 10) AS time_server_ms, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 11) AS time_active_ms, - cast(regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 12) as int) AS `http.response.status_code`, - cast(regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 13) as int) AS `http.response.bytes`, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 14) AS captured_request_cookie, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 15) AS captured_response_cookie, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 16) AS termination_state, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 17) AS active_connections, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 18) AS frontend_connections, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 19) AS backend_connections, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 20) AS server_connections, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 21) AS retries, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 22) AS server_queue, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 23) AS backend_queue, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 24) AS captured_request_headers, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 25) AS captured_response_headers, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 26) AS request_method, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 27) AS `http.url`, - regexp_extract(record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 28) AS request_protocol, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 1 + ) AS `connection.source.ip`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 2 + ) AS int) AS `connection.source.port`, + to_timestamp(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 3 + ), "dd/MMM/yyyy:HH:mm:ss.SSS") AS `@timestamp`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 4 + ) AS frontend_name, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 5 + ) AS backend_name, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 6 + ) AS server_name, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 7 + ) AS int) AS time_total_ms, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 8 + ) AS int) AS time_queue_ms, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 9 + ) AS int) AS time_connection_ms, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 10 + ) AS int) AS time_server_ms, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 11 + ) AS int) AS time_active_ms, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 12 + ) AS int) AS `http.response.status_code`, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 13 + ) AS int) AS `http.response.bytes`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 14 + ) AS `http.request.captured_cookie`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 15 + ) AS `http.response.captured_cookie`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 16 + ) AS termination_state, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 17 + ) AS int) AS active_connections, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 18 + ) AS int) AS frontend_connections, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 19 + ) AS int) AS backend_connections, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 20 + ) AS server_connections, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 21 + ) AS int) AS retries, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 22 + ) AS int) AS server_queue, + cast(regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 23 + ) AS int) AS backend_queue, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 24 + ) AS `http.request.captured_headers`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 25 + ) AS `http.response.captured_headers`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 26 + ) AS `http.request.method`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 27 + ) AS `http.url`, + regexp_extract( + record, + "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", + 28 + ) AS `http.flavor`, 'haproxy_access' AS `event.domain` FROM {table_name} WITH ( From a6a0945e8a6619ac5e0cee8a9c5cb857caeb6311 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 15 Apr 2024 11:42:25 -0700 Subject: [PATCH 3/4] Fix event domain and add config updates Signed-off-by: Simeon Widdis --- .../haproxy/assets/create_mv-1.0.0.sql | 2 +- .../repository/haproxy/haproxy-1.0.0.json | 30 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql index 4e8c4d1029..a05d9c9d99 100644 --- a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql @@ -140,7 +140,7 @@ SELECT "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 28 ) AS `http.flavor`, - 'haproxy_access' AS `event.domain` + 'haproxy.access' AS `event.domain` FROM {table_name} WITH ( auto_refresh = true, diff --git a/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json b/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json index c88e42c311..131a3f6961 100644 --- a/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/haproxy/haproxy-1.0.0.json @@ -8,6 +8,20 @@ "labels": ["Observability", "Logs"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/haproxy/info", + "workflows": [ + { + "name": "queries", + "label": "Queries (recommended)", + "description": "Tables and pre-written queries for quickly getting insights on your data.", + "enabled_by_default": true + }, + { + "name": "dashboards", + "label": "Dashboards & Visualizations", + "description": "Dashboards and indices that enable you to easily visualize important metrics.", + "enabled_by_default": false + } + ], "statics": { "logo": { "annotation": "HAProxy Logo", @@ -43,7 +57,21 @@ "name": "haproxy", "version": "1.0.0", "extension": "ndjson", - "type": "savedObjectBundle" + "type": "savedObjectBundle", + "workflows": ["dashboards"] + }, + { + "name": "create_table", + "version": "1.0.0", + "extension": "sql", + "type": "query" + }, + { + "name": "create_mv", + "version": "1.0.0", + "extension": "sql", + "type": "query", + "workflows": ["dashboards"] } ], "sampleData": { From 110685db438aa4747582c7493fc5664d5a6909a0 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 15 Apr 2024 12:13:28 -0700 Subject: [PATCH 4/4] Rename remaining fields Signed-off-by: Simeon Widdis --- .../haproxy/assets/create_mv-1.0.0.sql | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql index a05d9c9d99..5e2ce8047b 100644 --- a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql @@ -19,42 +19,42 @@ SELECT record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 4 - ) AS frontend_name, + ) AS `haproxy.service_name`, regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 5 - ) AS backend_name, + ) AS `haproxy.backend_name`, regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 6 - ) AS server_name, + ) AS `haproxy.proxy_name`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 7 - ) AS int) AS time_total_ms, + ) AS int) AS `haproxy.time_total_ms`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 8 - ) AS int) AS time_queue_ms, + ) AS int) AS `haproxy.time_queue_ms`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 9 - ) AS int) AS time_connection_ms, + ) AS int) AS `haproxy.time_connection_ms`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 10 - ) AS int) AS time_server_ms, + ) AS int) AS `haproxy.time_server_ms`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 11 - ) AS int) AS time_active_ms, + ) AS int) AS `haproxy.time_active_ms`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", @@ -84,37 +84,37 @@ SELECT record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 17 - ) AS int) AS active_connections, + ) AS int) AS `haproxy.active_connections`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 18 - ) AS int) AS frontend_connections, + ) AS int) AS `haproxy.frontend_connections`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 19 - ) AS int) AS backend_connections, - regexp_extract( + ) AS int) AS `haproxy.backend_connections`, + cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 20 - ) AS server_connections, + ) AS int) AS `haproxy.server_connections`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 21 - ) AS int) AS retries, + ) AS int) AS `haproxy.retries`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 22 - ) AS int) AS server_queue, + ) AS int) AS `haproxy.server_queue`, cast(regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$", 23 - ) AS int) AS backend_queue, + ) AS int) AS `haproxy.backend_queue`, regexp_extract( record, "^([\\d\\.]+):(\\d+) \\[(.+)\\] ([\\w\\-]+) ([\\w\\-]+)\\\/([\\w\\-]+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+) (\\d+) (.+) (.+) (.+) (\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+)\\\/(\\d+) (\\d+)\\\/(\\d+) \\{(.*)\\}(?: \\{(.*)\\})? \"(\\w+) (.+) (.+)\"+$",