From 9e81aa0a38e10a3f654ab24ab47cc0d20ec70f4b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 16:07:59 -0700 Subject: [PATCH] [Integrations] Add skipping index and saved queries for WAF integration (#1750) (#1753) * Add skipping index and saved queries for WAF integration * fixes * change the timestamp to min_max * add http source id with bloom filter --------- (cherry picked from commit afc8f4a8a84d120bc1804e6cccec604ec70f188c) Signed-off-by: Ryan Liang Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../aws_waf/assets/create_skipping_index-1.0.0.sql | 13 +++++++++++++ .../aws_waf/assets/example_queries-1.0.0.ndjson | 3 +++ .../__data__/repository/aws_waf/aws_waf-1.0.0.json | 14 ++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 server/adaptors/integrations/__data__/repository/aws_waf/assets/create_skipping_index-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/aws_waf/assets/example_queries-1.0.0.ndjson diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_skipping_index-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_skipping_index-1.0.0.sql new file mode 100644 index 0000000000..dd7b84faed --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_skipping_index-1.0.0.sql @@ -0,0 +1,13 @@ +CREATE SKIPPING INDEX ON {table_name} ( + `timestamp` MIN_MAX, + `webaclId` VALUE_SET, + `httpRequest` VALUE_SET, + `action` VALUE_SET, + `terminatingRuleType` VALUE_SET, + `httpSourceId` BLOOM_FILTER +) WITH ( + auto_refresh = true, + refresh_interval = '15 Minutes', + checkpoint_location = '{s3_checkpoint_location}', + watermark_delay = '1 Minute' +) diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/assets/example_queries-1.0.0.ndjson b/server/adaptors/integrations/__data__/repository/aws_waf/assets/example_queries-1.0.0.ndjson new file mode 100644 index 0000000000..9cab495ca7 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_waf/assets/example_queries-1.0.0.ndjson @@ -0,0 +1,3 @@ +{"attributes":{"createdTimeMs":1713289099101,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Allowed Web Access Logs","query":"SELECT `timestamp` as event_timestamp, `webaclId`, `action`, `httpSourceName`, `httpRequest`.clientIp, `httpRequest`.country, `httpRequest`.uri, `httpRequest`.httpMethod, `httpRequest`.requestId FROM {table_name} WHERE `action` = 'ALLOW' ORDER BY event_timestamp DESC LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Allowed Web Access Logs","version":1},"id":"1d07d010-fc18-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:52:30.414Z","version":"WzI3NTEsMV0="} +{"attributes":{"createdTimeMs":1713293269224,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Regular TerminatingRule WebACLs Limited","query":"SELECT `webaclId` FROM {table_name} WHERE `terminatingRuleType` = 'REGULAR' LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Regular TerminatingRule WebACLs Limited","version":1},"id":"d2a038a0-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:47:49.290Z","version":"WzI4MzIsMV0="} +{"attributes":{"createdTimeMs":1713294061574,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Allowed Actions WebACLs","query":"SELECT `webaclId`, `action` FROM {table_name} WHERE `action` = 'ALLOW' LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Allowed Actions WebACLs","version":1},"id":"aae73c80-fc23-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T19:01:01.640Z","version":"WzI4MzUsMV0="} diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json index e12b9233b9..f6f6eb6f64 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json @@ -73,6 +73,20 @@ "type": "query", "workflows": ["dashboards"] + }, + { + "name": "create_skipping_index", + "version": "1.0.0", + "extension": "sql", + "type": "query", + "workflows": ["queries"] + }, + { + "name": "example_queries", + "version": "1.0.0", + "extension": "ndjson", + "type": "savedObjectBundle", + "workflows": ["queries"] } ], "sampleData": {