add saved queries to vpc flow (opensearch-project#1744)

* add saved queries to vpc flow Signed-off-by: YANGDB <[email protected]> * add skipping index creation Signed-off-by: YANGDB <[email protected]> --------- Signed-off-by: YANGDB <[email protected]>
sumukhswamy · Apr 26, 2024 · 947385f · 947385f
1 parent 3dbe5c8
commit 947385f
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 9 deletions.
diff --git a/...tors/integrations/__data__/repository/aws_vpc_flow/assets/create_skipping_index-1.0.0.sql b/...tors/integrations/__data__/repository/aws_vpc_flow/assets/create_skipping_index-1.0.0.sql
@@ -0,0 +1,16 @@
+CREATE SKIPPING INDEX ON {table_name} (
+    accountid BLOOM_FILTER,
+    region VALUE_SET,
+    severity_id VALUE_SET,
+    src_endpoint.ip BLOOM_FILTER,
+    dst_endpoint.ip BLOOM_FILTER,
+    src_endpoint.svc_name VALUE_SET,
+    dst_endpoint.svc_name VALUE_SET,
+    request_processing_time MIN_MAX,
+    traffic.bytes MIN_MAX
+) WITH (
+    auto_refresh = true,
+    refresh_interval = '15 Minutes',
+    checkpoint_location = '{s3_checkpoint_location}',
+    watermark_delay = '1 Minute'
+)
diff --git a/...daptors/integrations/__data__/repository/aws_vpc_flow/assets/example_queries-1.0.0.ndjson b/...daptors/integrations/__data__/repository/aws_vpc_flow/assets/example_queries-1.0.0.ndjson
@@ -0,0 +1,4 @@
+{"attributes":{"createdTimeMs":1713289099101,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"agg_60_min_connections_view","query":"SELECT date_trunc('hour', from_unixtime(start_time / 1000)) AS interval_start_time, date_trunc('hour', from_unixtime(start_time / 1000)) + INTERVAL 1 HOUR AS interval_end_time, accountid as `aws.vpc.account-id`, region as `aws.vpc.region`, COUNT(*) AS total_connections, SUM(CAST(IFNULL(traffic.bytes, 0) AS LONG)) AS total_bytes, SUM(CAST(IFNULL(traffic.packets, 0) AS LONG)) AS total_packets FROM {table_name} GROUP BY date_trunc('hour', from_unixtime(start_time / 1000)), region, accountid\n","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Hourly count connections summary","version":1},"id":"1d07d010-fc18-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:52:30.414Z","version":"WzI3NTEsMV0="}
+{"attributes":{"createdTimeMs":1713293044079,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"window_agg_60_min_network_ip_bytes","query":"WITH hourly_buckets AS (   SELECT     date_trunc('hour', from_unixtime(start_time / 1000)) AS interval_start_time,     CAST(IFNULL(dst_endpoint.ip, '0.0.0.0') AS STRING)  AS dstaddr,     SUM(CAST(IFNULL(traffic.bytes, 0) AS LONG)) AS total_bytes   FROM  {table_name}   GROUP BY     interval_start_time,     dstaddr ), ranked_addresses AS (   SELECT     CAST(interval_start_time  AS TIMESTAMP),     dstaddr,     total_bytes,     RANK() OVER (PARTITION BY interval_start_time ORDER BY total_bytes DESC) AS bytes_rank   FROM     hourly_buckets ) SELECT   CAST(interval_start_time  AS TIMESTAMP),   dstaddr,   total_bytes FROM   ranked_addresses WHERE     bytes_rank <= 50 ORDER BY   interval_start_time ASC,   bytes_rank ASC","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"window hourly network ip bytes summary","version":1},"id":"4c6b8820-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:44:47.956Z","version":"WzI4MzAsMV0="}
+{"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"TopCommonErrorServicesQuery","query":"SELECT src_endpoint.svc_name AS source_service, dst_endpoint.svc_name AS destination_service, COUNT(*) AS error_count FROM {table_name} WHERE severity_id >= 4 GROUP BY src_endpoint.svc_name, dst_endpoint.svc_name ORDER BY error_count DESC LIMIT 10;\n","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Top 10 pairs of errored source / destination services","version":1},"id":"9e6a9b40-fc1a-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="}
+{"exportedCount":7,"missingRefCount":0,"missingReferences":[]}
diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json
@@ -75,6 +75,27 @@
     }
   ],
   "assets": [
+    {
+      "name": "create_table_parquet_vpc",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["flint-live-dashboards","flint-pre-agg-dashboards"]
+    },
+    {
+      "name": "example_queries",
+      "version": "1.0.0",
+      "extension": "ndjson",
+      "type": "savedObjectBundle",
+      "workflows": ["queries"]
+    },
+    {
+      "name": "create_skipping_index",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["queries"]
+    },
     {
       "name": "aws_vpc_flow",
       "version": "1.0.0",
@@ -96,14 +117,6 @@
       "type": "savedObjectBundle",
       "workflows": ["flint-pre-agg-dashboards"]
     },
-
-    {
-      "name": "create_table_parquet_vpc",
-      "version": "1.0.0",
-      "extension": "sql",
-      "type": "query",
-      "workflows": ["flint-live-dashboards","flint-pre-agg-dashboards"]
-    },
     {
       "name": "vpc_live_all_mv",
       "version": "1.0.0",
@@ -139,7 +152,6 @@
       "type": "query",
       "workflows": ["flint-pre-agg-dashboards"]
     },
-
     {
       "name": "vpc_live_week_refresh",
       "version": "1.0.0",