diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_skipping_index-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_skipping_index-1.0.0.sql new file mode 100644 index 000000000..977af0b95 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_skipping_index-1.0.0.sql @@ -0,0 +1,16 @@ +CREATE SKIPPING INDEX ON {table_name} ( + accountid BLOOM_FILTER, + region VALUE_SET, + severity_id VALUE_SET, + src_endpoint.ip BLOOM_FILTER, + dst_endpoint.ip BLOOM_FILTER, + src_endpoint.svc_name VALUE_SET, + dst_endpoint.svc_name VALUE_SET, + request_processing_time MIN_MAX, + traffic.bytes MIN_MAX +) WITH ( + auto_refresh = true, + refresh_interval = '15 Minutes', + checkpoint_location = '{s3_checkpoint_location}', + watermark_delay = '1 Minute' +) diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/example_queries-1.0.0.ndjson b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/example_queries-1.0.0.ndjson new file mode 100644 index 000000000..7c27b42bb --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/example_queries-1.0.0.ndjson @@ -0,0 +1,4 @@ +{"attributes":{"createdTimeMs":1713289099101,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"agg_60_min_connections_view","query":"SELECT date_trunc('hour', from_unixtime(start_time / 1000)) AS interval_start_time, date_trunc('hour', from_unixtime(start_time / 1000)) + INTERVAL 1 HOUR AS interval_end_time, accountid as `aws.vpc.account-id`, region as `aws.vpc.region`, COUNT(*) AS total_connections, SUM(CAST(IFNULL(traffic.bytes, 0) AS LONG)) AS total_bytes, SUM(CAST(IFNULL(traffic.packets, 0) AS LONG)) AS total_packets FROM {table_name} GROUP BY date_trunc('hour', from_unixtime(start_time / 1000)), region, accountid\n","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Hourly count connections summary","version":1},"id":"1d07d010-fc18-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:52:30.414Z","version":"WzI3NTEsMV0="} +{"attributes":{"createdTimeMs":1713293044079,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"window_agg_60_min_network_ip_bytes","query":"WITH hourly_buckets AS ( SELECT date_trunc('hour', from_unixtime(start_time / 1000)) AS interval_start_time, CAST(IFNULL(dst_endpoint.ip, '0.0.0.0') AS STRING) AS dstaddr, SUM(CAST(IFNULL(traffic.bytes, 0) AS LONG)) AS total_bytes FROM {table_name} GROUP BY interval_start_time, dstaddr ), ranked_addresses AS ( SELECT CAST(interval_start_time AS TIMESTAMP), dstaddr, total_bytes, RANK() OVER (PARTITION BY interval_start_time ORDER BY total_bytes DESC) AS bytes_rank FROM hourly_buckets ) SELECT CAST(interval_start_time AS TIMESTAMP), dstaddr, total_bytes FROM ranked_addresses WHERE bytes_rank <= 50 ORDER BY interval_start_time ASC, bytes_rank ASC","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"window hourly network ip bytes summary","version":1},"id":"4c6b8820-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:44:47.956Z","version":"WzI4MzAsMV0="} +{"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"TopCommonErrorServicesQuery","query":"SELECT src_endpoint.svc_name AS source_service, dst_endpoint.svc_name AS destination_service, COUNT(*) AS error_count FROM {table_name} WHERE severity_id >= 4 GROUP BY src_endpoint.svc_name, dst_endpoint.svc_name ORDER BY error_count DESC LIMIT 10;\n","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Top 10 pairs of errored source / destination services","version":1},"id":"9e6a9b40-fc1a-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="} +{"exportedCount":7,"missingRefCount":0,"missingReferences":[]} diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json index 5862571ce..53eea4118 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json @@ -75,6 +75,27 @@ } ], "assets": [ + { + "name": "create_table_parquet_vpc", + "version": "1.0.0", + "extension": "sql", + "type": "query", + "workflows": ["flint-live-dashboards","flint-pre-agg-dashboards"] + }, + { + "name": "example_queries", + "version": "1.0.0", + "extension": "ndjson", + "type": "savedObjectBundle", + "workflows": ["queries"] + }, + { + "name": "create_skipping_index", + "version": "1.0.0", + "extension": "sql", + "type": "query", + "workflows": ["queries"] + }, { "name": "aws_vpc_flow", "version": "1.0.0", @@ -96,14 +117,6 @@ "type": "savedObjectBundle", "workflows": ["flint-pre-agg-dashboards"] }, - - { - "name": "create_table_parquet_vpc", - "version": "1.0.0", - "extension": "sql", - "type": "query", - "workflows": ["flint-live-dashboards","flint-pre-agg-dashboards"] - }, { "name": "vpc_live_all_mv", "version": "1.0.0", @@ -139,7 +152,6 @@ "type": "query", "workflows": ["flint-pre-agg-dashboards"] }, - { "name": "vpc_live_week_refresh", "version": "1.0.0",