Add skipping indices for all integrations that have sample queries (o…

…pensearch-project#1747) * Update avg threshold for cloudfront query Signed-off-by: Simeon Widdis <[email protected]> * Add first draft skipping index for cloudfront integration Signed-off-by: Simeon Widdis <[email protected]> * Add skipping index for nginx Signed-off-by: Simeon Widdis <[email protected]> * Add skipping index for s3 Signed-off-by: Simeon Widdis <[email protected]> * Add skipping index for elb Signed-off-by: Simeon Widdis <[email protected]> * Update skipping indices to use bloom filters for client IDs Signed-off-by: Simeon Widdis <[email protected]> * Add skipping index for apache Signed-off-by: Simeon Widdis <[email protected]> --------- Signed-off-by: Simeon Widdis <[email protected]> (cherry picked from commit 685aeaf)
amsiglan · Apr 30, 2024 · b3ec999 · b3ec999
1 parent 777c38f
commit b3ec999
Show file tree

Hide file tree

Showing 11 changed files with 95 additions and 9 deletions.
diff --git a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json
@@ -69,6 +69,13 @@
       "extension": "sql",
       "type": "query"
     },
+    {
+      "name": "create_skipping_index",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["queries"]
+    },
     {
       "name": "create_mv",
       "version": "1.0.0",

diff --git a/...r/adaptors/integrations/__data__/repository/apache/assets/create_skipping_index-1.0.0.sql b/...r/adaptors/integrations/__data__/repository/apache/assets/create_skipping_index-1.0.0.sql
@@ -0,0 +1,10 @@
+CREATE SKIPPING INDEX ON {table_name} (
+    remote_addr BLOOM_FILTER,
+    `status` VALUE_SET,
+    body_bytes_sent MIN_MAX
+) WITH (
+    auto_refresh = true,
+    refresh_interval = '15 Minutes',
+    checkpoint_location = '{s3_checkpoint_location}',
+    watermark_delay = '1 Minute'
+)
diff --git a/...rs/integrations/__data__/repository/aws_cloudfront/assets/create_skipping_index-1.0.0.sql b/...rs/integrations/__data__/repository/aws_cloudfront/assets/create_skipping_index-1.0.0.sql
@@ -0,0 +1,10 @@
+CREATE SKIPPING INDEX ON {table_name} (
+    c_ip BLOOM_FILTER,
+    sc_status VALUE_SET,
+    time_to_first_byte MIN_MAX
+) WITH (
+    auto_refresh = true,
+    refresh_interval = '15 Minutes',
+    checkpoint_location = '{s3_checkpoint_location}',
+    watermark_delay = '1 Minute'
+)
diff --git a/...ptors/integrations/__data__/repository/aws_cloudfront/assets/example_queries-1.0.0.ndjson b/...ptors/integrations/__data__/repository/aws_cloudfront/assets/example_queries-1.0.0.ndjson
@@ -3,6 +3,6 @@
 {"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Number of Requests","query":"SELECT COUNT(*) AS request_count FROM {table_name};","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Number of Requests","version":1},"id":"9e6a9b40-fc1a-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="}
 {"attributes":{"createdTimeMs":1713293161193,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Total Bytes Served","query":"SELECT SUM(sc_bytes) AS total_bytes_served FROM {table_name};","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Total Bytes Served","version":1},"id":"92398eb0-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:46:01.242Z","version":"WzI4MzEsMV0="}
 {"attributes":{"createdTimeMs":1713293269224,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Average Time Taken","query":"SELECT AVG(time_taken) AS average_time_taken FROM {table_name};","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Average Time Taken","version":1},"id":"d2a038a0-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:47:49.290Z","version":"WzI4MzIsMV0="}
-{"attributes":{"createdTimeMs":1713293425335,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Slow Requests from Average Time threshold","query":"WITH avg_time AS (SELECT AVG(time_to_first_byte) AS avg_time FROM {table_name}) SELECT * FROM {table_name} CROSS JOIN avg_time WHERE time_to_first_byte > 1 * avg_time LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Slow Requests from Average Time threshold","version":1},"id":"2fac4250-fc22-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:59:34.785Z","version":"WzI4MzQsMV0="}
+{"attributes":{"createdTimeMs":1713293425335,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Slow Requests from Average Time threshold","query":"WITH avg_time AS (SELECT AVG(time_to_first_byte) AS avg_time FROM {table_name}) SELECT * FROM {table_name} CROSS JOIN avg_time WHERE time_to_first_byte > 2 * avg_time LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Slow Requests from Average Time threshold","version":1},"id":"2fac4250-fc22-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:59:34.785Z","version":"WzI4MzQsMV0="}
 {"attributes":{"createdTimeMs":1713294061574,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Requests by User Agent","query":"SELECT * FROM {table_name} WHERE cs_user_agent LIKE '%Chrome%' LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Requests by User Agent","version":1},"id":"aae73c80-fc23-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T19:01:01.640Z","version":"WzI4MzUsMV0="}
 {"exportedCount":7,"missingRefCount":0,"missingReferences":[]}
diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json
@@ -54,17 +54,24 @@
   ],
   "assets": [
     {
-      "name": "aws_cloudfront",
+      "name": "create_table",
       "version": "1.0.0",
-      "extension": "ndjson",
-      "type": "savedObjectBundle",
-      "workflows": ["dashboards"]
+      "extension": "sql",
+      "type": "query"
     },
     {
-      "name": "create_table",
+      "name": "create_skipping_index",
       "version": "1.0.0",
       "extension": "sql",
-      "type": "query"
+      "type": "query",
+      "workflows": ["queries"]
+    },
+    {
+      "name": "example_queries",
+      "version": "1.0.0",
+      "extension": "ndjson",
+      "type": "savedObjectBundle",
+      "workflows": ["queries"]
     },
     {
       "name": "create_mv",
@@ -74,11 +81,11 @@
       "workflows": ["dashboards"]
     },
     {
-      "name": "example_queries",
+      "name": "aws_cloudfront",
       "version": "1.0.0",
       "extension": "ndjson",
       "type": "savedObjectBundle",
-      "workflows": ["queries"]
+      "workflows": ["dashboards"]
     }
   ],
   "sampleData": {

diff --git a/.../adaptors/integrations/__data__/repository/aws_elb/assets/create_skipping_index-1.0.0.sql b/.../adaptors/integrations/__data__/repository/aws_elb/assets/create_skipping_index-1.0.0.sql
@@ -0,0 +1,11 @@
+CREATE SKIPPING INDEX ON {table_name} (
+    client_ip BLOOM_FILTER,
+    elb_status_code VALUE_SET,
+    request_processing_time MIN_MAX,
+    sent_bytes MIN_MAX
+) WITH (
+    auto_refresh = true,
+    refresh_interval = '15 Minutes',
+    checkpoint_location = '{s3_checkpoint_location}',
+    watermark_delay = '1 Minute'
+)
diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json
@@ -74,6 +74,13 @@
       "extension": "sql",
       "type": "query"
     },
+    {
+      "name": "create_skipping_index",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["queries"]
+    },
     {
       "name": "create_mv",
       "version": "1.0.0",

diff --git a/...r/adaptors/integrations/__data__/repository/aws_s3/assets/create_skipping_index-1.0.0.sql b/...r/adaptors/integrations/__data__/repository/aws_s3/assets/create_skipping_index-1.0.0.sql
@@ -0,0 +1,10 @@
+CREATE SKIPPING INDEX ON {table_name} (
+    requester BLOOM_FILTER,
+    http_status VALUE_SET,
+    request_time MIN_MAX
+) WITH (
+    auto_refresh = true,
+    refresh_interval = '15 Minutes',
+    checkpoint_location = '{s3_checkpoint_location}',
+    watermark_delay = '1 Minute'
+)
diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json
@@ -69,6 +69,13 @@
       "extension": "sql",
       "type": "query"
     },
+    {
+      "name": "create_skipping_index",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["queries"]
+    },
     {
       "name": "create_mv",
       "version": "1.0.0",

diff --git a/...er/adaptors/integrations/__data__/repository/nginx/assets/create_skipping_index-1.0.0.sql b/...er/adaptors/integrations/__data__/repository/nginx/assets/create_skipping_index-1.0.0.sql
@@ -0,0 +1,10 @@
+CREATE SKIPPING INDEX ON {table_name} (
+    remote_addr BLOOM_FILTER,
+    `status` VALUE_SET,
+    body_bytes_sent MIN_MAX
+) WITH (
+    auto_refresh = true,
+    refresh_interval = '15 Minutes',
+    checkpoint_location = '{s3_checkpoint_location}',
+    watermark_delay = '1 Minute'
+)
diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json
@@ -66,6 +66,13 @@
       "extension": "sql",
       "type": "query"
     },
+    {
+      "name": "create_skipping_index",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["queries"]
+    },
     {
       "name": "create_mv",
       "version": "1.0.0",