Cloud trails saved queries integration (#1737) (#1751)

* add cloud trails saved queries asset



* add cloud trails saved queries asset



* add skipping index creation



---------


(cherry picked from commit 57ae4e3)

Signed-off-by: YANGDB <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_skipping_index-1.0.0.sql

@@ -0,0 +1,16 @@
+CREATE SKIPPING INDEX ON {table_name} (
+    rec.userIdentity.principalId BLOOM_FILTER,
+    rec.userIdentity.accountId BLOOM_FILTER,
+    rec.userIdentity.userName BLOOM_FILTER,
+    rec.sourceIPAddress BLOOM_FILTER,
+    rec.eventId BLOOM_FILTER,
+    rec.userIdentity.type VALUE_SET,
+    rec.eventName VALUE_SET,
+    rec.eventType VALUE_SET,
+    rec.awsRegion VALUE_SET
+) WITH (
+    auto_refresh = true,
+    refresh_interval = '15 Minutes',
+    checkpoint_location = '{s3_checkpoint_location}',
+    watermark_delay = '1 Minute'
+)

server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/example_queries-1.0.0.ndjson

@@ -0,0 +1,5 @@
+{"attributes":{"createdTimeMs":1713289099101,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"CloudTrail_Error_and_No_MFA_Events","query":"SELECT rec.userIdentity.type, rec.userIdentity.principalId, rec.userIdentity.accountId, rec.userIdentity.userName, rec.eventName, rec.eventType, rec.eventId, rec.awsRegion, rec.sourceIPAddress, rec.errorCode, rec.errorMessage, CAST(rec.eventTime AS TIMESTAMP) AS event_timestamp FROM {table_name} LATERAL VIEW explode(Records) myTable AS rec WHERE rec.errorCode IS NOT NULL AND rec.userIdentity.sessionContext.attributes.mfaAuthenticated = 'false' ORDER BY event_timestamp DESC LIMIT 10;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"CloudTrail Error & No MFA Events","version":1},"id":"1d07d010-fc18-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:52:30.414Z","version":"WzI3NTEsMV0="}
+{"attributes":{"createdTimeMs":1713293044079,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Recent_Security_Risk_Events","query":"SELECT rec.userIdentity.userName, rec.eventName, rec.awsRegion, rec.sourceIPAddress, CAST(rec.eventTime AS TIMESTAMP) AS event_timestamp FROM {table_name}  LATERAL VIEW explode(Records) myTable AS rec WHERE rec.sourceIPAddress NOT LIKE '192.168.%.%' ORDER BY event_timestamp DESC LIMIT 50;\n","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Recent Security Risk Events","version":1},"id":"4c6b8820-fc21-11ee-ab45-d3075d0510e6","references":[],"type":"observability-search","updated_at":"2024-04-16T18:44:47.956Z","version":"WzI4MzAsMV0="}
+{"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Number_recent_Security_issues","query":"SELECT rec.userIdentity.userName, rec.eventName, rec.awsRegion, rec.sourceIPAddress, rec.errorCode, CAST(rec.eventTime AS TIMESTAMP) AS event_timestamp FROM {table_name} LATERAL VIEW explode(Records) myTable AS rec WHERE rec.errorCode IS NOT NULL AND rec.userIdentity.sessionContext.attributes.mfaAuthenticated = 'false' ORDER BY event_timestamp DESC LIMIT 5;","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Number of recent Security issues","version":1},"id":"9e6a9b40-fc1a-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="}
+{"attributes":{"createdTimeMs":1713290175184,"savedQuery":{"data_sources":"[{\"name\":\"mys3\",\"type\":\"s3glue\",\"label\":\"mys3\",\"value\":\"mys3\"}]","description":"","name":"Critical_Resource_Access_Events","query":"SELECT rec.userIdentity.userName, rec.eventName, rec.awsRegion, rec.sourceIPAddress, rec.resources, CAST(rec.eventTime AS TIMESTAMP) AS event_timestamp FROM {table_name} LATERAL VIEW explode(Records) myTable AS rec WHERE (rec.eventName LIKE '%Modify%' OR rec.eventName LIKE '%Delete%' OR rec.eventName LIKE '%Put%' OR rec.eventName LIKE '%Post%') ORDER BY event_timestamp DESC LIMIT 10;\n","query_lang":"SQL","selected_date_range":{"end":"now","start":"now-15m","text":""},"selected_fields":{"text":"","tokens":[]},"selected_timestamp":{"name":"","type":"timestamp"}},"title":"Critical Resource Access Events","version":1},"id":"9e6a9b40-fc1a-11ee-99c9-43e5dbd0692c","references":[],"type":"observability-search","updated_at":"2024-04-16T17:56:15.220Z","version":"WzI3NTIsMV0="}
+{"exportedCount":7,"missingRefCount":0,"missingReferences":[]}

server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json

@@ -73,6 +73,20 @@
       "type": "savedObjectBundle",
       "workflows": ["dashboards-flint"]
     },
+    {
+      "name": "create_skipping_index",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["queries"]
+    },
+    {
+      "name": "example_queries",
+      "version": "1.0.0",
+      "extension": "ndjson",
+      "type": "savedObjectBundle",
+      "workflows": ["queries"]
+    },
     {
       "name": "create_table_cloud-trail",
       "version": "1.0.0",