update cloud trail integration with flint-s3 based DDL assets and das… (

#1701) * update cloud trail integration with flint-s3 based DDL assets and dashboard Signed-off-by: YANGDB <[email protected]> * update time field using CAST to TIMESTAMP Signed-off-by: YANGDB <[email protected]> * update separate flint dashboard support and workflow Signed-off-by: YANGDB <[email protected]> * update separate flint dashboard support and workflow Signed-off-by: YANGDB <[email protected]> * update separate flint dashboard support and workflow Signed-off-by: YANGDB <[email protected]> * update refresh sync rate Signed-off-by: YANGDB <[email protected]> --------- Signed-off-by: YANGDB <[email protected]> (cherry picked from commit bccdc31) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Apr 16, 2024 · a03873a · a03873a
1 parent ebb076c
commit a03873a
Show file tree

Hide file tree

Showing 4 changed files with 181 additions and 2 deletions.
diff --git a/.../integrations/__data__/repository/aws_cloudtrail/assets/aws_cloudtrail-flint-1.0.0.ndjson b/.../integrations/__data__/repository/aws_cloudtrail/assets/aws_cloudtrail-flint-1.0.0.ndjson
diff --git a/...rs/integrations/__data__/repository/aws_cloudtrail/assets/create_mv_cloud-trail-1.0.0.sql b/...rs/integrations/__data__/repository/aws_cloudtrail/assets/create_mv_cloud-trail-1.0.0.sql
@@ -0,0 +1,54 @@
+CREATE MATERIALIZED VIEW {table_name}_mview AS
+SELECT
+  rec.userIdentity.type AS `aws.cloudtrail.userIdentity.type`,
+  rec.userIdentity.principalId AS `aws.cloudtrail.userIdentity.principalId`,
+  rec.userIdentity.arn AS `aws.cloudtrail.userIdentity.arn`,
+  rec.userIdentity.accountId AS `aws.cloudtrail.userIdentity.accountId`,
+  rec.userIdentity.invokedBy AS `aws.cloudtrail.userIdentity.invokedBy`,
+  rec.userIdentity.accessKeyId AS `aws.cloudtrail.userIdentity.accessKeyId`,
+  rec.userIdentity.userName AS `aws.cloudtrail.userIdentity.userName`,
+  rec.userIdentity.sessionContext.attributes.mfaAuthenticated AS `aws.cloudtrail.userIdentity.sessionContext.attributes.mfaAuthenticated`,
+  CAST(rec.userIdentity.sessionContext.attributes.creationDate  AS TIMESTAMP) AS `aws.cloudtrail.userIdentity.sessionContext.attributes.creationDate`,
+  rec.userIdentity.sessionContext.sessionIssuer.type AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.type`,
+  rec.userIdentity.sessionContext.sessionIssuer.principalId AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.principalId`,
+  rec.userIdentity.sessionContext.sessionIssuer.arn AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.arn`,
+  rec.userIdentity.sessionContext.sessionIssuer.accountId AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.accountId`,
+  rec.userIdentity.sessionContext.sessionIssuer.userName AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.userName`,
+  rec.userIdentity.sessionContext.ec2RoleDelivery AS `aws.cloudtrail.userIdentity.sessionContext.ec2RoleDelivery`,
+
+  rec.eventVersion AS `aws.cloudtrail.eventVersion`,
+  CAST(rec.eventTime AS TIMESTAMP)  AS `@timestamp`,
+  rec.eventSource AS `aws.cloudtrail.eventSource`,
+  rec.eventName AS `aws.cloudtrail.eventName`,
+  rec.eventCategory AS `aws.cloudtrail.eventCategory`,
+  rec.eventType AS `aws.cloudtrail.eventType`,
+  rec.eventId AS `aws.cloudtrail.eventId`,
+
+  rec.awsRegion AS `aws.cloudtrail.awsRegion`,
+  rec.sourceIPAddress AS `aws.cloudtrail.sourceIPAddress`,
+  rec.userAgent AS `aws.cloudtrail.userAgent`,
+  rec.errorCode AS `errorCode`,
+  rec.errorMessage AS `errorMessage`,
+  rec.requestParameters AS `aws.cloudtrail.requestParameter`,
+  rec.responseElements AS `aws.cloudtrail.responseElements`,
+  rec.additionalEventData AS `aws.cloudtrail.additionalEventData`,
+  rec.requestId AS `aws.cloudtrail.requestId`,
+  rec.resources AS `aws.cloudtrail.resources`,
+  rec.apiVersion AS `aws.cloudtrail.apiVersion`,
+  rec.readOnly AS `aws.cloudtrail.readOnly`,
+  rec.recipientAccountId AS `aws.cloudtrail.recipientAccountId`,
+  rec.serviceEventDetails AS `aws.cloudtrail.serviceEventDetails`,
+  rec.sharedEventId AS `aws.cloudtrail.sharedEventId`,
+  rec.vpcEndpointId AS `aws.cloudtrail.vpcEndpointId`,
+  rec.tlsDetails.tlsVersion AS `aws.cloudtrail.tlsDetails.tls_version`,
+  rec.tlsDetails.cipherSuite AS `aws.cloudtrail.tlsDetailscipher_suite`,
+  rec.tlsDetails.clientProvidedHostHeader AS `aws.cloudtrail.tlsDetailsclient_provided_host_header`
+FROM
+  {table_name}
+    LATERAL VIEW explode(Records) myTable AS rec
+WITH (
+    auto_refresh = true,
+    checkpoint_location = '{s3_checkpoint_location}',
+    watermark_delay = '1 min',
+    extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
+)
diff --git a/...integrations/__data__/repository/aws_cloudtrail/assets/create_table_cloud-trail-1.0.0.sql b/...integrations/__data__/repository/aws_cloudtrail/assets/create_table_cloud-trail-1.0.0.sql
@@ -0,0 +1,65 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
+  Records ARRAY<STRUCT<
+    eventVersion STRING,
+    userIdentity STRUCT<
+      type:STRING,
+      principalId:STRING,
+      arn:STRING,
+      accountId:STRING,
+      invokedBy:STRING,
+      accessKeyId:STRING,
+      userName:STRING,
+      sessionContext:STRUCT<
+        attributes:STRUCT<
+          mfaAuthenticated:STRING,
+          creationDate:STRING
+        >,
+        sessionIssuer:STRUCT<
+          type:STRING,
+          principalId:STRING,
+          arn:STRING,
+          accountId:STRING,
+          userName:STRING
+        >,
+        ec2RoleDelivery:STRING,
+        webIdFederationData:MAP<STRING,STRING>
+      >
+    >,
+    eventTime STRING,
+    eventSource STRING,
+    eventName STRING,
+    awsRegion STRING,
+    sourceIPAddress STRING,
+    userAgent STRING,
+    errorCode STRING,
+    errorMessage STRING,
+    requestParameters STRING,
+    responseElements STRING,
+    additionalEventData STRING,
+    requestId STRING,
+    eventId STRING,
+    resources ARRAY<STRUCT<
+      arn:STRING,
+      accountId:STRING,
+      type:STRING
+    >>,
+    eventType STRING,
+    apiVersion STRING,
+    readOnly STRING,
+    recipientAccountId STRING,
+    serviceEventDetails STRING,
+    sharedEventId STRING,
+    vpcEndpointId STRING,
+    eventCategory STRING,
+    tlsDetails STRUCT<
+      tlsVersion:STRING,
+      cipherSuite:STRING,
+      clientProvidedHostHeader:STRING
+    >
+  >>
+) USING json
+LOCATION '{s3_bucket_location}'
+OPTIONS (
+  compression='gzip',
+  recursivefilelookup='true'
+);
diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json
@@ -5,9 +5,29 @@
   "description": "Analyze CloudTrail events.",
   "license": "Apache-2.0",
   "type": "logs-aws_cloudtrail",
-  "labels": ["Observability", "Logs", "AWS", "Cloud"],
+  "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
   "author": "OpenSearch",
   "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info",
+  "workflows": [
+    {
+      "name": "queries",
+      "label": "Queries (recommended)",
+      "description": "Tables and pre-written queries for quickly getting insights on your data.",
+      "enabled_by_default": true
+    },
+    {
+      "name": "dashboards",
+      "label": "Dashboards & Visualizations",
+      "description": "Dashboards and indices that enable you to easily visualize important metrics.",
+      "enabled_by_default": false
+    },
+    {
+      "name": "dashboards-flint",
+      "label": "Dashboards & Visualizations adapted to Flint",
+      "description": "Dashboards and visualizations adapted to Flint S3 datasource that enable you to easily visualize data residing on S3",
+      "enabled_by_default": false
+    }
+  ],
   "statics": {
     "logo": {
       "annotation": "CloudTrail Logo",
@@ -43,7 +63,29 @@
       "name": "aws_cloudtrail",
       "version": "1.0.0",
       "extension": "ndjson",
-      "type": "savedObjectBundle"
+      "type": "savedObjectBundle",
+      "workflows": ["dashboards"]
+    },
+    {
+      "name": "aws_cloudtrail-flint",
+      "version": "1.0.0",
+      "extension": "ndjson",
+      "type": "savedObjectBundle",
+      "workflows": ["dashboards-flint"]
+    },
+    {
+      "name": "create_table_cloud-trail",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["dashboards-flint"]
+    },
+    {
+      "name": "create_mv_cloud-trail",
+      "version": "1.0.0",
+      "extension": "sql",
+      "type": "query",
+      "workflows": ["dashboards-flint"]
     }
   ],
   "sampleData": {