Add S3 integration for Nginx and VPC (opensearch-project#1214) (opens…

…earch-project#1216)

* Add nginx integration



* Add VPC S3 integration



---------


(cherry picked from commit e0155e8)

Signed-off-by: Simeon Widdis <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql

@@ -0,0 +1,24 @@
+CREATE MATERIALIZED VIEW {table_name}_mview AS
+    SELECT
+        CAST(FROM_UNIXTIME(start) AS TIMESTAMP) as `@timestamp`,
+        version as `aws.vpc.version`,
+        account_id as `aws.vpc.account-id`,
+        interface_id as `aws.vpc.interface-id`,
+        srcaddr as `aws.vpc.srcaddr`,
+        dstaddr as `aws.vpc.dstaddr`,
+        CAST(srcport AS LONG) as `aws.vpc.srcport`,
+        CAST(dstport AS LONG) as `aws.vpc.dstport`,
+        protocol as `aws.vpc.protocol`,
+        CAST(packets AS LONG) as `aws.vpc.packets`,
+        CAST(bytes AS LONG) as `aws.vpc.bytes`,
+        CAST(FROM_UNIXTIME(start) AS TIMESTAMP) as `aws.vpc.start`,
+        CAST(FROM_UNIXTIME(end) AS TIMESTAMP) as `aws.vpc.end`,
+        action as `aws.vpc.action`,
+        log_status as `aws.vpc.log-status`,
+        CASE
+          WHEN regexp(dstaddr, '(10\\..*)|(192\\.168\\..*)|(172\\.1[6-9]\\..*)|(172\\.2[0-9]\\..*)|(172\\.3[0-1]\\.*)')
+            THEN 'ingress'
+          ELSE 'egress'
+        END AS `aws.vpc.flow-direction`
+FROM
+    {table_name};

server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql

@@ -0,0 +1,20 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
+    version INT,
+    account_id STRING,
+    interface_id STRING,
+    srcaddr STRING,
+    dstaddr STRING,
+    srcport STRING,
+    dstport STRING,
+    protocol STRING,
+    packets STRING,
+    bytes STRING,
+    start BIGINT,
+    end BIGINT,
+    action STRING,
+    log_status STRING
+) USING csv
+LOCATION '{s3_bucket_location}'
+OPTIONS (
+  sep=' '
+);

server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql

@@ -0,0 +1 @@
+REFRESH MATERIALIZED VIEW {table_name}_mview

server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json

@@ -5,7 +5,7 @@
   "description": "AWS VPC Flow log collector",
   "license": "Apache-2.0",
   "type": "logs_vpc",
-  "labels": ["Observability", "Logs", "AWS", "Cloud"],
+  "labels": ["Observability", "Logs", "AWS", "Cloud", "Flint S3"],
   "author": "Haidong Wang",
   "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info",
   "statics": {
@@ -46,7 +46,24 @@
     "savedObjects": {
       "name": "aws_vpc_flow",
       "version": "1.0.0"
-    }
+    },
+    "queries": [
+      {
+        "name": "create_table_vpc",
+        "version": "1.0.0",
+        "language": "sql"
+      },
+      {
+        "name": "create_mv_vpc",
+        "version": "1.0.0",
+        "language": "sql"
+      },
+      {
+        "name": "refresh_mv_vpc",
+        "version": "1.0.0",
+        "language": "sql"
+      }
+    ]
   },
   "sampleData": {
     "path": "sample.json"

server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql

@@ -0,0 +1,10 @@
+CREATE MATERIALIZED VIEW {table_name}_mview AS
+SELECT
+    to_timestamp(trim(BOTH '[]' FROM concat(time_local_1, ' ', time_local_2)), 'dd/MMM/yyyy:HH:mm:ss Z') AS `@timestamp`,
+    split_part (request, ' ', 1) as `http.request.method`,
+    split_part (request, ' ', 2) as `http.url`,
+    split_part (request, ' ', 3) as `http.flavor`,
+    status AS `http.response.status_code`,
+    body_bytes_sent AS `http.response.bytes`,
+    'nginx.access' AS `event.domain`
+FROM {table_name}

server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql

@@ -0,0 +1,18 @@
+CREATE EXTERNAL TABLE {table_name} (
+  remote_addr STRING,
+  empty_col STRING,
+  remote_user STRING,
+  time_local_1 STRING,
+  time_local_2 STRING,
+  request STRING,
+  status INT,
+  body_bytes_sent INT,
+  http_referer STRING,
+  http_user_agent STRING,
+  gzip_ratio STRING
+) USING csv
+LOCATION '{s3_bucket_location}'
+OPTIONS (
+  sep=' ',
+  nullValue='-'
+)

server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql

@@ -0,0 +1 @@
+REFRESH MATERIALIZED VIEW {table_name}_mview

server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json

@@ -5,7 +5,7 @@
   "description": "Nginx HTTP server collector",
   "license": "Apache-2.0",
   "type": "logs",
-  "labels": ["Observability", "Logs"],
+  "labels": ["Observability", "Logs", "Flint S3"],
   "author": "OpenSearch",
   "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
   "statics": {
@@ -38,7 +38,24 @@
     "savedObjects": {
       "name": "nginx",
       "version": "1.0.0"
-    }
+    },
+    "queries": [
+      {
+        "name": "create_table",
+        "version": "1.0.0",
+        "language": "sql"
+      },
+      {
+        "name": "create_mv",
+        "version": "1.0.0",
+        "language": "sql"
+      },
+      {
+        "name": "refresh_mv",
+        "version": "1.0.0",
+        "language": "sql"
+      }
+    ]
   },
   "sampleData": {
     "path": "sample.json"