From 186e7134d89e231e9711fe743320d357e98a6b50 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 20:18:39 -0700 Subject: [PATCH] [Integrations] Reformatting integration queries (#1726) (#1727) * Reformatting integration queries to address the job type issue for MV creation * Adding reformatting for elb integration --------- (cherry picked from commit 5bff29dd0dea30666a5061a5ac41b018d0c32e02) Signed-off-by: Ryan Liang Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] (cherry picked from commit 1050dabd36e1b8eab310bb5d3c1b650ccbf1e1c1) --- .../__data__/repository/apache/assets/create_mv-1.0.0.sql | 2 +- .../repository/aws_cloudfront/assets/create_mv-1.0.0.sql | 2 +- .../repository/aws_cloudfront/assets/create_table-1.0.0.sql | 2 +- .../aws_cloudtrail/assets/create_table_cloud-trail-1.0.0.sql | 2 +- .../__data__/repository/aws_elb/assets/create_mv-1.0.0.sql | 2 +- .../__data__/repository/aws_elb/assets/create_table-1.0.0.sql | 2 +- .../__data__/repository/aws_s3/assets/create_mv-1.0.0.sql | 2 +- .../__data__/repository/aws_s3/assets/create_table-1.0.0.sql | 2 +- .../repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql | 2 +- .../aws_vpc_flow/assets/vpc_agg_60min_connections_mv-1.0.0.sql | 1 - .../repository/aws_vpc_flow/assets/vpc_live_all_mv-1.0.0.sql | 1 - .../repository/aws_vpc_flow/assets/vpc_live_week_mv-1.0.0.sql | 1 - .../vpc_window-agg_60min_dest_ip_cardinality_mv-1.0.0.sql | 1 - .../vpc_window-agg_60min_dest_ip_total-bytes_mv-1.0.0.sql | 1 - .../__data__/repository/aws_waf/assets/create_mv-1.0.0.sql | 2 +- .../__data__/repository/aws_waf/assets/create_table-1.0.0.sql | 2 +- .../__data__/repository/haproxy/assets/create_mv-1.0.0.sql | 2 +- .../__data__/repository/haproxy/assets/create_table-1.0.0.sql | 2 +- .../__data__/repository/nginx/assets/create_mv-1.0.0.sql | 2 +- 19 files changed, 14 insertions(+), 19 deletions(-) diff --git a/server/adaptors/integrations/__data__/repository/apache/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/apache/assets/create_mv-1.0.0.sql index c738b14650..7201fbc252 100644 --- a/server/adaptors/integrations/__data__/repository/apache/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/apache/assets/create_mv-1.0.0.sql @@ -14,4 +14,4 @@ WITH ( checkpoint_location = '{s3_checkpoint_location}', watermark_delay = '1 Minute', extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' -); +) diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_mv-1.0.0.sql index 70c0ad2b96..ddaccf08ca 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_mv-1.0.0.sql @@ -40,4 +40,4 @@ WITH ( checkpoint_location = '{s3_checkpoint_location}', watermark_delay = '1 Minute', extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' -); \ No newline at end of file +) diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_table-1.0.0.sql index 06f6d25250..d6345312c3 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_table-1.0.0.sql @@ -36,4 +36,4 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( USING csv LOCATION '{s3_bucket_location}' OPTIONS ( sep='\t' ) -TBLPROPERTIES ( 'skip.header.line.count'='2' ); \ No newline at end of file +TBLPROPERTIES ( 'skip.header.line.count'='2' ) diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_table_cloud-trail-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_table_cloud-trail-1.0.0.sql index fb17187a81..0ca7fd0aec 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_table_cloud-trail-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_table_cloud-trail-1.0.0.sql @@ -62,4 +62,4 @@ LOCATION '{s3_bucket_location}' OPTIONS ( compression='gzip', recursivefilelookup='true' -); +) diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql index c1e8887313..e80adb4364 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql @@ -57,4 +57,4 @@ WITH ( checkpoint_location = '{s3_checkpoint_location}', watermark_delay = '1 Minute', extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' -); +) diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql index a0129b0e4a..b862127825 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql @@ -35,4 +35,4 @@ OPTIONS ( sep=' ', compression='gzip', recursiveFileLookup='true' -); +) diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql index 2823c21058..38c2f1c366 100644 --- a/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql @@ -33,4 +33,4 @@ WITH ( checkpoint_location = '{s3_checkpoint_location}', watermark_delay = '1 Minute', extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' -); \ No newline at end of file +) diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql index 9261e65e4e..d2fcd7ba23 100644 --- a/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql @@ -29,4 +29,4 @@ USING csv OPTIONS ( sep=' ' ) -LOCATION '{s3_bucket_location}'; \ No newline at end of file +LOCATION '{s3_bucket_location}' diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql index bb73fa9340..aa4099c545 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql @@ -17,4 +17,4 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( LOCATION '{s3_bucket_location}' OPTIONS ( sep=' ' -); +) diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_agg_60min_connections_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_agg_60min_connections_mv-1.0.0.sql index e7a30dc159..fd1a598122 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_agg_60min_connections_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_agg_60min_connections_mv-1.0.0.sql @@ -29,4 +29,3 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS {table_name}__agg_60_min_connections_mvie WITH ( auto_refresh = false ) - diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_live_all_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_live_all_mv-1.0.0.sql index 155c4f41bf..d9310f4f1d 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_live_all_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_live_all_mv-1.0.0.sql @@ -59,4 +59,3 @@ WITH ( watermark_delay = '1 Minute', extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' ) - diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_live_week_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_live_week_mv-1.0.0.sql index 170a9b1276..4f4c629ed8 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_live_week_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_live_week_mv-1.0.0.sql @@ -58,4 +58,3 @@ CREATE MATERIALIZED VIEW {table_name}__week_live_mview AS WITH ( auto_refresh = false ) - diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_window-agg_60min_dest_ip_cardinality_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_window-agg_60min_dest_ip_cardinality_mv-1.0.0.sql index 24d3519dc9..1450995a09 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_window-agg_60min_dest_ip_cardinality_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_window-agg_60min_dest_ip_cardinality_mv-1.0.0.sql @@ -33,4 +33,3 @@ ORDER BY WITH ( auto_refresh = false ) - diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_window-agg_60min_dest_ip_total-bytes_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_window-agg_60min_dest_ip_total-bytes_mv-1.0.0.sql index f68d09c371..26831be80c 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_window-agg_60min_dest_ip_total-bytes_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_window-agg_60min_dest_ip_total-bytes_mv-1.0.0.sql @@ -33,4 +33,3 @@ WITH hourly_buckets AS ( WITH ( auto_refresh = false ) - diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_mv-1.0.0.sql index 48e6ec163c..74c9d6d253 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_mv-1.0.0.sql @@ -26,4 +26,4 @@ WITH ( checkpoint_location = '{s3_checkpoint_location}', watermark_delay = '1 Minute', extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' -); \ No newline at end of file +) diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql index 504bb0241a..f0178c8142 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql @@ -125,4 +125,4 @@ CREATE EXTERNAL TABLE {table_name} ( `ja3Fingerprint` string ) USING json -LOCATION '{s3_bucket_location}' \ No newline at end of file +LOCATION '{s3_bucket_location}' diff --git a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql index ee1957ee9f..72a664aa83 100644 --- a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql @@ -148,4 +148,4 @@ WITH ( checkpoint_location = '{s3_checkpoint_location}', watermark_delay = '1 Minute', extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' -); +) diff --git a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql index 5e749e0998..37ba164bf8 100644 --- a/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql @@ -4,4 +4,4 @@ CREATE EXTERNAL TABLE {table_name} ( LOCATION '{s3_bucket_location}' OPTIONS ( sep='\0x1E' -); +) diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql index d1646df20d..f273c5e071 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql @@ -14,4 +14,4 @@ WITH ( checkpoint_location = '{s3_checkpoint_location}', watermark_delay = '1 Minute', extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' -); +)