Skip to content

Commit

Permalink
[Integrations] Reformatting integration queries (opensearch-project#1726
Browse files Browse the repository at this point in the history 
) (opensearch-project#1727)

* Reformatting integration queries to address the job type issue for MV creation

* Adding reformatting for elb integration

---------

(cherry picked from commit 5bff29d)

Signed-off-by: Ryan Liang <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
(cherry picked from commit 1050dab)
  • Loading branch information
@opensearch-trigger-bot @sejli
opensearch-trigger-bot[bot] authored and sejli committed Apr 19, 2024
1 parent 6e6dca3 commit 186e713
Show file tree
Hide file tree
Showing 19 changed files with 14 additions and 19 deletions.
2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/apache/assets/create_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ WITH (
checkpoint_location = '{s3_checkpoint_location}',
watermark_delay = '1 Minute',
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
);
)
2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,4 @@ WITH (
checkpoint_location = '{s3_checkpoint_location}',
watermark_delay = '1 Minute',
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
);
)
2 changes: 1 addition & 1 deletion ...er/adaptors/integrations/__data__/repository/aws_cloudfront/assets/create_table-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,4 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
USING csv
LOCATION '{s3_bucket_location}'
OPTIONS ( sep='\t' )
TBLPROPERTIES ( 'skip.header.line.count'='2' );
TBLPROPERTIES ( 'skip.header.line.count'='2' )
2 changes: 1 addition & 1 deletion ...integrations/__data__/repository/aws_cloudtrail/assets/create_table_cloud-trail-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,4 +62,4 @@ LOCATION '{s3_bucket_location}'
OPTIONS (
compression='gzip',
recursivefilelookup='true'
);
)
2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,4 +57,4 @@ WITH (
checkpoint_location = '{s3_checkpoint_location}',
watermark_delay = '1 Minute',
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
);
)
2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,4 @@ OPTIONS (
sep=' ',
compression='gzip',
recursiveFileLookup='true'
);
)
2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/aws_s3/assets/create_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,4 @@ WITH (
checkpoint_location = '{s3_checkpoint_location}',
watermark_delay = '1 Minute',
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
);
)
2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/aws_s3/assets/create_table-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,4 @@ USING csv
OPTIONS (
sep=' '
)
LOCATION '{s3_bucket_location}';
LOCATION '{s3_bucket_location}'
2 changes: 1 addition & 1 deletion .../adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_table_vpc-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
LOCATION '{s3_bucket_location}'
OPTIONS (
sep=' '
);
)
1 change: 0 additions & 1 deletion ...tegrations/__data__/repository/aws_vpc_flow/assets/vpc_agg_60min_connections_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,3 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS {table_name}__agg_60_min_connections_mvie
WITH (
auto_refresh = false
)

1 change: 0 additions & 1 deletion ...r/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_live_all_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,3 @@ WITH (
watermark_delay = '1 Minute',
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
)

1 change: 0 additions & 1 deletion .../adaptors/integrations/__data__/repository/aws_vpc_flow/assets/vpc_live_week_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,3 @@ CREATE MATERIALIZED VIEW {table_name}__week_live_mview AS
WITH (
auto_refresh = false
)

1 change: 0 additions & 1 deletion ...ta__/repository/aws_vpc_flow/assets/vpc_window-agg_60min_dest_ip_cardinality_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,3 @@ ORDER BY
WITH (
auto_refresh = false
)

1 change: 0 additions & 1 deletion ...ta__/repository/aws_vpc_flow/assets/vpc_window-agg_60min_dest_ip_total-bytes_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,3 @@ WITH hourly_buckets AS (
WITH (
auto_refresh = false
)

2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/aws_waf/assets/create_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ WITH (
checkpoint_location = '{s3_checkpoint_location}',
watermark_delay = '1 Minute',
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
);
)
2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/aws_waf/assets/create_table-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,4 +125,4 @@ CREATE EXTERNAL TABLE {table_name} (
`ja3Fingerprint` string
)
USING json
LOCATION '{s3_bucket_location}'
LOCATION '{s3_bucket_location}'
2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/haproxy/assets/create_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,4 +148,4 @@ WITH (
checkpoint_location = '{s3_checkpoint_location}',
watermark_delay = '1 Minute',
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
);
)
2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/haproxy/assets/create_table-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ CREATE EXTERNAL TABLE {table_name} (
LOCATION '{s3_bucket_location}'
OPTIONS (
sep='\0x1E'
);
)
2 changes: 1 addition & 1 deletion server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ WITH (
checkpoint_location = '{s3_checkpoint_location}',
watermark_delay = '1 Minute',
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
);
)

0 comments on commit 186e713

Please sign in to comment.