From 9be2e355c36769fc18d66afcb43360888248077f Mon Sep 17 00:00:00 2001 From: jbeemster Date: Wed, 3 Apr 2024 17:37:37 +0200 Subject: [PATCH] Add support for AWS China regions (closes #12) --- README.md | 1 + main.tf | 142 +++++++++++++++++++++--------------- templates/user-data.sh.tmpl | 6 +- variables.tf | 8 ++ 4 files changed, 98 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 3cff8f9..6746eef 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,7 @@ module "rs_loader" { | [iam\_permissions\_boundary](#input\_iam\_permissions\_boundary) | The permissions boundary ARN to set on IAM roles created | `string` | `""` | no | | [instance\_type](#input\_instance\_type) | The instance type to use | `string` | `"t3a.micro"` | no | | [java\_opts](#input\_java\_opts) | Custom JAVA Options | `string` | `"-XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75"` | no | +| [private\_ecr\_registry](#input\_private\_ecr\_registry) | The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/') | `string` | `""` | no | | [redshift\_aws\_s3\_folder\_monitoring\_stage\_url](#input\_redshift\_aws\_s3\_folder\_monitoring\_stage\_url) | AWS bucket URL of folder monitoring stage - must be within 'redshift\_aws\_s3\_bucket\_name' (NOTE: must be set if 'folder\_monitoring\_enabled' is true) | `string` | `""` | no | | [redshift\_aws\_s3\_folder\_monitoring\_transformer\_output\_stage\_url](#input\_redshift\_aws\_s3\_folder\_monitoring\_transformer\_output\_stage\_url) | AWS bucket URL of transformer output stage - must be within 'redshift\_aws\_s3\_bucket\_name' (NOTE: must be set if 'folder\_monitoring\_enabled' is true) | `string` | `""` | no | | [redshift\_jsonpaths\_bucket](#input\_redshift\_jsonpaths\_bucket) | S3 path that holds JSONPaths | `string` | `""` | no | diff --git a/main.tf b/main.tf index 22c2502..e042933 100644 --- a/main.tf +++ b/main.tf @@ -24,6 +24,25 @@ locals { data "aws_region" "current" {} data "aws_caller_identity" "current" {} +locals { + is_aws_global = replace(data.aws_region.current.name, "cn-", "") == data.aws_region.current.name + iam_partition = local.is_aws_global ? "aws" : "aws-cn" + + is_private_ecr_registry = var.private_ecr_registry != "" + private_ecr_registry_statement = [{ + Action = [ + "ecr:GetAuthorizationToken", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer" + ] + Effect = "Allow" + Resource = [ + "*" + ] + }] + private_ecr_registry_statement_final = local.is_private_ecr_registry ? local.private_ecr_registry_statement : [] +} + module "telemetry" { source = "snowplow-devops/telemetry/snowplow" version = "0.5.0" @@ -79,61 +98,64 @@ resource "aws_iam_policy" "iam_policy" { policy = jsonencode({ Version = "2012-10-17", - Statement = [ - { - Effect = "Allow", - Action = [ - "s3:ListBucket", - "s3:PutObject", - "s3:GetObject" - ], - Resource = [ - "arn:aws:s3:::${var.redshift_aws_s3_bucket_name}/", - "arn:aws:s3:::${var.redshift_aws_s3_bucket_name}/*" - ] - }, - { - Effect = "Allow", - Action = ["s3:GetObject"], - Resource = [ - "arn:aws:s3:::${var.redshift_aws_s3_bucket_name}/*/shredding_complete.json" - ] - }, - { - Effect = "Allow", - Action = [ - "sqs:DeleteMessage", - "sqs:GetQueueUrl", - "sqs:ListQueues", - "sqs:ChangeMessageVisibility", - "sqs:ReceiveMessage", - "sqs:SendMessage" - ], - Resource = [ - "arn:aws:sqs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${var.sqs_queue_name}" - ] - }, - { - Effect = "Allow", - Action = [ - "logs:PutLogEvents", - "logs:CreateLogStream", - "logs:DescribeLogStreams" - ], - Resource = [ - "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${local.cloudwatch_log_group_name}:*" - ] - }, - { - Effect = "Allow", - Action = [ - "sts:AssumeRole" - ], - Resource = [ - aws_iam_role.sts_credentials_role.arn - ] - } - ] + Statement = concat( + local.private_ecr_registry_statement_final, + [ + { + Effect = "Allow", + Action = [ + "s3:ListBucket", + "s3:PutObject", + "s3:GetObject" + ], + Resource = [ + "arn:${local.iam_partition}:s3:::${var.redshift_aws_s3_bucket_name}/", + "arn:${local.iam_partition}:s3:::${var.redshift_aws_s3_bucket_name}/*" + ] + }, + { + Effect = "Allow", + Action = ["s3:GetObject"], + Resource = [ + "arn:${local.iam_partition}:s3:::${var.redshift_aws_s3_bucket_name}/*/shredding_complete.json" + ] + }, + { + Effect = "Allow", + Action = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ListQueues", + "sqs:ChangeMessageVisibility", + "sqs:ReceiveMessage", + "sqs:SendMessage" + ], + Resource = [ + "arn:${local.iam_partition}:sqs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${var.sqs_queue_name}" + ] + }, + { + Effect = "Allow", + Action = [ + "logs:PutLogEvents", + "logs:CreateLogStream", + "logs:DescribeLogStreams" + ], + Resource = [ + "arn:${local.iam_partition}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${local.cloudwatch_log_group_name}:*" + ] + }, + { + Effect = "Allow", + Action = [ + "sts:AssumeRole" + ], + Resource = [ + aws_iam_role.sts_credentials_role.arn + ] + } + ] + ) }) } @@ -184,9 +206,9 @@ resource "aws_iam_policy" "sts_credentials_policy" { "s3:GetObjectVersion", ], Resource = [ - "arn:aws:s3:::${var.redshift_aws_s3_bucket_name}", - "arn:aws:s3:::${var.redshift_aws_s3_bucket_name}/", - "arn:aws:s3:::${var.redshift_aws_s3_bucket_name}/*" + "arn:${local.iam_partition}:s3:::${var.redshift_aws_s3_bucket_name}", + "arn:${local.iam_partition}:s3:::${var.redshift_aws_s3_bucket_name}/", + "arn:${local.iam_partition}:s3:::${var.redshift_aws_s3_bucket_name}/*" ] } ] @@ -377,6 +399,10 @@ locals { container_memory = "${module.instance_type_metrics.memory_application_mb}m" java_opts = var.java_opts + + is_private_ecr_registry = local.is_private_ecr_registry + private_ecr_registry = var.private_ecr_registry + region = data.aws_region.current.name }) } diff --git a/templates/user-data.sh.tmpl b/templates/user-data.sh.tmpl index 6f4fe09..85117a6 100644 --- a/templates/user-data.sh.tmpl +++ b/templates/user-data.sh.tmpl @@ -1,3 +1,7 @@ +%{ if is_private_ecr_registry } +aws ecr get-login-password --region ${region} | docker login --username AWS --password-stdin ${private_ecr_registry} +%{ endif ~} + # Launch the loader sudo docker run \ -d \ @@ -16,7 +20,7 @@ sudo docker run \ --env JDK_JAVA_OPTIONS='${java_opts}' \ --env ACCEPT_LIMITED_USE_LICENSE=${accept_limited_use_license} \ --env INSTANCE_ID=$(get_instance_id) \ - snowplow/rdb-loader-redshift:${version} \ + ${private_ecr_registry}snowplow/rdb-loader-redshift:${version} \ --config ${config_b64} \ --iglu-config ${iglu_resolver_b64} diff --git a/variables.tf b/variables.tf index 28962b5..1e551a7 100644 --- a/variables.tf +++ b/variables.tf @@ -393,3 +393,11 @@ variable "redshift_aws_s3_folder_monitoring_transformer_output_stage_url" { type = string default = "" } + +# --- Image Repositories + +variable "private_ecr_registry" { + description = "The URL of an ECR registry that the sub-account has access to (e.g. '000000000000.dkr.ecr.cn-north-1.amazonaws.com.cn/')" + type = string + default = "" +}