diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index a9c499ae..a4b43379 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -193,13 +193,13 @@ jobs: branch: env/test/test secrets: inherit - deploy_perf: - needs: fast_forward_env_branches - if: ${{needs.fast_forward_env_branches.outputs.deploy_perf}} - uses: ./.github/workflows/well-known-environment.yml - with: - branch: env/test/perf - secrets: inherit +# deploy_perf: +# needs: fast_forward_env_branches +# if: ${{needs.fast_forward_env_branches.outputs.deploy_perf}} +# uses: ./.github/workflows/well-known-environment.yml +# with: +# branch: env/test/perf +# secrets: inherit deploy_staging: needs: fast_forward_env_branches diff --git a/README.md b/README.md index d425c802..34d9804e 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ region = eu-west-2 #### Updating the config files directly -The `~/.aws/config` should be updated with the profile names we use. Please follow the [instructions in Confluence](https://digitaltools.phe.org.uk/confluence/display/DPD/Configuring+the+AWS+CLI). +The `~/.aws/config` should be updated with the profile names we use. Please follow the [instructions in Confluence](https://confluence.collab.test-and-trace.nhs.uk/display/DPD/Configuring+the+AWS+CLI). ### Login to the GitHub CLI @@ -510,3 +510,6 @@ These repos contain the app source code: - [UKHSA-Internal/data-dashboard-frontend](https://github.com/UKHSA-Internal/data-dashboard-frontend) - [UKHSA-Internal/data-dashboard-api](https://github.com/UKHSA-Internal/data-dashboard-api) + +This repo contains the infra for the part of the ETL pipeline which sits within AWS: +- [data-dashboard-etl-infra](https://github.com/UKHSA-Internal/data-dashboard-etl-infra) diff --git a/scripts/_docker.sh b/scripts/_docker.sh index 6d223606..5a82747a 100644 --- a/scripts/_docker.sh +++ b/scripts/_docker.sh @@ -40,7 +40,10 @@ function _docker_build_with_custom_tag() { return 1 fi - local dev_account_id=$(_get_target_aws_account_id "dev") + local account_name="dev" + uhd docker ecr:login ${account_name} + + local dev_account_id=$(_get_target_aws_account_id ${account_name}) local env=$(_get_env_name) if [[ ${repo} == "ingestion" ]]; then diff --git a/scripts/_terraform.sh b/scripts/_terraform.sh index 9c9db074..d6f7aa45 100644 --- a/scripts/_terraform.sh +++ b/scripts/_terraform.sh @@ -142,6 +142,8 @@ function _terraform_plan_layer() { return 1 fi + local etl_account_id=$(_get_etl_sibling_aws_account_id $target_account_name) + local var_file="etc/${target_account_name}.tfvars" cd $terraform_dir @@ -151,6 +153,7 @@ function _terraform_plan_layer() { -var "assume_account_id=${assume_account_id}" \ -var "tools_account_id=${tools_account_id}" \ -var "python_version=${python_version}" \ + -var "etl_account_id=${etl_account_id}" \ -var-file=$var_file || return 1 } @@ -194,6 +197,8 @@ function _terraform_import_layer() { return 1 fi + local etl_account_id=$(_get_etl_sibling_aws_account_id $target_account_name) + local var_file="etc/${target_account_name}.tfvars" cd $terraform_dir @@ -203,6 +208,7 @@ function _terraform_import_layer() { -var "assume_account_id=${assume_account_id}" \ -var "tools_account_id=${tools_account_id}" \ -var "python_version=${python_version}" \ + -var "etl_account_id=${etl_account_id}" \ -var-file=$var_file \ $address \ $id || return 0 @@ -242,6 +248,8 @@ function _terraform_apply_layer() { return 1 fi + local etl_account_id=$(_get_etl_sibling_aws_account_id $target_account_name) + local var_file="etc/${target_account_name}.tfvars" cd $terraform_dir @@ -251,6 +259,7 @@ function _terraform_apply_layer() { -var "assume_account_id=${assume_account_id}" \ -var "tools_account_id=${tools_account_id}" \ -var "python_version=${python_version}" \ + -var "etl_account_id=${etl_account_id}" \ -var-file=$var_file \ -auto-approve || return 1 @@ -360,6 +369,8 @@ function _terraform_destroy_layer() { return 1 fi + local etl_account_id=$(_get_etl_sibling_aws_account_id $target_account_name) + local var_file="etc/${target_account_name}.tfvars" cd $terraform_dir @@ -369,6 +380,7 @@ function _terraform_destroy_layer() { -var "assume_account_id=${assume_account_id}" \ -var "tools_account_id=${tools_account_id}" \ -var "python_version=${python_version}" \ + -var "etl_account_id=${etl_account_id}" \ -var-file=$var_file \ -auto-approve || return 1 @@ -456,6 +468,8 @@ function _terraform_cleanup() { echo "Environment $env is a test environment. It will be destroyed... " echo uhd terraform destroy:layer 20-app $env + elif [[ $env == etl-ci-* ]]; then + echo "Environment $env is a CI environment belonging to the ETL infra. Skipping this. " else echo "Environment $env is an engineer's dev or well known environment." fi @@ -480,6 +494,16 @@ function _get_target_aws_account_id() { aws secretsmanager get-secret-value --secret-id "aws/account-id/$account" --query SecretString --output text } +function _get_etl_sibling_aws_account_id() { + local account=$1 + local tools_account_id=$(_get_tools_account_id) + + aws secretsmanager get-secret-value \ + --secret-id "aws/account-id/etl-$account" \ + --query SecretString \ + --output text +} + function _get_target_aws_account_name() { local layer=$1 local workspace=$2 diff --git a/terraform/10-account/.terraform.lock.hcl b/terraform/10-account/.terraform.lock.hcl index ca360415..6b296faa 100644 --- a/terraform/10-account/.terraform.lock.hcl +++ b/terraform/10-account/.terraform.lock.hcl @@ -2,26 +2,26 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "5.72.0" - constraints = ">= 2.49.0, >= 3.74.0, >= 4.0.0, >= 4.36.0, >= 4.40.0, >= 5.27.0, >= 5.37.0, 5.72.0" + version = "5.78.0" + constraints = ">= 2.49.0, >= 3.74.0, >= 4.0.0, >= 4.36.0, >= 4.40.0, >= 5.27.0, >= 5.37.0, >= 5.49.0, 5.78.0" hashes = [ - "h1:CCf0c/ycpptDMDvQhvxkpM9JTb6fsY/uhVIqw3VxeT0=", - "h1:aQYhYMeYdg/EYlOG/WSifLQ+w/EqyBOPQlNnieAKhik=", - "h1:g7DtrWPK6KyZgqy6ZxYsSNhiVu/iPNLWnV7LKsJn/Q8=", - "zh:1a5e979b0c68063e1cdde592864fbfd0566f81c2125f811e96c9d85815d9a5fb", - "zh:2c1cd5f43748ab9cf4127fcc7cb73535b01ebb0408b14edf4f0e9475521ab8ea", - "zh:32749d892279fb14ce2a02ca4f7d25891983464f6696ed62e4a4350ee7931d02", - "zh:32ece5073409de4a25ca7b35e246117da7915683eb6317797b0c98e8b9b4cfbd", - "zh:3b7bb7be87e2ab432c0a30d085ef9a06b87def29f69cf5154bfc07bec7811db4", - "zh:5f986460e947675c9f85cc233c68b9af16385ac2e66c0ae22ed3501cd482abf5", - "zh:6ac164504e04d1b6d83762f50896005362c250a69203e5f25313b410711e826b", - "zh:6e639b8ddc44a299fef194d5792f6df69b26e350e44185600bbac8e1a8b2fbd3", + "h1:/EKXECKi3XzLR0SwaEyQucvsQx2TdyIF2DYabr9DVqM=", + "h1:OUmta/bL/0S6g4K/Mn1LBkEnMWNCq4dLsfYrdrllcEo=", + "h1:o7jz+dFixEcwjfdubken5ldmDJm1tkvM2adPtNDei3g=", + "zh:0ae7d41b96441d0cf7ce2e1337657bdb2e1e5c9f1c2227b0642e1dcec2f9dfba", + "zh:21f8f1edf477681ea3b095c02cad6b8e85262e45015de58e84e0c7b2bfe9a1f6", + "zh:2bdc335e341bf98445255549ae93d66cfb9bca706e62b949da98fe467c182cad", + "zh:2fe4096e260367a225a9faf4a424d62b87e5498f12cb43bdb6f4e713d11b82c3", + "zh:3c63bb7a7925d65118d17461f4691a22dbb55ea39a7404e4d71f6ccca8765f8b", + "zh:6609a28a1c638a1901d8007b5386868ccfd313b4df2e98b35d9fdef436974e3b", + "zh:7ae3aef43bc4b365824cca4659cf92459d766800656e354bdbf83feabab835e8", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9b396ba2682256f5321162306d2bea5c276b8cd1a5f877599746518804554e80", - "zh:ab0688723e14acdd9d9f244086be7c22e2b938add611b814ce7b1662484e5d08", - "zh:c2a6453f223b7a2491a75481ef30421bc15e1983530e082467bcfa99553dca51", - "zh:d794b429ef1c259188a31626725af18e6ac8a9bbe1478484f8d0a0db57703335", - "zh:d86fb03571f8446928659efad065a235dda6320fdf893b575c54ae9df5adfda8", - "zh:ec27eddf86ee2f2003ae87faf269fac379311531394cf5292b9c31d011b9a873", + "zh:c314efe454adc6ca483261c6906e64315aeb9db0c0332818714e9b81e07df0f0", + "zh:cd3e30396b554bbc1d260252db8a0f344065d619038fe60ea870689cd32c6aa9", + "zh:d1ba48fd9d8a1cb1daa927fb9e8bb708b857f2792d796e110460c6fdcd896a47", + "zh:d31c8abe75cb9cdc1c59ad9d356a1c3ae1ba8cd29ac15eb7e01b6cd01221ab04", + "zh:dc27c5c2116b4d9b404753f73bccaa635bce21f3bfb4bb7bc8e63225c36c98fe", + "zh:de491f0d05408378413187475c815d8cb2ac6bfa63d0b42a30ad5ee492e51c07", + "zh:eb44b45a40f80a309dd5b0eb7d7fcb2cbfe588fe2f18b173ef5851346898a662", ] } diff --git a/terraform/10-account/iam.automation-user.tf b/terraform/10-account/iam.automation-user.tf deleted file mode 100644 index deb27615..00000000 --- a/terraform/10-account/iam.automation-user.tf +++ /dev/null @@ -1,30 +0,0 @@ -module "iam_data_ingestion_automation_user" { - source = "terraform-aws-modules/iam/aws//modules/iam-user" - version = "5.44.0" - - name = "DataIngestionAutomation" - - create_iam_user_login_profile = false - create_iam_access_key = false - policy_arns = [module.iam_data_ingestion_automation_policy.arn] -} - -module "iam_data_ingestion_automation_policy" { - source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "5.44.0" - - name = "uhd-data-ingestion-automation-policy" - - policy = jsonencode( - { - Version = "2012-10-17", - Statement = [ - { - Action = ["sts:AssumeRole"], - Effect = "Allow", - Resource = module.iam_operations_role.iam_role_arn - } - ] - } - ) -} diff --git a/terraform/10-account/iam.operations-role.tf b/terraform/10-account/iam.operations-role.tf index 0d679c22..682dcf10 100644 --- a/terraform/10-account/iam.operations-role.tf +++ b/terraform/10-account/iam.operations-role.tf @@ -16,11 +16,9 @@ module "iam_operations_role" { local.sso_role_arns.administrator, local.sso_role_arns.developer, local.sso_role_arns.operations, - module.iam_data_ingestion_automation_user.iam_user_arn ] : [ local.sso_role_arns.administrator, local.sso_role_arns.operations, - module.iam_data_ingestion_automation_user.iam_user_arn ] } diff --git a/terraform/10-account/vars.tf b/terraform/10-account/vars.tf index 58308a6b..6aab1527 100644 --- a/terraform/10-account/vars.tf +++ b/terraform/10-account/vars.tf @@ -15,4 +15,8 @@ variable "tools_account_id" { sensitive = true } +variable "etl_account_id" { + sensitive = true +} + variable "halo_account_type" {} \ No newline at end of file diff --git a/terraform/10-account/versions.tf b/terraform/10-account/versions.tf index 0069b852..4a661529 100644 --- a/terraform/10-account/versions.tf +++ b/terraform/10-account/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "5.72.0" + version = "5.78.0" } } required_version = ">= 1.4.5" diff --git a/terraform/20-app/.terraform.lock.hcl b/terraform/20-app/.terraform.lock.hcl index 60abf3a5..3f4156db 100644 --- a/terraform/20-app/.terraform.lock.hcl +++ b/terraform/20-app/.terraform.lock.hcl @@ -2,27 +2,27 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "5.72.0" - constraints = ">= 3.29.0, >= 3.74.0, >= 4.66.1, >= 5.0.0, >= 5.12.0, >= 5.25.0, >= 5.27.0, >= 5.32.0, >= 5.37.0, >= 5.46.0, >= 5.49.0, >= 5.58.0, >= 5.61.0, >= 5.72.0, 5.72.0" + version = "5.78.0" + constraints = ">= 3.29.0, >= 3.74.0, >= 4.66.1, >= 5.0.0, >= 5.12.0, >= 5.25.0, >= 5.27.0, >= 5.32.0, >= 5.37.0, >= 5.46.0, >= 5.49.0, >= 5.58.0, >= 5.61.0, >= 5.75.1, 5.78.0" hashes = [ - "h1:CCf0c/ycpptDMDvQhvxkpM9JTb6fsY/uhVIqw3VxeT0=", - "h1:aQYhYMeYdg/EYlOG/WSifLQ+w/EqyBOPQlNnieAKhik=", - "h1:g7DtrWPK6KyZgqy6ZxYsSNhiVu/iPNLWnV7LKsJn/Q8=", - "zh:1a5e979b0c68063e1cdde592864fbfd0566f81c2125f811e96c9d85815d9a5fb", - "zh:2c1cd5f43748ab9cf4127fcc7cb73535b01ebb0408b14edf4f0e9475521ab8ea", - "zh:32749d892279fb14ce2a02ca4f7d25891983464f6696ed62e4a4350ee7931d02", - "zh:32ece5073409de4a25ca7b35e246117da7915683eb6317797b0c98e8b9b4cfbd", - "zh:3b7bb7be87e2ab432c0a30d085ef9a06b87def29f69cf5154bfc07bec7811db4", - "zh:5f986460e947675c9f85cc233c68b9af16385ac2e66c0ae22ed3501cd482abf5", - "zh:6ac164504e04d1b6d83762f50896005362c250a69203e5f25313b410711e826b", - "zh:6e639b8ddc44a299fef194d5792f6df69b26e350e44185600bbac8e1a8b2fbd3", + "h1:/EKXECKi3XzLR0SwaEyQucvsQx2TdyIF2DYabr9DVqM=", + "h1:OUmta/bL/0S6g4K/Mn1LBkEnMWNCq4dLsfYrdrllcEo=", + "h1:o7jz+dFixEcwjfdubken5ldmDJm1tkvM2adPtNDei3g=", + "zh:0ae7d41b96441d0cf7ce2e1337657bdb2e1e5c9f1c2227b0642e1dcec2f9dfba", + "zh:21f8f1edf477681ea3b095c02cad6b8e85262e45015de58e84e0c7b2bfe9a1f6", + "zh:2bdc335e341bf98445255549ae93d66cfb9bca706e62b949da98fe467c182cad", + "zh:2fe4096e260367a225a9faf4a424d62b87e5498f12cb43bdb6f4e713d11b82c3", + "zh:3c63bb7a7925d65118d17461f4691a22dbb55ea39a7404e4d71f6ccca8765f8b", + "zh:6609a28a1c638a1901d8007b5386868ccfd313b4df2e98b35d9fdef436974e3b", + "zh:7ae3aef43bc4b365824cca4659cf92459d766800656e354bdbf83feabab835e8", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9b396ba2682256f5321162306d2bea5c276b8cd1a5f877599746518804554e80", - "zh:ab0688723e14acdd9d9f244086be7c22e2b938add611b814ce7b1662484e5d08", - "zh:c2a6453f223b7a2491a75481ef30421bc15e1983530e082467bcfa99553dca51", - "zh:d794b429ef1c259188a31626725af18e6ac8a9bbe1478484f8d0a0db57703335", - "zh:d86fb03571f8446928659efad065a235dda6320fdf893b575c54ae9df5adfda8", - "zh:ec27eddf86ee2f2003ae87faf269fac379311531394cf5292b9c31d011b9a873", + "zh:c314efe454adc6ca483261c6906e64315aeb9db0c0332818714e9b81e07df0f0", + "zh:cd3e30396b554bbc1d260252db8a0f344065d619038fe60ea870689cd32c6aa9", + "zh:d1ba48fd9d8a1cb1daa927fb9e8bb708b857f2792d796e110460c6fdcd896a47", + "zh:d31c8abe75cb9cdc1c59ad9d356a1c3ae1ba8cd29ac15eb7e01b6cd01221ab04", + "zh:dc27c5c2116b4d9b404753f73bccaa635bce21f3bfb4bb7bc8e63225c36c98fe", + "zh:de491f0d05408378413187475c815d8cb2ac6bfa63d0b42a30ad5ee492e51c07", + "zh:eb44b45a40f80a309dd5b0eb7d7fcb2cbfe588fe2f18b173ef5851346898a662", ] } diff --git a/terraform/20-app/aurora-db.app.tf b/terraform/20-app/aurora-db.app.tf index a01f63b6..e9e44597 100644 --- a/terraform/20-app/aurora-db.app.tf +++ b/terraform/20-app/aurora-db.app.tf @@ -25,7 +25,7 @@ module "aurora_db_app" { min_capacity = 1 max_capacity = 50 } - instances = local.use_prod_sizing ? { 1 : {}, 2 : {}, 3: {} } : { 1 : {} } + instances = local.use_prod_sizing ? { 1 : {}, 2 : {}, 3 : {} } : { 1 : {} } vpc_id = module.vpc.vpc_id db_subnet_group_name = module.vpc.database_subnet_group_name @@ -50,6 +50,12 @@ module "aurora_db_app" { protocol = "tcp" source_security_group_id = module.ecs_service_cms_admin.security_group_id }, + feedback_api_tasks_to_db = { + type = "ingress" + description = "feedback api tasks to main db" + protocol = "tcp" + source_security_group_id = module.ecs_service_feedback_api.security_group_id + }, utility_worker_tasks_to_db = { type = "ingress" description = "utility worker tasks to main db" @@ -72,11 +78,7 @@ locals { db_name = module.aurora_db_app.cluster_database_name address = module.aurora_db_app.cluster_endpoint } - public_api_replica = { - db_name = module.aurora_db_app.cluster_database_name - address = module.aurora_db_app.cluster_reader_endpoint - } - private_api_replica = { + secondary = { db_name = module.aurora_db_app.cluster_database_name address = module.aurora_db_app.cluster_reader_endpoint } diff --git a/terraform/20-app/ecs.service.feedback-api.tf b/terraform/20-app/ecs.service.feedback-api.tf index 30452e73..0693cf4a 100644 --- a/terraform/20-app/ecs.service.feedback-api.tf +++ b/terraform/20-app/ecs.service.feedback-api.tf @@ -42,12 +42,28 @@ module "ecs_service_feedback_api" { name = "APP_MODE" value = "FEEDBACK_API" }, + { + name = "POSTGRES_DB" + value = local.aurora.app.secondary.db_name + }, + { + name = "POSTGRES_HOST" + value = local.aurora.app.secondary.address + }, { name = "APIENV" - value = "STANDALONE" + value = "PROD" }, ], secrets = [ + { + name = "POSTGRES_USER" + valueFrom = "${local.main_db_aurora_password_secret_arn}:username::" + }, + { + name = "POSTGRES_PASSWORD" + valueFrom = "${local.main_db_aurora_password_secret_arn}:password::" + }, { name = "SECRET_KEY", valueFrom = aws_secretsmanager_secret.backend_cryptographic_signing_key.arn @@ -113,6 +129,13 @@ module "ecs_service_feedback_api" { protocol = "tcp" cidr_blocks = ["0.0.0.0/0"] } + db_egress = { + type = "egress" + from_port = 5432 + to_port = 5432 + protocol = "tcp" + source_security_group_id = module.aurora_db_app.security_group_id + } internet_egress = { type = "egress" from_port = 443 diff --git a/terraform/20-app/ecs.service.private-api.tf b/terraform/20-app/ecs.service.private-api.tf index 57542778..d3740b7d 100644 --- a/terraform/20-app/ecs.service.private-api.tf +++ b/terraform/20-app/ecs.service.private-api.tf @@ -46,11 +46,11 @@ module "ecs_service_private_api" { }, { name = "POSTGRES_DB" - value = local.aurora.app.private_api_replica.db_name + value = local.aurora.app.secondary.db_name }, { name = "POSTGRES_HOST" - value = local.aurora.app.private_api_replica.address + value = local.aurora.app.secondary.address }, { name = "APIENV" diff --git a/terraform/20-app/ecs.service.public-api.tf b/terraform/20-app/ecs.service.public-api.tf index acc6ca0e..0c5dffb5 100644 --- a/terraform/20-app/ecs.service.public-api.tf +++ b/terraform/20-app/ecs.service.public-api.tf @@ -48,11 +48,11 @@ module "ecs_service_public_api" { }, { name = "POSTGRES_DB" - value = local.aurora.app.public_api_replica.db_name + value = local.aurora.app.secondary.db_name }, { name = "POSTGRES_HOST" - value = local.aurora.app.public_api_replica.address + value = local.aurora.app.secondary.address }, { name = "APIENV" diff --git a/terraform/20-app/ecs.service.utility-worker.tf b/terraform/20-app/ecs.service.utility-worker.tf index 317d8671..51a12ac7 100644 --- a/terraform/20-app/ecs.service.utility-worker.tf +++ b/terraform/20-app/ecs.service.utility-worker.tf @@ -42,11 +42,11 @@ module "ecs_service_utility_worker" { }, { name = "POSTGRES_DB" - value = local.aurora.app.private_api_replica.db_name + value = local.aurora.app.secondary.db_name }, { name = "POSTGRES_HOST" - value = local.aurora.app.private_api_replica.address + value = local.aurora.app.secondary.address }, { name = "APIENV" diff --git a/terraform/20-app/ip-allow-lists.tf b/terraform/20-app/ip-allow-lists.tf index 17b64f1b..80ebc3cd 100644 --- a/terraform/20-app/ip-allow-lists.tf +++ b/terraform/20-app/ip-allow-lists.tf @@ -6,13 +6,14 @@ locals { "154.51.68.102/32", # Burendo Leeds "167.98.124.170/32", # Burendo London "90.219.251.228/32", # Phil - "81.78.13.149/32", # Rhys + "84.67.254.137/32", # Rhys + "176.254.91.127/32", # Rhys 2 "35.176.13.254/32", # UKHSA test EC2 "35.176.178.91/32", # UKHSA test EC2 "35.179.30.107/32", # UKHSA test EC2 "18.133.111.70/32", # UKHSA test gateway "81.108.89.51/32", # Krishna - Macbook - "165.225.199.78/32", # Krishna - Windows + "165.225.197.26/32", # Krishna - Windows "80.7.227.61/32", # Kiran "92.234.44.48/32", # Zesh "51.241.222.137/32", # Temitope Akinsoto @@ -48,6 +49,7 @@ locals { ncc = [ "5.148.69.16/28", "167.98.200.192/27", + "167.98.200.196/27", "195.95.131.0/24", "5.148.32.192/26", ] diff --git a/terraform/20-app/locals.tf b/terraform/20-app/locals.tf index 3b13e738..90777480 100644 --- a/terraform/20-app/locals.tf +++ b/terraform/20-app/locals.tf @@ -5,6 +5,7 @@ locals { prefix = "${local.project}-${local.environment}" account_id = var.assume_account_id + etl_account_id = var.etl_account_id default_log_retention_in_days = 30 alb_security_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" @@ -26,6 +27,7 @@ locals { enable_public_db = local.is_dev is_dev = var.environment_type == "dev" is_prod = local.environment == "prod" + is_ready_for_etl = contains(["dev", "test", "dpd", "staging", "prod"], local.environment) use_ip_allow_list = local.environment != "prod" diff --git a/terraform/20-app/s3.ingest.tf b/terraform/20-app/s3.ingest.tf index fb380957..ef7d7ff5 100644 --- a/terraform/20-app/s3.ingest.tf +++ b/terraform/20-app/s3.ingest.tf @@ -49,10 +49,10 @@ module "s3_ingest" { } ] - attach_policy = true - policy = jsonencode({ + attach_policy = true + policy = jsonencode({ Version = "2012-10-17", - Statement = [ + Statement = concat([ { Sid = "OnlyAllowJsonFilesToTargetFolders", Effect = "Deny", @@ -67,7 +67,21 @@ module "s3_ingest" { "${module.s3_ingest.s3_bucket_arn}/processed/*.json", ] } - ] + ], + local.is_ready_for_etl ? [ + { + Sid = "AllowCrossAccountAccessFromETLPublisherLambda", + Effect = "Allow", + Principal = { + AWS = "arn:aws:iam::${local.etl_account_id}:role/${local.project}-etl-${local.environment}-publisher" + } + Action = ["s3:PutObject", "s3:ListBucket"] + Resource = [ + module.s3_ingest.s3_bucket_arn, + "${module.s3_ingest.s3_bucket_arn}/*", + ] + } + ] : []) }) } diff --git a/terraform/20-app/vars.tf b/terraform/20-app/vars.tf index e5c9ee7f..7868ffef 100644 --- a/terraform/20-app/vars.tf +++ b/terraform/20-app/vars.tf @@ -22,6 +22,10 @@ variable "tools_account_id" { sensitive = true } +variable "etl_account_id" { + sensitive = true +} + variable "single_nat_gateway" { default = true } diff --git a/terraform/20-app/versions.tf b/terraform/20-app/versions.tf index 0b442ab9..632bc779 100644 --- a/terraform/20-app/versions.tf +++ b/terraform/20-app/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "5.72.0" + version = "5.78.0" } random = { source = "hashicorp/random"