Merge branch 'main' into task/make-private-api-private/CDD-1177

.github/workflows/production.yml

@@ -193,13 +193,13 @@ jobs:
       branch: env/test/test
     secrets: inherit
 
-  deploy_perf:
-    needs: fast_forward_env_branches
-    if: ${{needs.fast_forward_env_branches.outputs.deploy_perf}}
-    uses: ./.github/workflows/well-known-environment.yml
-    with:
-      branch: env/test/perf
-    secrets: inherit
+#  deploy_perf:
+#    needs: fast_forward_env_branches
+#    if: ${{needs.fast_forward_env_branches.outputs.deploy_perf}}
+#    uses: ./.github/workflows/well-known-environment.yml
+#    with:
+#      branch: env/test/perf
+#    secrets: inherit
 
   deploy_staging:
     needs: fast_forward_env_branches

README.md

@@ -96,7 +96,7 @@ region = eu-west-2
 
 #### Updating the config files directly
 
-The `~/.aws/config` should be updated with the profile names we use. Please follow the [instructions in Confluence](https://digitaltools.phe.org.uk/confluence/display/DPD/Configuring+the+AWS+CLI).
+The `~/.aws/config` should be updated with the profile names we use. Please follow the [instructions in Confluence](https://confluence.collab.test-and-trace.nhs.uk/display/DPD/Configuring+the+AWS+CLI).
 
 ### Login to the GitHub CLI
 
@@ -510,3 +510,6 @@ These repos contain the app source code:
 
 - [UKHSA-Internal/data-dashboard-frontend](https://github.com/UKHSA-Internal/data-dashboard-frontend)
 - [UKHSA-Internal/data-dashboard-api](https://github.com/UKHSA-Internal/data-dashboard-api)
+
+This repo contains the infra for the part of the ETL pipeline which sits within AWS:
+- [data-dashboard-etl-infra](https://github.com/UKHSA-Internal/data-dashboard-etl-infra)

scripts/_docker.sh

@@ -40,7 +40,10 @@ function _docker_build_with_custom_tag() {
         return 1
     fi
 
-    local dev_account_id=$(_get_target_aws_account_id "dev")
+    local account_name="dev"
+    uhd docker ecr:login ${account_name}
+
+    local dev_account_id=$(_get_target_aws_account_id ${account_name})
     local env=$(_get_env_name)
 
     if [[ ${repo} == "ingestion" ]]; then

scripts/_terraform.sh

@@ -142,6 +142,8 @@ function _terraform_plan_layer() {
         return 1
     fi
 
+    local etl_account_id=$(_get_etl_sibling_aws_account_id $target_account_name)
+
     local var_file="etc/${target_account_name}.tfvars"
 
     cd $terraform_dir
@@ -151,6 +153,7 @@ function _terraform_plan_layer() {
         -var "assume_account_id=${assume_account_id}" \
         -var "tools_account_id=${tools_account_id}" \
         -var "python_version=${python_version}" \
+        -var "etl_account_id=${etl_account_id}" \
         -var-file=$var_file || return 1
 }
 
@@ -194,6 +197,8 @@ function _terraform_import_layer() {
         return 1
     fi
 
+    local etl_account_id=$(_get_etl_sibling_aws_account_id $target_account_name)
+
     local var_file="etc/${target_account_name}.tfvars"
 
     cd $terraform_dir
@@ -203,6 +208,7 @@ function _terraform_import_layer() {
         -var "assume_account_id=${assume_account_id}" \
         -var "tools_account_id=${tools_account_id}" \
         -var "python_version=${python_version}" \
+        -var "etl_account_id=${etl_account_id}" \
         -var-file=$var_file \
         $address \
         $id || return 0
@@ -242,6 +248,8 @@ function _terraform_apply_layer() {
         return 1
     fi
 
+    local etl_account_id=$(_get_etl_sibling_aws_account_id $target_account_name)
+
     local var_file="etc/${target_account_name}.tfvars"
 
     cd $terraform_dir
@@ -251,6 +259,7 @@ function _terraform_apply_layer() {
         -var "assume_account_id=${assume_account_id}" \
         -var "tools_account_id=${tools_account_id}" \
         -var "python_version=${python_version}" \
+        -var "etl_account_id=${etl_account_id}" \
         -var-file=$var_file \
         -auto-approve || return 1
 
@@ -360,6 +369,8 @@ function _terraform_destroy_layer() {
         return 1
     fi
 
+    local etl_account_id=$(_get_etl_sibling_aws_account_id $target_account_name)
+
     local var_file="etc/${target_account_name}.tfvars"
 
     cd $terraform_dir
@@ -369,6 +380,7 @@ function _terraform_destroy_layer() {
         -var "assume_account_id=${assume_account_id}" \
         -var "tools_account_id=${tools_account_id}" \
         -var "python_version=${python_version}" \
+        -var "etl_account_id=${etl_account_id}" \
         -var-file=$var_file \
         -auto-approve || return 1
 
@@ -456,6 +468,8 @@ function _terraform_cleanup() {
                 echo "Environment $env is a test environment.  It will be destroyed... "
                 echo
                 uhd terraform destroy:layer 20-app $env
+            elif [[ $env == etl-ci-* ]]; then
+                echo "Environment $env is a CI environment belonging to the ETL infra. Skipping this. "
             else
                 echo "Environment $env is an engineer's dev or well known environment."
             fi
@@ -480,6 +494,16 @@ function _get_target_aws_account_id() {
   aws secretsmanager get-secret-value --secret-id "aws/account-id/$account" --query SecretString --output text
 }
 
+function _get_etl_sibling_aws_account_id() {
+  local account=$1
+  local tools_account_id=$(_get_tools_account_id)
+
+  aws secretsmanager get-secret-value \
+    --secret-id "aws/account-id/etl-$account" \
+    --query SecretString \
+    --output text
+}
+
 function _get_target_aws_account_name() {
     local layer=$1
     local workspace=$2 

terraform/10-account/iam.operations-role.tf

@@ -16,11 +16,9 @@ module "iam_operations_role" {
     local.sso_role_arns.administrator,
     local.sso_role_arns.developer,
     local.sso_role_arns.operations,
-    module.iam_data_ingestion_automation_user.iam_user_arn
     ] : [
     local.sso_role_arns.administrator,
     local.sso_role_arns.operations,
-    module.iam_data_ingestion_automation_user.iam_user_arn
   ]
 }
 

terraform/10-account/vars.tf

@@ -15,4 +15,8 @@ variable "tools_account_id" {
   sensitive = true
 }
 
+variable "etl_account_id" {
+  sensitive = true
+}
+
 variable "halo_account_type" {}

terraform/10-account/versions.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "5.72.0"
+      version = "5.78.0"
     }
   }
   required_version = ">= 1.4.5"

terraform/20-app/aurora-db.app.tf

@@ -25,7 +25,7 @@ module "aurora_db_app" {
     min_capacity = 1
     max_capacity = 50
   }
-  instances = local.use_prod_sizing ? { 1 : {}, 2 : {}, 3: {} } : { 1 : {} }
+  instances = local.use_prod_sizing ? { 1 : {}, 2 : {}, 3 : {} } : { 1 : {} }
 
   vpc_id               = module.vpc.vpc_id
   db_subnet_group_name = module.vpc.database_subnet_group_name
@@ -50,6 +50,12 @@ module "aurora_db_app" {
       protocol                 = "tcp"
       source_security_group_id = module.ecs_service_cms_admin.security_group_id
     },
+    feedback_api_tasks_to_db = {
+      type                     = "ingress"
+      description              = "feedback api tasks to main db"
+      protocol                 = "tcp"
+      source_security_group_id = module.ecs_service_feedback_api.security_group_id
+    },
     utility_worker_tasks_to_db = {
       type                     = "ingress"
       description              = "utility worker tasks to main db"
@@ -72,11 +78,7 @@ locals {
         db_name = module.aurora_db_app.cluster_database_name
         address = module.aurora_db_app.cluster_endpoint
       }
-      public_api_replica = {
-        db_name = module.aurora_db_app.cluster_database_name
-        address = module.aurora_db_app.cluster_reader_endpoint
-      }
-      private_api_replica = {
+      secondary = {
         db_name = module.aurora_db_app.cluster_database_name
         address = module.aurora_db_app.cluster_reader_endpoint
       }

terraform/20-app/ecs.service.feedback-api.tf

@@ -42,12 +42,28 @@ module "ecs_service_feedback_api" {
           name  = "APP_MODE"
           value = "FEEDBACK_API"
         },
+        {
+          name  = "POSTGRES_DB"
+          value = local.aurora.app.secondary.db_name
+        },
+        {
+          name  = "POSTGRES_HOST"
+          value = local.aurora.app.secondary.address
+        },
         {
           name  = "APIENV"
-          value = "STANDALONE"
+          value = "PROD"
         },
       ],
       secrets = [
+        {
+          name      = "POSTGRES_USER"
+          valueFrom = "${local.main_db_aurora_password_secret_arn}:username::"
+        },
+        {
+          name      = "POSTGRES_PASSWORD"
+          valueFrom = "${local.main_db_aurora_password_secret_arn}:password::"
+        },
         {
           name      = "SECRET_KEY",
           valueFrom = aws_secretsmanager_secret.backend_cryptographic_signing_key.arn
@@ -113,6 +129,13 @@ module "ecs_service_feedback_api" {
       protocol    = "tcp"
       cidr_blocks = ["0.0.0.0/0"]
     }
+    db_egress = {
+      type                     = "egress"
+      from_port                = 5432
+      to_port                  = 5432
+      protocol                 = "tcp"
+      source_security_group_id = module.aurora_db_app.security_group_id
+    }
     internet_egress = {
       type        = "egress"
       from_port   = 443

terraform/20-app/ecs.service.private-api.tf

@@ -46,11 +46,11 @@ module "ecs_service_private_api" {
         },
         {
           name  = "POSTGRES_DB"
-          value = local.aurora.app.private_api_replica.db_name
+          value = local.aurora.app.secondary.db_name
         },
         {
           name  = "POSTGRES_HOST"
-          value = local.aurora.app.private_api_replica.address
+          value = local.aurora.app.secondary.address
         },
         {
           name  = "APIENV"