From b7477eab691c24f327f2bf51692d4965da9dde72 Mon Sep 17 00:00:00 2001 From: tom-ogle-moj <142220790+tom-ogle-moj@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:43:10 +0100 Subject: [PATCH] DPR2-893: Glue connection and placeholder operational datastore secret (#6670) * DPR2-893: Glue connection and placeholder operational datastore secrets for use in datahub jobs. * DPR2-893: Add data source for operational_datastore secret. * DPR2-893: Fix typo and add in initial attempt at glue connection security group rules * DPR2-893: Fix name * DPR2-893: Fix reference * DPR2-893: Fix to ports in security group * DPR2-893: try removing allow all traffic to any IP. * Revert "DPR2-893: try removing allow all traffic to any IP." This reverts commit 3c5ce9abf3c389eaf193b2de71192d53b13e775f. * DPR2-893: Remove egress allowed via security group * DPR2-893: Switch to all tcp traffic ingress allowed from same SG * DPR2-893: restrict glue connection and operational datastore secret related resources to development environment only for now. --- .../digital-prison-reporting/data.tf | 15 +++++++ .../digital-prison-reporting/locals.tf | 6 +++ .../operational_datastore.tf | 45 +++++++++++++++++++ .../digital-prison-reporting/secrets.tf | 25 +++++++++++ 4 files changed, 91 insertions(+) create mode 100644 terraform/environments/digital-prison-reporting/operational_datastore.tf diff --git a/terraform/environments/digital-prison-reporting/data.tf b/terraform/environments/digital-prison-reporting/data.tf index 0dbde46d914..89d824dc2eb 100644 --- a/terraform/environments/digital-prison-reporting/data.tf +++ b/terraform/environments/digital-prison-reporting/data.tf @@ -29,6 +29,21 @@ data "aws_secretsmanager_secret_version" "datamart" { depends_on = [aws_secretsmanager_secret.redshift] } +# Operational DataStore Secrets for use in DataHub +data "aws_secretsmanager_secret" "operational_datastore" { + count = (local.environment == "development" ? 1 : 0) + name = aws_secretsmanager_secret.operational_datastore[0].id + + depends_on = [aws_secretsmanager_secret_version.operational_datastore[0]] +} + +data "aws_secretsmanager_secret_version" "operational_datastore" { + count = (local.environment == "development" ? 1 : 0) + secret_id = data.aws_secretsmanager_secret.operational_datastore[0].id + + depends_on = [aws_secretsmanager_secret.operational_datastore[0]] +} + # AWS _IAM_ Policy data "aws_iam_policy" "rds_full_access" { diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf index d8aa3ac5dca..2b1ccee2145 100644 --- a/terraform/environments/digital-prison-reporting/locals.tf +++ b/terraform/environments/digital-prison-reporting/locals.tf @@ -323,6 +323,12 @@ locals { port = "5432" } + # Operational DataStore Secrets PlaceHolder + operational_datastore_secrets_placeholder = { + username = "placeholder" + password = "placeholder" + } + # biprws Secrets Placeholder enable_biprws_secrets = local.application_data.accounts[local.environment].biprws.enable biprws_secrets_placeholder = { diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf new file mode 100644 index 00000000000..3c8bd6a18cc --- /dev/null +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -0,0 +1,45 @@ +resource "aws_glue_connection" "glue_operational_datastore_connection" { + count = (local.environment == "development" ? 1 : 0) + name = "${local.project}-operational-datastore-connection" + connection_type = "JDBC" + + connection_properties = { + # This will be replaced by the details for the real Operational Data Store + JDBC_CONNECTION_URL = "jdbc:postgresql://dpr2-834-instance-1.cja8lnnvvipo.eu-west-2.rds.amazonaws.com:5432/postgres" + SECRET_ID = data.aws_secretsmanager_secret.operational_datastore[0].name + } + + physical_connection_requirements { + availability_zone = data.aws_subnet.private_subnets_a.availability_zone + security_group_id_list = [aws_security_group.glue_operational_datastore_connection_sg[0].id] + subnet_id = data.aws_subnet.private_subnets_a.id + } +} + +resource aws_security_group "glue_operational_datastore_connection_sg" { + count = (local.environment == "development" ? 1 : 0) + name = "${local.project}-operational-datastore-connection_sg" + description = "Security group to allow glue access to Operational Datastore via JDBC Connection" + vpc_id = data.aws_vpc.shared.id + + # This SG is attached to the Glue connection and should also be attached to the Operational Datastore RDS + # See https://docs.aws.amazon.com/glue/latest/dg/setup-vpc-for-glue-access.html + + # A self-referencing inbound rule for all TCP ports to enable AWS Glue to communicate between its components + ingress { + from_port = 0 + to_port = 65535 + protocol = "TCP" + self = true + description = "Security Group can Ingress to itself on all ports - required for Glue to communicate with itself" + } + + # Allow all traffic out + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + description = "Allow all traffic out from this Security Group" + } +} \ No newline at end of file diff --git a/terraform/environments/digital-prison-reporting/secrets.tf b/terraform/environments/digital-prison-reporting/secrets.tf index 76a6976c217..efdc8642f26 100644 --- a/terraform/environments/digital-prison-reporting/secrets.tf +++ b/terraform/environments/digital-prison-reporting/secrets.tf @@ -53,6 +53,31 @@ resource "aws_secretsmanager_secret_version" "dps" { } } +# Operational DataStore Secrets for use in DataHub +# PlaceHolder Secrets +resource "aws_secretsmanager_secret" "operational_datastore" { + count = (local.environment == "development" ? 1 : 0) + name = "external/operational_data_store" + + tags = merge( + local.all_tags, + { + Name = "external/operational_data_store" + Resource_Type = "Secrets" + } + ) +} + +resource "aws_secretsmanager_secret_version" "operational_datastore" { + count = (local.environment == "development" ? 1 : 0) + secret_id = aws_secretsmanager_secret.operational_datastore[0].id + secret_string = jsonencode(local.operational_datastore_secrets_placeholder) + + lifecycle { + ignore_changes = [secret_string,] + } +} + # Redshift Access Secrets resource "aws_secretsmanager_secret" "redshift" { name = "dpr-redshift-sqlworkbench-${local.env}"