From f1e0a45e6db17cc5702a4736717196c62149e54c Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Wed, 19 Jun 2024 17:30:39 +0100 Subject: [PATCH 01/11] DPR2-893: Glue connection and placeholder operational datastore secrets for use in datahub jobs. --- .../digital-prison-reporting/locals.tf | 6 +++++ .../operational_datastore.tf | 16 +++++++++++++ .../digital-prison-reporting/secrets.tf | 23 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 terraform/environments/digital-prison-reporting/operational_datastore.tf diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf index d8aa3ac5dca..2b1ccee2145 100644 --- a/terraform/environments/digital-prison-reporting/locals.tf +++ b/terraform/environments/digital-prison-reporting/locals.tf @@ -323,6 +323,12 @@ locals { port = "5432" } + # Operational DataStore Secrets PlaceHolder + operational_datastore_secrets_placeholder = { + username = "placeholder" + password = "placeholder" + } + # biprws Secrets Placeholder enable_biprws_secrets = local.application_data.accounts[local.environment].biprws.enable biprws_secrets_placeholder = { diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf new file mode 100644 index 00000000000..5397989ed66 --- /dev/null +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -0,0 +1,16 @@ +resource "aws_glue_connection" "glue_operational_datastore_connection" { + name = "${local.project}-operational-datastore-connection" + connection_type = "JDBC" + + connection_properties = { + # This will be replaced by the details for the real Operational Data Store + JDBC_CONNECTION_URL = "jdbc:postgresql://dpr2-834-instance-1.cja8lnnvvipo.eu-west-2.rds.amazonaws.com:5432/postgres" + SECRET_ID = data.aws_secretmanager_secret.operational_datastore.name + } + + physical_connection_requirements { + availability_zone = data.aws_subnet.private_subnets_a.availability_zone + security_group_id_list = [aws_security_group.glue_vpc_access_connection_sg[0].id] + subnet_id = data.aws_subnet.private_subnets_a.id + } +} \ No newline at end of file diff --git a/terraform/environments/digital-prison-reporting/secrets.tf b/terraform/environments/digital-prison-reporting/secrets.tf index 76a6976c217..b646ae37bd6 100644 --- a/terraform/environments/digital-prison-reporting/secrets.tf +++ b/terraform/environments/digital-prison-reporting/secrets.tf @@ -53,6 +53,29 @@ resource "aws_secretsmanager_secret_version" "dps" { } } +# Operational DataStore Secrets for use in DataHub +# PlaceHolder Secrets +resource "aws_secretsmanager_secret" "operational_datastore" { + name = "external/operational_data_store" + + tags = merge( + local.all_tags, + { + Name = "external/operational_data_store" + Resource_Type = "Secrets" + } + ) +} + +resource "aws_secretsmanager_secret_version" "operational_datastore" { + secret_id = aws_secretsmanager_secret.operational_datastore.id + secret_string = jsonencode(local.operational_datastore_secrets_placeholder) + + lifecycle { + ignore_changes = [secret_string, ] + } +} + # Redshift Access Secrets resource "aws_secretsmanager_secret" "redshift" { name = "dpr-redshift-sqlworkbench-${local.env}" From 3818167ff733b3f5ff2a1491154f3666063645dc Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Thu, 20 Jun 2024 10:05:01 +0100 Subject: [PATCH 02/11] DPR2-893: Add data source for operational_datastore secret. --- .../environments/digital-prison-reporting/data.tf | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/terraform/environments/digital-prison-reporting/data.tf b/terraform/environments/digital-prison-reporting/data.tf index 0dbde46d914..56012eaf1bf 100644 --- a/terraform/environments/digital-prison-reporting/data.tf +++ b/terraform/environments/digital-prison-reporting/data.tf @@ -29,6 +29,19 @@ data "aws_secretsmanager_secret_version" "datamart" { depends_on = [aws_secretsmanager_secret.redshift] } +# Operational DataStore Secrets for use in DataHub +data "aws_secretsmanager_secret" "operational_datastore" { + name = aws_secretsmanager_secret.operational_datastore.id + + depends_on = [aws_secretsmanager_secret_version.operational_datastore] +} + +data "aws_secretsmanager_secret_version" "operational_datastore" { + secret_id = data.aws_secretsmanager_secret.operational_datastore.id + + depends_on = [aws_secretsmanager_secret.operational_datastore] +} + # AWS _IAM_ Policy data "aws_iam_policy" "rds_full_access" { From 9ea4c39b09c7c021ace498230552148e61448388 Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Thu, 20 Jun 2024 10:58:01 +0100 Subject: [PATCH 03/11] DPR2-893: Fix typo and add in initial attempt at glue connection security group rules --- .../operational_datastore.tf | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf index 5397989ed66..f72838b2162 100644 --- a/terraform/environments/digital-prison-reporting/operational_datastore.tf +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -1,11 +1,12 @@ resource "aws_glue_connection" "glue_operational_datastore_connection" { + count = (local.environment == "development" ? 1 : 0) name = "${local.project}-operational-datastore-connection" connection_type = "JDBC" connection_properties = { # This will be replaced by the details for the real Operational Data Store JDBC_CONNECTION_URL = "jdbc:postgresql://dpr2-834-instance-1.cja8lnnvvipo.eu-west-2.rds.amazonaws.com:5432/postgres" - SECRET_ID = data.aws_secretmanager_secret.operational_datastore.name + SECRET_ID = data.aws_secretsmanager_secret.operational_datastore.name } physical_connection_requirements { @@ -13,4 +14,37 @@ resource "aws_glue_connection" "glue_operational_datastore_connection" { security_group_id_list = [aws_security_group.glue_vpc_access_connection_sg[0].id] subnet_id = data.aws_subnet.private_subnets_a.id } +} + +resource aws_security_group "glue_vpc_access_connection_sg" { + count = (local.environment == "development" ? 1 : 0) + name = "${local.project}-operational-datastore-connection_sg" + description = "Security group to allow glue access to Operational Datastore via JDBC Connection" + vpc_id = data.aws_vpc.shared.id + + # TODO Tighten these up if necessary once it works + # Allow all traffic in from this security group + ingress { + from_port = 0 + to_port = 65535 + protocol = "-1" + self = true + description = "Security Group can Ingress to itself on all ports - required by Glue" + } + + # Allow all traffic out to this security group + egress { + from_port = 0 + to_port = 65535 + protocol = "-1" + self = true + } + + # Allow all traffic out + egress { + from_port = 0 + to_port = 65535 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } } \ No newline at end of file From 2d44d69eec39571a9120aeae7851faabcdcbb425 Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Thu, 20 Jun 2024 11:08:53 +0100 Subject: [PATCH 04/11] DPR2-893: Fix name --- .../digital-prison-reporting/operational_datastore.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf index f72838b2162..fe6d157ee85 100644 --- a/terraform/environments/digital-prison-reporting/operational_datastore.tf +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -16,7 +16,7 @@ resource "aws_glue_connection" "glue_operational_datastore_connection" { } } -resource aws_security_group "glue_vpc_access_connection_sg" { +resource aws_security_group "glue_operational_datastore_connection_sg" { count = (local.environment == "development" ? 1 : 0) name = "${local.project}-operational-datastore-connection_sg" description = "Security group to allow glue access to Operational Datastore via JDBC Connection" From a2d9a2defd80f1ca5a602997d8e9c4c92dfc143a Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Thu, 20 Jun 2024 11:14:48 +0100 Subject: [PATCH 05/11] DPR2-893: Fix reference --- .../digital-prison-reporting/operational_datastore.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf index fe6d157ee85..eb791824058 100644 --- a/terraform/environments/digital-prison-reporting/operational_datastore.tf +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -11,7 +11,7 @@ resource "aws_glue_connection" "glue_operational_datastore_connection" { physical_connection_requirements { availability_zone = data.aws_subnet.private_subnets_a.availability_zone - security_group_id_list = [aws_security_group.glue_vpc_access_connection_sg[0].id] + security_group_id_list = [aws_security_group.glue_operational_datastore_connection_sg[0].id] subnet_id = data.aws_subnet.private_subnets_a.id } } From 186d32f972000bddada99ffca77951c76b99cbf5 Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Thu, 20 Jun 2024 11:24:50 +0100 Subject: [PATCH 06/11] DPR2-893: Fix to ports in security group --- .../digital-prison-reporting/operational_datastore.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf index eb791824058..f92ba8fd8e1 100644 --- a/terraform/environments/digital-prison-reporting/operational_datastore.tf +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -26,7 +26,7 @@ resource aws_security_group "glue_operational_datastore_connection_sg" { # Allow all traffic in from this security group ingress { from_port = 0 - to_port = 65535 + to_port = 0 protocol = "-1" self = true description = "Security Group can Ingress to itself on all ports - required by Glue" @@ -35,7 +35,7 @@ resource aws_security_group "glue_operational_datastore_connection_sg" { # Allow all traffic out to this security group egress { from_port = 0 - to_port = 65535 + to_port = 0 protocol = "-1" self = true } @@ -43,7 +43,7 @@ resource aws_security_group "glue_operational_datastore_connection_sg" { # Allow all traffic out egress { from_port = 0 - to_port = 65535 + to_port = 0 protocol = "-1" cidr_blocks = ["0.0.0.0/0"] } From 3c5ce9abf3c389eaf193b2de71192d53b13e775f Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Thu, 20 Jun 2024 12:44:17 +0100 Subject: [PATCH 07/11] DPR2-893: try removing allow all traffic to any IP. --- .../digital-prison-reporting/operational_datastore.tf | 8 -------- 1 file changed, 8 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf index f92ba8fd8e1..23aca433e42 100644 --- a/terraform/environments/digital-prison-reporting/operational_datastore.tf +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -39,12 +39,4 @@ resource aws_security_group "glue_operational_datastore_connection_sg" { protocol = "-1" self = true } - - # Allow all traffic out - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } } \ No newline at end of file From 6365cc7b16fe6dd0dee2e17b20bc57ad95b43a0a Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Thu, 20 Jun 2024 13:03:22 +0100 Subject: [PATCH 08/11] Revert "DPR2-893: try removing allow all traffic to any IP." This reverts commit 3c5ce9abf3c389eaf193b2de71192d53b13e775f. --- .../digital-prison-reporting/operational_datastore.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf index 23aca433e42..f92ba8fd8e1 100644 --- a/terraform/environments/digital-prison-reporting/operational_datastore.tf +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -39,4 +39,12 @@ resource aws_security_group "glue_operational_datastore_connection_sg" { protocol = "-1" self = true } + + # Allow all traffic out + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } } \ No newline at end of file From a24809dccccfa6a7eede0cfa22bb333fc47644b2 Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Thu, 20 Jun 2024 13:04:13 +0100 Subject: [PATCH 09/11] DPR2-893: Remove egress allowed via security group --- .../digital-prison-reporting/operational_datastore.tf | 9 --------- 1 file changed, 9 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf index f92ba8fd8e1..38549e512dc 100644 --- a/terraform/environments/digital-prison-reporting/operational_datastore.tf +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -22,7 +22,6 @@ resource aws_security_group "glue_operational_datastore_connection_sg" { description = "Security group to allow glue access to Operational Datastore via JDBC Connection" vpc_id = data.aws_vpc.shared.id - # TODO Tighten these up if necessary once it works # Allow all traffic in from this security group ingress { from_port = 0 @@ -32,14 +31,6 @@ resource aws_security_group "glue_operational_datastore_connection_sg" { description = "Security Group can Ingress to itself on all ports - required by Glue" } - # Allow all traffic out to this security group - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - self = true - } - # Allow all traffic out egress { from_port = 0 From be324926432f4039ba75d72ccafa84ae8e0cee07 Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Thu, 20 Jun 2024 14:48:29 +0100 Subject: [PATCH 10/11] DPR2-893: Switch to all tcp traffic ingress allowed from same SG --- .../operational_datastore.tf | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf index 38549e512dc..f5e1c84c512 100644 --- a/terraform/environments/digital-prison-reporting/operational_datastore.tf +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -22,13 +22,16 @@ resource aws_security_group "glue_operational_datastore_connection_sg" { description = "Security group to allow glue access to Operational Datastore via JDBC Connection" vpc_id = data.aws_vpc.shared.id - # Allow all traffic in from this security group + # This SG is attached to the Glue connection and should also be attached to the Operational Datastore RDS + # See https://docs.aws.amazon.com/glue/latest/dg/setup-vpc-for-glue-access.html + + # A self-referencing inbound rule for all TCP ports to enable AWS Glue to communicate between its components ingress { from_port = 0 - to_port = 0 - protocol = "-1" + to_port = 65535 + protocol = "TCP" self = true - description = "Security Group can Ingress to itself on all ports - required by Glue" + description = "Security Group can Ingress to itself on all ports - required for Glue to communicate with itself" } # Allow all traffic out @@ -37,5 +40,6 @@ resource aws_security_group "glue_operational_datastore_connection_sg" { to_port = 0 protocol = "-1" cidr_blocks = ["0.0.0.0/0"] + description = "Allow all traffic out from this Security Group" } } \ No newline at end of file From 3a78a6e6acf3048b55734cec9776d2fc2522c15f Mon Sep 17 00:00:00 2001 From: Tom Ogle Date: Thu, 20 Jun 2024 14:54:25 +0100 Subject: [PATCH 11/11] DPR2-893: restrict glue connection and operational datastore secret related resources to development environment only for now. --- .../environments/digital-prison-reporting/data.tf | 10 ++++++---- .../digital-prison-reporting/operational_datastore.tf | 2 +- .../environments/digital-prison-reporting/secrets.tf | 8 +++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/data.tf b/terraform/environments/digital-prison-reporting/data.tf index 56012eaf1bf..89d824dc2eb 100644 --- a/terraform/environments/digital-prison-reporting/data.tf +++ b/terraform/environments/digital-prison-reporting/data.tf @@ -31,15 +31,17 @@ data "aws_secretsmanager_secret_version" "datamart" { # Operational DataStore Secrets for use in DataHub data "aws_secretsmanager_secret" "operational_datastore" { - name = aws_secretsmanager_secret.operational_datastore.id + count = (local.environment == "development" ? 1 : 0) + name = aws_secretsmanager_secret.operational_datastore[0].id - depends_on = [aws_secretsmanager_secret_version.operational_datastore] + depends_on = [aws_secretsmanager_secret_version.operational_datastore[0]] } data "aws_secretsmanager_secret_version" "operational_datastore" { - secret_id = data.aws_secretsmanager_secret.operational_datastore.id + count = (local.environment == "development" ? 1 : 0) + secret_id = data.aws_secretsmanager_secret.operational_datastore[0].id - depends_on = [aws_secretsmanager_secret.operational_datastore] + depends_on = [aws_secretsmanager_secret.operational_datastore[0]] } diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf index f5e1c84c512..3c8bd6a18cc 100644 --- a/terraform/environments/digital-prison-reporting/operational_datastore.tf +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -6,7 +6,7 @@ resource "aws_glue_connection" "glue_operational_datastore_connection" { connection_properties = { # This will be replaced by the details for the real Operational Data Store JDBC_CONNECTION_URL = "jdbc:postgresql://dpr2-834-instance-1.cja8lnnvvipo.eu-west-2.rds.amazonaws.com:5432/postgres" - SECRET_ID = data.aws_secretsmanager_secret.operational_datastore.name + SECRET_ID = data.aws_secretsmanager_secret.operational_datastore[0].name } physical_connection_requirements { diff --git a/terraform/environments/digital-prison-reporting/secrets.tf b/terraform/environments/digital-prison-reporting/secrets.tf index b646ae37bd6..efdc8642f26 100644 --- a/terraform/environments/digital-prison-reporting/secrets.tf +++ b/terraform/environments/digital-prison-reporting/secrets.tf @@ -56,7 +56,8 @@ resource "aws_secretsmanager_secret_version" "dps" { # Operational DataStore Secrets for use in DataHub # PlaceHolder Secrets resource "aws_secretsmanager_secret" "operational_datastore" { - name = "external/operational_data_store" + count = (local.environment == "development" ? 1 : 0) + name = "external/operational_data_store" tags = merge( local.all_tags, @@ -68,11 +69,12 @@ resource "aws_secretsmanager_secret" "operational_datastore" { } resource "aws_secretsmanager_secret_version" "operational_datastore" { - secret_id = aws_secretsmanager_secret.operational_datastore.id + count = (local.environment == "development" ? 1 : 0) + secret_id = aws_secretsmanager_secret.operational_datastore[0].id secret_string = jsonencode(local.operational_datastore_secrets_placeholder) lifecycle { - ignore_changes = [secret_string, ] + ignore_changes = [secret_string,] } }