DPR2-893: Glue connection and placeholder operational datastore secret (

#6670)

* DPR2-893: Glue connection and placeholder operational datastore secrets for use in datahub jobs.

* DPR2-893: Add data source for operational_datastore secret.

* DPR2-893: Fix typo and add in initial attempt at glue connection security group rules

* DPR2-893: Fix name

* DPR2-893: Fix reference

* DPR2-893: Fix to ports in security group

* DPR2-893: try removing allow all traffic to any IP.

* Revert "DPR2-893: try removing allow all traffic to any IP."

This reverts commit 3c5ce9a.

* DPR2-893: Remove egress allowed via security group

* DPR2-893: Switch to all tcp traffic ingress allowed from same SG

* DPR2-893: restrict glue connection and operational datastore secret related resources to development environment only for now.

terraform/environments/digital-prison-reporting/data.tf

@@ -29,6 +29,21 @@ data "aws_secretsmanager_secret_version" "datamart" {
   depends_on = [aws_secretsmanager_secret.redshift]
 }
 
+# Operational DataStore Secrets for use in DataHub
+data "aws_secretsmanager_secret" "operational_datastore" {
+  count = (local.environment == "development" ? 1 : 0)
+  name  = aws_secretsmanager_secret.operational_datastore[0].id
+
+  depends_on = [aws_secretsmanager_secret_version.operational_datastore[0]]
+}
+
+data "aws_secretsmanager_secret_version" "operational_datastore" {
+  count     = (local.environment == "development" ? 1 : 0)
+  secret_id = data.aws_secretsmanager_secret.operational_datastore[0].id
+
+  depends_on = [aws_secretsmanager_secret.operational_datastore[0]]
+}
+
 
 # AWS _IAM_ Policy
 data "aws_iam_policy" "rds_full_access" {

terraform/environments/digital-prison-reporting/locals.tf

@@ -323,6 +323,12 @@ locals {
     port     = "5432"
   }
 
+  # Operational DataStore Secrets PlaceHolder
+  operational_datastore_secrets_placeholder = {
+    username = "placeholder"
+    password = "placeholder"
+  }
+
   # biprws Secrets Placeholder
   enable_biprws_secrets = local.application_data.accounts[local.environment].biprws.enable
   biprws_secrets_placeholder = {

terraform/environments/digital-prison-reporting/operational_datastore.tf

@@ -0,0 +1,45 @@
+resource "aws_glue_connection" "glue_operational_datastore_connection" {
+  count           = (local.environment == "development" ? 1 : 0)
+  name            = "${local.project}-operational-datastore-connection"
+  connection_type = "JDBC"
+
+  connection_properties = {
+    # This will be replaced by the details for the real Operational Data Store
+    JDBC_CONNECTION_URL = "jdbc:postgresql://dpr2-834-instance-1.cja8lnnvvipo.eu-west-2.rds.amazonaws.com:5432/postgres"
+    SECRET_ID           = data.aws_secretsmanager_secret.operational_datastore[0].name
+  }
+
+  physical_connection_requirements {
+    availability_zone = data.aws_subnet.private_subnets_a.availability_zone
+    security_group_id_list = [aws_security_group.glue_operational_datastore_connection_sg[0].id]
+    subnet_id         = data.aws_subnet.private_subnets_a.id
+  }
+}
+
+resource aws_security_group "glue_operational_datastore_connection_sg" {
+  count       = (local.environment == "development" ? 1 : 0)
+  name        = "${local.project}-operational-datastore-connection_sg"
+  description = "Security group to allow glue access to Operational Datastore via JDBC Connection"
+  vpc_id      = data.aws_vpc.shared.id
+
+  # This SG is attached to the Glue connection and should also be attached to the Operational Datastore RDS
+  # See https://docs.aws.amazon.com/glue/latest/dg/setup-vpc-for-glue-access.html
+
+  # A self-referencing inbound rule for all TCP ports to enable AWS Glue to communicate between its components
+  ingress {
+    from_port = 0
+    to_port   = 65535
+    protocol  = "TCP"
+    self      = true
+    description = "Security Group can Ingress to itself on all ports - required for Glue to communicate with itself"
+  }
+
+  # Allow all traffic out
+  egress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "Allow all traffic out from this Security Group"
+  }
+}

terraform/environments/digital-prison-reporting/secrets.tf

@@ -53,6 +53,31 @@ resource "aws_secretsmanager_secret_version" "dps" {
   }
 }
 
+# Operational DataStore Secrets for use in DataHub
+# PlaceHolder Secrets
+resource "aws_secretsmanager_secret" "operational_datastore" {
+  count = (local.environment == "development" ? 1 : 0)
+  name  = "external/operational_data_store"
+
+  tags = merge(
+    local.all_tags,
+    {
+      Name          = "external/operational_data_store"
+      Resource_Type = "Secrets"
+    }
+  )
+}
+
+resource "aws_secretsmanager_secret_version" "operational_datastore" {
+  count     = (local.environment == "development" ? 1 : 0)
+  secret_id = aws_secretsmanager_secret.operational_datastore[0].id
+  secret_string = jsonencode(local.operational_datastore_secrets_placeholder)
+
+  lifecycle {
+    ignore_changes = [secret_string,]
+  }
+}
+
 # Redshift Access Secrets
 resource "aws_secretsmanager_secret" "redshift" {
   name = "dpr-redshift-sqlworkbench-${local.env}"