merge in main

terraform/environments/apex/platform_providers.tf

@@ -35,7 +35,7 @@ provider "aws" {
   alias  = "core-network-services"
   region = "eu-west-2"
   assume_role {
-    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-dns-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
+    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-log-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
   }
 }
 

terraform/environments/ccms-ebs-upgrade/platform_providers.tf

@@ -35,7 +35,7 @@ provider "aws" {
   alias  = "core-network-services"
   region = "eu-west-2"
   assume_role {
-    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-dns-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
+    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-log-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
   }
 }
 

terraform/environments/ccms-ebs/platform_providers.tf

@@ -35,7 +35,7 @@ provider "aws" {
   alias  = "core-network-services"
   region = "eu-west-2"
   assume_role {
-    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-dns-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
+    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-log-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
   }
 }
 

terraform/environments/cooker/platform_providers.tf

@@ -35,7 +35,7 @@ provider "aws" {
   alias  = "core-network-services"
   region = "eu-west-2"
   assume_role {
-    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-dns-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
+    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-log-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
   }
 }
 

terraform/environments/corporate-staff-rostering/platform_providers.tf

@@ -35,7 +35,7 @@ provider "aws" {
   alias  = "core-network-services"
   region = "eu-west-2"
   assume_role {
-    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-dns-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
+    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-log-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
   }
 }
 

terraform/environments/dacp/platform_providers.tf

@@ -35,7 +35,7 @@ provider "aws" {
   alias  = "core-network-services"
   region = "eu-west-2"
   assume_role {
-    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-dns-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
+    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-log-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
   }
 }
 

terraform/environments/data-and-insights-wepi/platform_providers.tf

@@ -35,7 +35,7 @@ provider "aws" {
   alias  = "core-network-services"
   region = "eu-west-2"
   assume_role {
-    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-dns-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
+    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-log-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
   }
 }
 

terraform/environments/data-platform-apps-and-tools/platform_providers.tf

@@ -35,7 +35,7 @@ provider "aws" {
   alias  = "core-network-services"
   region = "eu-west-2"
   assume_role {
-    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-dns-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
+    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-log-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
   }
 }
 

terraform/environments/data-platform/application_variables.auto.tfvars.json

@@ -18,10 +18,10 @@
     "production": "1.1.1"
   },
   "presigned_url_versions": {
-    "development": "1.2.4",
-    "test": "1.2.4",
-    "preproduction": "1.2.4",
-    "production": "1.2.4"
+    "development": "1.2.6",
+    "test": "1.2.6",
+    "preproduction": "1.2.6",
+    "production": "1.2.6"
   },
   "athena_load_versions": {
     "development": "1.2.1",

terraform/environments/data-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json

@@ -0,0 +1,113 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "type": "object",
+  "title": "Data Product Metadata",
+  "description": "Specification for MoJ Data Platform Data Product metadata",
+  "required": [
+    "name",
+    "description",
+    "domain",
+    "dataProductOwner",
+    "dataProductOwnerDisplayName",
+    "email",
+    "status",
+    "dpiaRequired"
+  ],
+  "properties": {
+    "name": {
+      "type": "string",
+      "description": "The name of the Data Product. Must contain only lowercase letters, numbers, and the underscore character.",
+      "pattern": "^[a-z0-9_]+$",
+      "example": "my_data_product"
+    },
+    "description": {
+      "type": "string",
+      "description": "Detailed description about what functional area this Data Product is representing, what purpose it has and business related information.",
+      "example": "this data product hold lots of useful information I want to share with those who may have use for it."
+    },
+    "domain": {
+      "type": "string",
+      "description": "The identifier of the domain this Data Product belongs to. Should be one of HQ, HMPPS, OPG, LAA, HMCTS, CICA, or Platforms",
+      "example": "HMPPS"
+    },
+    "dataProductOwner": {
+      "type": "string",
+      "description": "Data Product owner, the unique identifier of the actual user that owns, manages, and receives notifications about the Data Product. To make it technology independent it is usually the email address of the owner.",
+      "example": "[email protected]"
+    },
+    "dataProductOwnerDisplayName": {
+      "type": "string",
+      "description": "The human-readable version of dataProductOwner",
+      "example": "Jane Doe"
+    },
+    "dataProductMaintainer": {
+      "type": "string",
+      "description": "Secondary party who is able to approve DPIA access requests, but who may or may not be legally responsible for the data",
+      "example": "[email protected]"
+    },
+    "dataProductMaintainerDisplayName": {
+      "type": "string",
+      "description": "The human-readable version of dataProductMaintainer",
+      "example": "Jonny Data"
+    },
+    "email": {
+      "type": "string",
+      "description": "point of contact between consumers and maintainers of the Data Product. It could be the owner or a distribution list, but must be reliable and responsive.",
+      "example": "[email protected]"
+    },
+    "status": {
+      "type": "string",
+      "description": "this is an enum representing the status of this version of the Data Product. Allowed values are: [draft|published|retired]. This is a metadata that communicates the overall status of the Data Product but is not reflected to the actual deployment status.",
+      "enum": ["draft", "published", "retired"]
+    },
+    "dpiaRequired": {
+      "type": "boolean",
+      "description": "Bool for if a data privacy impact assessment (dpia) is required to access this data product",
+      "example": true
+    },
+    "dpiaLocation": {
+      "type": "string",
+      "description": "Data Privacy Impact Assessment (DPIA) file s3 location for this data product. Generated by data platform."
+    },
+    "retentionPeriod": {
+      "type": "integer",
+      "description": "Retention period of the data in this data product in days.",
+      "example": 3650
+    },
+    "tags": {
+      "type": "object",
+      "description": "Additional tags to add.",
+      "example": { "sandbox": true }
+    },
+    "version": {
+      "type": "string",
+      "description": "Data product version of form [major].[minor]. Generated by data platform."
+    },
+    "id": {
+      "type": "string",
+      "description": "Data product unique id. Generated by data platform.",
+      "example": "dp:civil-courts-data:v1.1"
+    },
+    "lastUpdated": {
+      "type": "string",
+      "description": "Last data upload date to this data product. Generated by data platform."
+    },
+    "creationDate": {
+      "type": "string",
+      "description": "Creation date of the data product. Generated by data platform."
+    },
+    "s3Location": {
+      "type": "string",
+      "description": "S3 path to data in this data product. Generated by data platform."
+    },
+    "rowCount": {
+      "type": "object",
+      "description": "Total row count of all tables in the data product, as a heuristic. Generated by data platform."
+    },
+    "schemas": {
+      "type": "array",
+      "description": "This contains a list of schemas or tables that are part of the data product. Generated by data platform"
+    }
+  },
+  "additionalProperties": false
+}

terraform/environments/data-platform/platform_providers.tf

@@ -35,7 +35,7 @@ provider "aws" {
   alias  = "core-network-services"
   region = "eu-west-2"
   assume_role {
-    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-dns-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
+    role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-log-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records"
   }
 }
 

terraform/environments/data-platform/s3.tf

@@ -126,9 +126,9 @@ resource "aws_s3_bucket_notification" "bucket_notification" {
 # load the json schema for data product metadata
 resource "aws_s3_object" "object" {
   bucket                 = module.s3-bucket.bucket.id
-  key                    = "data_product_metadata_spec/v1.0.0/moj_data_product_metadata_spec.json"
-  source                 = "data-product-metadata-json-schema/v1.0.0/moj_data_product_metadata_spec.json"
-  etag                   = filemd5("data-product-metadata-json-schema/v1.0.0/moj_data_product_metadata_spec.json")
+  key                    = "data_product_metadata_spec/v1.1.0/moj_data_product_metadata_spec.json"
+  source                 = "data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json"
+  etag                   = filemd5("data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json")
   acl                    = "bucket-owner-full-control"
   server_side_encryption = "AES256"
 }

terraform/environments/delius-core/locals.tf

@@ -13,4 +13,12 @@ locals {
   frontend_container_port       = 8080
 
   ldap_port = 389
+
+  delius_environments_per_account = {
+    # account = [env1, env2]
+    prod     = []
+    pre_prod = []
+    test     = []
+    dev      = ["dev", "test"]
+  }
 }

terraform/environments/delius-core/locals_development_dev1.tf

@@ -23,13 +23,14 @@ locals {
   }
 
   ldap_config_dev = {
-    name                        = try(local.ldap_config_lower_environments.name, "ldap")
-    migration_source_account_id = local.ldap_config_lower_environments.migration_source_account_id
-    migration_lambda_role       = local.ldap_config_lower_environments.migration_lambda_role
-    efs_throughput_mode         = local.ldap_config_lower_environments.efs_throughput_mode
-    efs_provisioned_throughput  = local.ldap_config_lower_environments.efs_provisioned_throughput
-    efs_backup_schedule         = "cron(0 19 * * ? *)",
-    efs_backup_retention_period = "30"
+    name                         = try(local.ldap_config_lower_environments.name, "ldap")
+    migration_source_account_id  = local.ldap_config_lower_environments.migration_source_account_id
+    migration_lambda_role        = local.ldap_config_lower_environments.migration_lambda_role
+    efs_throughput_mode          = local.ldap_config_lower_environments.efs_throughput_mode
+    efs_provisioned_throughput   = local.ldap_config_lower_environments.efs_provisioned_throughput
+    efs_backup_schedule          = "cron(0 19 * * ? *)",
+    efs_backup_retention_period  = "30"
+    efs_datasync_destination_arn = module.environment_test[0].ldap_efs_location
   }
 
   db_config_dev = {

terraform/environments/delius-core/locals_development_test.tf

@@ -23,13 +23,14 @@ locals {
   }
 
   ldap_config_test = {
-    name                        = try(local.ldap_config_lower_environments.name, "ldap")
-    migration_source_account_id = local.ldap_config_lower_environments.migration_source_account_id
-    migration_lambda_role       = local.ldap_config_lower_environments.migration_lambda_role
-    efs_throughput_mode         = local.ldap_config_lower_environments.efs_throughput_mode
-    efs_provisioned_throughput  = local.ldap_config_lower_environments.efs_provisioned_throughput
-    efs_backup_schedule         = "cron(0 19 * * ? *)",
-    efs_backup_retention_period = "30"
+    name                         = try(local.ldap_config_lower_environments.name, "ldap")
+    migration_source_account_id  = local.ldap_config_lower_environments.migration_source_account_id
+    migration_lambda_role        = local.ldap_config_lower_environments.migration_lambda_role
+    efs_throughput_mode          = local.ldap_config_lower_environments.efs_throughput_mode
+    efs_provisioned_throughput   = local.ldap_config_lower_environments.efs_provisioned_throughput
+    efs_backup_schedule          = "cron(0 19 * * ? *)",
+    efs_backup_retention_period  = "30"
+    efs_datasync_destination_arn = null
   }
 
   db_config_test = {

terraform/environments/delius-core/main_development.tf

@@ -27,6 +27,8 @@ module "environment_dev" {
 
   account_info = local.account_info
 
+  environments_in_account = local.delius_environments_per_account.dev
+
   tags = local.tags
 }
 
@@ -55,5 +57,7 @@ module "environment_test" {
 
   account_info = local.account_info
 
+  environments_in_account = local.delius_environments_per_account.dev
+
   tags = local.tags
 }

terraform/environments/delius-core/modules/environment_all_components/db_ec2.tf

@@ -75,8 +75,8 @@ resource "aws_instance" "db_ec2_primary_instance" {
     iops        = var.db_config.ebs_volumes.iops
     throughput  = var.db_config.ebs_volumes.throughput
     encrypted   = true
-    # We want to include kms_key_id here
-    tags = local.tags
+    kms_key_id  = var.db_config.ebs_volumes.kms_key_id
+    tags        = local.tags
   }
 
   dynamic "ephemeral_block_device" {

terraform/environments/delius-core/modules/environment_all_components/ldap_datasync.tf

@@ -0,0 +1,24 @@
+resource "aws_datasync_location_efs" "destination" {
+  count = var.ldap_config.efs_datasync_destination_arn != null ? 1 : 0
+  ec2_config {
+    security_group_arns = [aws_security_group.ldap_efs.arn]
+    subnet_arn          = "arn:aws:ec2:${var.account_info.region}:${var.account_info.id}:subnet/${var.account_config.private_subnet_ids[0]}"
+  }
+  efs_file_system_arn = var.ldap_config.efs_datasync_destination_arn
+}
+
+resource "aws_datasync_location_efs" "source" {
+  ec2_config {
+    security_group_arns = [aws_security_group.ldap_efs.arn]
+    subnet_arn          = "arn:aws:ec2:${var.account_info.region}:${var.account_info.id}:subnet/${var.account_config.private_subnet_ids[0]}"
+  }
+  efs_file_system_arn = aws_efs_file_system.ldap.arn
+}
+
+resource "aws_datasync_task" "ldap_refresh_task" {
+  count                    = var.ldap_config.efs_datasync_destination_arn != null ? 1 : 0
+  destination_location_arn = aws_datasync_location_efs.destination[0].arn
+  source_location_arn      = aws_datasync_location_efs.source.arn
+
+  name = "ldap-datasync-task-push-from-${var.env_name}"
+}

terraform/environments/delius-core/modules/environment_all_components/ldap_ecs.tf

@@ -270,5 +270,5 @@ resource "aws_iam_role_policy" "ecs_exec" {
 # temp log group for testing ldap
 resource "aws_cloudwatch_log_group" "ldap_test" {
   name              = "/ecs/ldap_${var.env_name}"
-  retention_in_days = 7
+  retention_in_days = 5
 }

terraform/environments/delius-core/modules/environment_all_components/ldap_efs.tf

@@ -7,7 +7,7 @@ resource "aws_efs_file_system" "ldap" {
   tags = merge(
     local.tags,
     {
-      Name = "${var.env_name}-ldap-efs"
+      Name = "ldap-efs-${var.env_name}"
     }
   )
 }
@@ -35,10 +35,21 @@ resource "aws_efs_access_point" "ldap" {
 }
 
 resource "aws_security_group" "ldap_efs" {
-  name        = "${var.env_name}-ldap-efs"
+  name        = "ldap-efs-${var.env_name}"
   description = "Allow traffic between ldap service and efs in ${var.env_name}"
   vpc_id      = var.account_info.vpc_id
-  tags        = local.tags
+  tags = merge(
+    local.tags,
+    {
+      Name = "ldap-efs-${var.env_name}"
+    }
+  )
+
+  # see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group#recreating-a-security-group
+  lifecycle {
+    create_before_destroy = true
+  }
+
 }
 
 resource "aws_security_group_rule" "efs_ingress" {

terraform/environments/delius-core/modules/environment_all_components/outputs.tf

@@ -0,0 +1,11 @@
+##
+# Output variables here
+##
+
+output "ldap_efs_location" {
+  value = aws_efs_file_system.ldap.arn
+}
+
+output "ldap_efs_security_group_id" {
+  value = aws_security_group.ldap.id
+}