From 82c9a54ec2f04937f55db5b6d0dae64b804637d5 Mon Sep 17 00:00:00 2001 From: ranbeersingh1 <43067999+ranbeersingh1@users.noreply.github.com> Date: Wed, 19 Jun 2024 15:46:39 +0100 Subject: [PATCH 01/17] Oracle application secrets for both Delius and MIS refactor --- .../components/oracle_db_shared/iam.tf | 31 +++-------- .../components/oracle_db_shared/locals.tf | 3 +- .../components/oracle_db_shared/secrets.tf | 27 ++++++++++ .../modules/delius_environment/database.tf | 52 +------------------ .../modules/delius_environment/locals.tf | 1 - 5 files changed, 36 insertions(+), 78 deletions(-) diff --git a/terraform/environments/delius-core/modules/components/oracle_db_shared/iam.tf b/terraform/environments/delius-core/modules/components/oracle_db_shared/iam.tf index 8ff1ffec7c0..ccb6f05178c 100644 --- a/terraform/environments/delius-core/modules/components/oracle_db_shared/iam.tf +++ b/terraform/environments/delius-core/modules/components/oracle_db_shared/iam.tf @@ -138,6 +138,8 @@ resource "aws_iam_policy" "ec2_access_for_ansible" { # policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" #} +# Policy document for both Oracle database DBA and application secrets + data "aws_iam_policy_document" "db_access_to_secrets_manager" { statement { sid = "DbAccessToSecretsManager" @@ -151,41 +153,20 @@ data "aws_iam_policy_document" "db_access_to_secrets_manager" { ] effect = "Allow" resources = [ - aws_secretsmanager_secret.delius_core_dba_passwords.arn - ] - } -} - -data "aws_iam_policy_document" "allow_access_to_delius_application_passwords" { - statement { - sid = "DbAccessToDeliusSecretsManager" - actions = ["secretsmanager:GetSecretValue"] - effect = "Allow" - resources = [ - "arn:aws:secretsmanager:*:${local.delius_account_id}:secret:delius-core-${var.env_name}-oracle-db-application-passwords*" + aws_secretsmanager_secret.delius_core_dba_passwords.arn, + aws_secretsmanager_secret.delius_core_application_passwords.arn, ] } } -data "aws_iam_policy_document" "combined_policy_documents" { - source_policy_documents = flatten([ - data.aws_iam_policy_document.db_access_to_secrets_manager.json, - data.aws_iam_policy_document.allow_access_to_delius_application_passwords.json - ]) -} +# Policy to allow access to both Oracle database DBA and application secrets resource "aws_iam_policy" "db_access_to_secrets_manager" { name = "${var.account_info.application_name}-${var.env_name}-${var.db_suffix}-secrets-manager-access" - policy = data.aws_iam_policy_document.combined_policy_documents.json + policy = data.aws_iam_policy_document.db_access_to_secrets_manager.json } -#resource "aws_iam_role_policy_attachment" "db_access_to_secrets_manager" { -# role = aws_iam_role.db_ec2_instance_iam_role.name -# policy_arn = aws_iam_policy.db_access_to_secrets_manager.arn -#} - - data "aws_iam_policy_document" "instance_ssm" { statement { sid = "SSMManagedSSM" diff --git a/terraform/environments/delius-core/modules/components/oracle_db_shared/locals.tf b/terraform/environments/delius-core/modules/components/oracle_db_shared/locals.tf index 6382a4553d9..5b3ff89e8ca 100644 --- a/terraform/environments/delius-core/modules/components/oracle_db_shared/locals.tf +++ b/terraform/environments/delius-core/modules/components/oracle_db_shared/locals.tf @@ -17,6 +17,8 @@ locals { delius_account_id = var.platform_vars.environment_management.account_ids[join("-", ["delius-core", var.account_info.mp_environment])] + has_mis_environment = lookup(var.environment_config, "has_mis_environment", false) + oracle_statistics_map = { "dev" = { # "target_account_id" = var.platform_vars.environment_management.account_ids["delius-core-test"] @@ -65,5 +67,4 @@ locals { oracle_backup_bucket_prefix = "${var.account_info.application_name}-${var.env_name}-oracle-${var.db_suffix}-backups" - } diff --git a/terraform/environments/delius-core/modules/components/oracle_db_shared/secrets.tf b/terraform/environments/delius-core/modules/components/oracle_db_shared/secrets.tf index 62f00a64776..e546442d8d7 100644 --- a/terraform/environments/delius-core/modules/components/oracle_db_shared/secrets.tf +++ b/terraform/environments/delius-core/modules/components/oracle_db_shared/secrets.tf @@ -1,3 +1,5 @@ +# Oracle Database DBA Secret + resource "aws_secretsmanager_secret" "delius_core_dba_passwords" { name = local.dba_secret_name description = "DBA Users Credentials" @@ -22,3 +24,28 @@ resource "aws_secretsmanager_secret_policy" "delius_core_dba_passwords" { secret_arn = aws_secretsmanager_secret.delius_core_dba_passwords.arn policy = data.aws_iam_policy_document.delius_core_dba_passwords.json } + +# Oracle Database Application Secret + +resource "aws_secretsmanager_secret" "delius_core_application_passwords" { + name = local.application_secret_name + description = "Application Users Credentials" + kms_key_id = var.account_config.kms_keys.general_shared + tags = var.tags +} + +# Allow Access To Delius Core Application Secret From MIS Primary EC2 Instance Role + +data "aws_iam_policy_document" "delius_core_application_passwords" { + count = local.has_mis_environment && var.account_info.application_name == "delius-core" ? 1 : 0 + statement { + sid = "MisAWSAccountToReadTheSecret" + effect = "Allow" + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${local.mis_account_id}:role/instance-role-delius-mis-${var.env_name}-mis-db-1"] + } + actions = ["secretsmanager:GetSecretValue"] + resources = [aws_secretsmanager_secret.delius_core_application_passwords.arn] + } +} \ No newline at end of file diff --git a/terraform/environments/delius-core/modules/delius_environment/database.tf b/terraform/environments/delius-core/modules/delius_environment/database.tf index 9d3328037b6..c1efe74662f 100644 --- a/terraform/environments/delius-core/modules/delius_environment/database.tf +++ b/terraform/environments/delius-core/modules/delius_environment/database.tf @@ -118,54 +118,4 @@ module "oracle_db_standby" { providers = { aws.core-vpc = aws.core-vpc } -} - -resource "aws_secretsmanager_secret" "delius_core_application_passwords_secret" { - count = local.has_mis_environment ? 1 : 0 - - name = local.application_secret_name - description = "Application Users Credentials" - kms_key_id = var.account_config.kms_keys.general_shared - tags = var.tags -} - -data "aws_iam_policy_document" "delius_core_application_passwords_policy_doc" { - - count = local.has_mis_environment ? 1 : 0 - statement { - sid = "MisAWSAccountToReadTheSecret" - effect = "Allow" - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${local.mis_account_id}:role/instance-role-delius-mis-${var.env_name}-mis-db-1"] - } - actions = ["secretsmanager:GetSecretValue"] - resources = [aws_secretsmanager_secret.delius_core_application_passwords_secret[count.index].arn] - } -} - -resource "aws_secretsmanager_secret_policy" "delius_core_application_passwords_pol" { - count = local.has_mis_environment ? 1 : 0 - - secret_arn = aws_secretsmanager_secret.delius_core_application_passwords_secret[count.index].arn - policy = data.aws_iam_policy_document.delius_core_application_passwords_policy_doc[count.index].json -} - -data "aws_iam_policy_document" "db_access_to_secrets_manager" { - count = local.has_mis_environment ? 1 : 0 - statement { - sid = "DbAccessToSecretsManager" - actions = [ - "secretsmanager:Describe*", - "secretsmanager:Get*", - "secretsmanager:ListSecret*", - "secretsmanager:Put*", - "secretsmanager:RestoreSecret", - "secretsmanager:Update*" - ] - effect = "Allow" - resources = [ - aws_secretsmanager_secret.delius_core_application_passwords_secret[count.index].arn - ] - } -} +} \ No newline at end of file diff --git a/terraform/environments/delius-core/modules/delius_environment/locals.tf b/terraform/environments/delius-core/modules/delius_environment/locals.tf index 0ee1201ea5e..c87dda0fae1 100644 --- a/terraform/environments/delius-core/modules/delius_environment/locals.tf +++ b/terraform/environments/delius-core/modules/delius_environment/locals.tf @@ -42,5 +42,4 @@ locals { application_secret_name = "${local.secret_prefix}-application-passwords" mis_account_id = var.platform_vars.environment_management.account_ids[join("-", ["delius-mis", var.account_info.mp_environment])] - has_mis_environment = lookup(var.environment_config, "has_mis_environment", false) } From 9bf5566cfe37a55417948d6ee54d703d7575824f Mon Sep 17 00:00:00 2001 From: Seb Date: Thu, 20 Jun 2024 14:29:02 +0100 Subject: [PATCH 02/17] gdpr api and merge api snapshots ids from ssm (#6435) * gdpr api and merge api snapshots ids from ssm --- .../delius-core/locals_development.tf | 2 +- .../delius_environment/gdpr_api_service.tf | 19 ++++++++++++++++++- .../delius_environment/merge_api_service.tf | 19 ++++++++++++++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/terraform/environments/delius-core/locals_development.tf b/terraform/environments/delius-core/locals_development.tf index b9712fdb457..7f47f581d43 100644 --- a/terraform/environments/delius-core/locals_development.tf +++ b/terraform/environments/delius-core/locals_development.tf @@ -104,7 +104,7 @@ locals { rds_engine_version = "15" rds_instance_class = "db.t3.small" rds_allocated_storage = 30 - rds_username = "postgres" + rds_username = "dbadmin" rds_port = 5432 rds_license_model = "postgresql-license" rds_deletion_protection = false diff --git a/terraform/environments/delius-core/modules/delius_environment/gdpr_api_service.tf b/terraform/environments/delius-core/modules/delius_environment/gdpr_api_service.tf index c11877c12e3..5871d6019d2 100644 --- a/terraform/environments/delius-core/modules/delius_environment/gdpr_api_service.tf +++ b/terraform/environments/delius-core/modules/delius_environment/gdpr_api_service.tf @@ -46,7 +46,7 @@ module "gdpr_api_service" { rds_backup_retention_period = var.delius_microservice_configs.gdpr_api.rds_backup_retention_period rds_backup_window = var.delius_microservice_configs.gdpr_api.rds_backup_window rds_deletion_protection = var.delius_microservice_configs.gdpr_api.rds_deletion_protection - snapshot_identifier = var.delius_microservice_configs.gdpr_api.snapshot_identifier + snapshot_identifier = data.aws_ssm_parameter.gdpr_api_snapshot_identifier.value rds_skip_final_snapshot = var.delius_microservice_configs.gdpr_api.rds_skip_final_snapshot container_vars_default = { @@ -80,3 +80,20 @@ module "gdpr_api_service" { frontend_lb_arn_suffix = aws_lb.delius_core_frontend.arn_suffix enable_platform_backups = var.enable_platform_backups } + +####################### +# GDPR API Params # +####################### + +resource "aws_ssm_parameter" "gpdr_api_snapshot_identifier" { + name = "/delius-core-${var.env_name}/gdpr-api/snapshot_id" + type = "String" + value = "DEFAULT" + lifecycle { + ignore_changes = [value] + } +} + +data "aws_ssm_parameter" "gdpr_api_snapshot_identifier" { + name = aws_ssm_parameter.gpdr_api_snapshot_identifier.name +} \ No newline at end of file diff --git a/terraform/environments/delius-core/modules/delius_environment/merge_api_service.tf b/terraform/environments/delius-core/modules/delius_environment/merge_api_service.tf index be24e3e9fac..fe80a83eff5 100644 --- a/terraform/environments/delius-core/modules/delius_environment/merge_api_service.tf +++ b/terraform/environments/delius-core/modules/delius_environment/merge_api_service.tf @@ -36,7 +36,7 @@ module "merge_api_service" { rds_username = var.delius_microservice_configs.merge_api.rds_username rds_license_model = var.delius_microservice_configs.merge_api.rds_license_model rds_deletion_protection = var.delius_microservice_configs.merge_api.rds_deletion_protection - snapshot_identifier = var.delius_microservice_configs.merge_api.snapshot_identifier + snapshot_identifier = data.aws_ssm_parameter.merge_api_snapshot_identifier.value rds_skip_final_snapshot = var.delius_microservice_configs.merge_api.rds_skip_final_snapshot maintenance_window = var.delius_microservice_configs.merge_api.maintenance_window rds_backup_retention_period = var.delius_microservice_configs.merge_api.rds_backup_retention_period @@ -79,3 +79,20 @@ module "merge_api_service" { frontend_lb_arn_suffix = aws_lb.delius_core_frontend.arn_suffix enable_platform_backups = var.enable_platform_backups } + +####################### +# Merge API Params # +####################### + +resource "aws_ssm_parameter" "merge_api_snapshot_identifier" { + name = "/delius-core-${var.env_name}/merge-api/snapshot_id" + type = "String" + value = "DEFAULT" + lifecycle { + ignore_changes = [value] + } +} + +data "aws_ssm_parameter" "merge_api_snapshot_identifier" { + name = aws_ssm_parameter.merge_api_snapshot_identifier.name +} \ No newline at end of file From bbf924ca7df3f46e0f1ad7747ed2694c8c1faa1c Mon Sep 17 00:00:00 2001 From: Robert Sweetman Date: Thu, 20 Jun 2024 14:44:08 +0100 Subject: [PATCH 03/17] move tflint-ignore string (#6677) --- terraform/modules/baseline/variables.tf | 5 +++-- terraform/modules/baseline_presets/variables.tf | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/terraform/modules/baseline/variables.tf b/terraform/modules/baseline/variables.tf index bd20adc41bb..53a95695a08 100644 --- a/terraform/modules/baseline/variables.tf +++ b/terraform/modules/baseline/variables.tf @@ -84,8 +84,9 @@ variable "bastion_linux" { # see https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/CloudWatch-Dashboard-Body-Structure.html # cannot define a type without fully defining the entire cloudwatch dashboard json structure +# tflint-ignore: terraform_typed_variables variable "cloudwatch_dashboards" { - # tflint-ignore: terraform_typed_variables + description = "map of cloudwatch dashboards where key is the dashboard name. Use widget_groups if you want baseline to work out x,y,width,height" #type = map(object({ # account_name = optional(string) # for monitoring account, limit to given account @@ -474,8 +475,8 @@ variable "efs" { default = {} } -variable "environment" { # tflint-ignore: terraform_typed_variables + variable "environment" { # Not defining 'type' as it is defined in the output of the environment module description = "Standard environmental data resources from the environment module" } diff --git a/terraform/modules/baseline_presets/variables.tf b/terraform/modules/baseline_presets/variables.tf index e5e2c6878ae..f08738401bd 100644 --- a/terraform/modules/baseline_presets/variables.tf +++ b/terraform/modules/baseline_presets/variables.tf @@ -1,11 +1,11 @@ + # tflint-ignore: terraform_typed_variables variable "environment" { - # tflint-ignore: terraform_typed_variables # Not defining 'type' as it is defined in the output of the environment module description = "Standard environmental data resources from the environment module" } +# tflint-ignore: terraform_typed_variables variable "ip_addresses" { - # tflint-ignore: terraform_typed_variables # Not defining 'type' as it is defined in the output of the ip_addresses module description = "ip address resources from the ip_address module" } From 61be4d71857dfa30b85231f7dfce22beb4f1f0c1 Mon Sep 17 00:00:00 2001 From: Robert Sweetman Date: Thu, 20 Jun 2024 15:39:35 +0100 Subject: [PATCH 04/17] fix trivy flagging s3 key in modules - static analysis should pass now (#6680) * fix trivy flagging s3 key in modules - static analysis should pass now * use terraform 0.12 interpolation --- terraform/modules/baseline/bastion_linux.tf | 2 +- terraform/modules/cost_usage_report/main.tf | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/terraform/modules/baseline/bastion_linux.tf b/terraform/modules/baseline/bastion_linux.tf index 82d49ba2a6a..0e8149b5f4e 100644 --- a/terraform/modules/baseline/bastion_linux.tf +++ b/terraform/modules/baseline/bastion_linux.tf @@ -3,7 +3,7 @@ module "bastion_linux" { count = var.bastion_linux.public_key_data != null ? 1 : 0 - source = "github.com/ministryofjustice/modernisation-platform-terraform-bastion-linux?ref=v4.2.0" + source = "github.com/ministryofjustice/modernisation-platform-terraform-bastion-linux?ref=6c4f0918a2db00ababbb40648b2ee57556ab90ab" # temp guid will be replaced with a release ref=v4.2.2? next week providers = { aws.share-host = aws.core-vpc # core-vpc-(environment) holds the networking for all accounts diff --git a/terraform/modules/cost_usage_report/main.tf b/terraform/modules/cost_usage_report/main.tf index 2e0532feee7..1dfaa1fb9c7 100644 --- a/terraform/modules/cost_usage_report/main.tf +++ b/terraform/modules/cost_usage_report/main.tf @@ -14,6 +14,7 @@ resource "aws_cur_report_definition" "cost_usage_report" { depends_on = [module.s3_bucket] #ensures bucket permissions are applied before athena bucket access validation checks run } +#tfsec:ignore:avd-aws-0132 - The bucket policy is attached to the bucket module "s3_bucket" { #checkov:skip=CKV_TF_1:Ensure Terraform module sources use a commit hash; skip as this is MoJ Repo @@ -56,7 +57,7 @@ data "aws_iam_policy_document" "cur_bucket_policy" { condition { test = "StringEquals" variable = "aws:SourceAccount" - values = ["${var.account_number}"] + values = [var.account_number] } principals { @@ -80,7 +81,7 @@ data "aws_iam_policy_document" "cur_bucket_policy" { condition { test = "StringEquals" variable = "aws:SourceAccount" - values = ["${var.account_number}"] + values = [var.account_number] } principals { From 1dc61cea67a4bda43bf5256e4dbd43e356e7b4a6 Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Thu, 20 Jun 2024 15:45:18 +0100 Subject: [PATCH 05/17] attempted fix with empty programme failed to compile, so attempting empty shell script --- scripts/echo-hello.sh | 3 +++ .../electronic-monitoring-data/modules/lambdas/main.tf | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 scripts/echo-hello.sh diff --git a/scripts/echo-hello.sh b/scripts/echo-hello.sh new file mode 100644 index 00000000000..407e3cdfda0 --- /dev/null +++ b/scripts/echo-hello.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "hello world" \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf index 58c7a88134c..81ced01dda7 100644 --- a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf @@ -7,6 +7,10 @@ resource "aws_sqs_queue" "lambda_dlq" { kms_master_key_id = aws_kms_key.lambda_env_key.id } +data "external" "latest_image_update_log_table" { + program = ["bash", "${path.root}/bash_scripts/echo_hello.sh"] # var.ecr_repo_name, var.function_name +} + resource "aws_kms_key" "lambda_env_key" { description = "KMS key for encrypting Lambda environment variables for ${var.function_name}" enable_key_rotation = true From 2eda3df1a327c2e7a5a54db52b69f5255373c4f4 Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Thu, 20 Jun 2024 15:51:00 +0100 Subject: [PATCH 06/17] misspelled bash script name --- .../electronic-monitoring-data/modules/lambdas/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf index 81ced01dda7..cfd37e9189a 100644 --- a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf @@ -8,7 +8,7 @@ resource "aws_sqs_queue" "lambda_dlq" { } data "external" "latest_image_update_log_table" { - program = ["bash", "${path.root}/bash_scripts/echo_hello.sh"] # var.ecr_repo_name, var.function_name + program = ["bash", "${path.root}/bash_scripts/echo-hello.sh"] # var.ecr_repo_name, var.function_name } resource "aws_kms_key" "lambda_env_key" { From 707a24ca1a805a4814c8e3b572b6fa330485e499 Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Thu, 20 Jun 2024 15:54:08 +0100 Subject: [PATCH 07/17] trying to find out where the scripts are saved, changed directory --- .../electronic-monitoring-data/modules/lambdas/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf index cfd37e9189a..4b793b30b23 100644 --- a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf @@ -8,7 +8,7 @@ resource "aws_sqs_queue" "lambda_dlq" { } data "external" "latest_image_update_log_table" { - program = ["bash", "${path.root}/bash_scripts/echo-hello.sh"] # var.ecr_repo_name, var.function_name + program = ["bash", "${path.root}/scripts/echo-hello.sh"] # var.ecr_repo_name, var.function_name } resource "aws_kms_key" "lambda_env_key" { From ee198ed3d85b44b7f4af956fd1ef1ba74a83f3d9 Mon Sep 17 00:00:00 2001 From: ranbeersingh1 <43067999+ranbeersingh1@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:54:48 +0100 Subject: [PATCH 08/17] Create secret policy for Delius application secret --- .../modules/components/oracle_db_shared/secrets.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/terraform/environments/delius-core/modules/components/oracle_db_shared/secrets.tf b/terraform/environments/delius-core/modules/components/oracle_db_shared/secrets.tf index e546442d8d7..0f3c8087c7d 100644 --- a/terraform/environments/delius-core/modules/components/oracle_db_shared/secrets.tf +++ b/terraform/environments/delius-core/modules/components/oracle_db_shared/secrets.tf @@ -48,4 +48,10 @@ data "aws_iam_policy_document" "delius_core_application_passwords" { actions = ["secretsmanager:GetSecretValue"] resources = [aws_secretsmanager_secret.delius_core_application_passwords.arn] } +} + +resource "aws_secretsmanager_secret_policy" "delius_core_application_passwords" { + count = local.has_mis_environment && var.account_info.application_name == "delius-core" ? 1 : 0 + secret_arn = aws_secretsmanager_secret.delius_core_application_passwords.arn + policy = data.aws_iam_policy_document.delius_core_application_passwords[count.index].json } \ No newline at end of file From 5594161aef7c6bbe6b7a167785cee38aba2a81a6 Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Thu, 20 Jun 2024 15:58:04 +0100 Subject: [PATCH 09/17] tried hardcoding the bash script --- .../electronic-monitoring-data/modules/lambdas/main.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf index 4b793b30b23..1fa7541688e 100644 --- a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf @@ -8,7 +8,9 @@ resource "aws_sqs_queue" "lambda_dlq" { } data "external" "latest_image_update_log_table" { - program = ["bash", "${path.root}/scripts/echo-hello.sh"] # var.ecr_repo_name, var.function_name + program = [ + "bash", "-c", + "echo 'hello world'"] } resource "aws_kms_key" "lambda_env_key" { From 4fe079e1cf8bb39a87f1145ef173057bc92ee8a9 Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Thu, 20 Jun 2024 16:06:39 +0100 Subject: [PATCH 10/17] attempted to recreate similar to Matt H within Lambda module, attempting a map output --- scripts/echo-hello.sh | 3 -- .../modules/lambdas/main.tf | 30 ++++++++++++------- 2 files changed, 19 insertions(+), 14 deletions(-) delete mode 100644 scripts/echo-hello.sh diff --git a/scripts/echo-hello.sh b/scripts/echo-hello.sh deleted file mode 100644 index 407e3cdfda0..00000000000 --- a/scripts/echo-hello.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "hello world" \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf index 1fa7541688e..548203991ca 100644 --- a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf @@ -7,12 +7,6 @@ resource "aws_sqs_queue" "lambda_dlq" { kms_master_key_id = aws_kms_key.lambda_env_key.id } -data "external" "latest_image_update_log_table" { - program = [ - "bash", "-c", - "echo 'hello world'"] -} - resource "aws_kms_key" "lambda_env_key" { description = "KMS key for encrypting Lambda environment variables for ${var.function_name}" enable_key_rotation = true @@ -126,18 +120,32 @@ resource "aws_cloudwatch_log_group" "lambda_cloudwatch_group" { kms_key_id = aws_kms_key.lambda_env_key.arn } +data "external" "latest_image_update_log_table" { + for_each = var.is_image ? { image = 1 } : {} # Use empty map if not fetching image + + program = [ + "bash", "-c", + "echo 'hello world'" + ] +} resource "aws_lambda_function" "this" { #checkov:skip=CKV_AWS_272:Lambda needs code-signing, see ELM-1975 - filename = var.filename + # Zip File config + filename = var.is_image ? null : var.filename + handler = var.is_image ? null : var.handler + layers = var.is_image ? null : var.layers + source_code_hash = var.is_image ? null : var.source_code_hash + runtime = var.is_image ? null : var.runtime + # Image config + image_uri = var.is_image ? data.external.latest_image_update_log_table["image"].result["latest_image_uri"] : null + package_type = var.is_image ? "Image" : null + architectures = var.is_image ? ["arm64"] : null + # Constants function_name = var.function_name role = var.role_arn - handler = var.handler - layers = var.layers - source_code_hash = var.source_code_hash timeout = var.timeout memory_size = var.memory_size - runtime = var.runtime dynamic "vpc_config" { for_each = local.use_vpc_config ? [1] : [] From 4b1289898a912e818ddcad5a6dc1ac0daff92260 Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Thu, 20 Jun 2024 16:14:18 +0100 Subject: [PATCH 11/17] attempted to output simple empty json from bash --- .../modules/lambdas/main.tf | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf index 548203991ca..74a46d2b5fe 100644 --- a/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/lambdas/main.tf @@ -7,6 +7,12 @@ resource "aws_sqs_queue" "lambda_dlq" { kms_master_key_id = aws_kms_key.lambda_env_key.id } +data "external" "latest_image_update_log_table" { + program = [ + "bash", "-c", + "echo {}"] +} + resource "aws_kms_key" "lambda_env_key" { description = "KMS key for encrypting Lambda environment variables for ${var.function_name}" enable_key_rotation = true @@ -120,32 +126,18 @@ resource "aws_cloudwatch_log_group" "lambda_cloudwatch_group" { kms_key_id = aws_kms_key.lambda_env_key.arn } -data "external" "latest_image_update_log_table" { - for_each = var.is_image ? { image = 1 } : {} # Use empty map if not fetching image - - program = [ - "bash", "-c", - "echo 'hello world'" - ] -} resource "aws_lambda_function" "this" { #checkov:skip=CKV_AWS_272:Lambda needs code-signing, see ELM-1975 - # Zip File config - filename = var.is_image ? null : var.filename - handler = var.is_image ? null : var.handler - layers = var.is_image ? null : var.layers - source_code_hash = var.is_image ? null : var.source_code_hash - runtime = var.is_image ? null : var.runtime - # Image config - image_uri = var.is_image ? data.external.latest_image_update_log_table["image"].result["latest_image_uri"] : null - package_type = var.is_image ? "Image" : null - architectures = var.is_image ? ["arm64"] : null - # Constants + filename = var.filename function_name = var.function_name role = var.role_arn + handler = var.handler + layers = var.layers + source_code_hash = var.source_code_hash timeout = var.timeout memory_size = var.memory_size + runtime = var.runtime dynamic "vpc_config" { for_each = local.use_vpc_config ? [1] : [] From 8c8e4e9a9e190dc9159a3534218af33b46820f0a Mon Sep 17 00:00:00 2001 From: Fawaz Shafaat Date: Thu, 20 Jun 2024 16:36:17 +0100 Subject: [PATCH 12/17] Made some changes for db password --- terraform/environments/cdpt-ifs/ecs.tf | 2 +- terraform/environments/cdpt-ifs/secrets.tf | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/environments/cdpt-ifs/ecs.tf b/terraform/environments/cdpt-ifs/ecs.tf index 252da8342ea..d2f4bc1d90c 100644 --- a/terraform/environments/cdpt-ifs/ecs.tf +++ b/terraform/environments/cdpt-ifs/ecs.tf @@ -133,7 +133,7 @@ resource "aws_ecs_task_definition" "ifs_task_definition" { secrets = [ { name : "RDS_PASSWORD", - valueFrom : aws_secretsmanager_secret_version.db_password.arn + valueFrom : aws_secretsmanager_secret_version.dbase_password.arn } ], } diff --git a/terraform/environments/cdpt-ifs/secrets.tf b/terraform/environments/cdpt-ifs/secrets.tf index 0e22ff06e85..f49bdec9fe6 100644 --- a/terraform/environments/cdpt-ifs/secrets.tf +++ b/terraform/environments/cdpt-ifs/secrets.tf @@ -1,5 +1,5 @@ -resource "aws_secretsmanager_secret" "db_password" { - name = "database_password" +resource "aws_secretsmanager_secret" "dbase_password" { + name = "dbase_password" } resource "random_password" "password_long" { @@ -7,7 +7,7 @@ resource "random_password" "password_long" { special = false } -resource "aws_secretsmanager_secret_version" "db_password" { - secret_id = aws_secretsmanager_secret.db_password.id +resource "aws_secretsmanager_secret_version" "dbase_password" { + secret_id = aws_secretsmanager_secret.dbase_password.id secret_string = random_password.password_long.result } From f96d5b55228df0ad525fa5233410b8acc22c0491 Mon Sep 17 00:00:00 2001 From: Fawaz Shafaat Date: Thu, 20 Jun 2024 16:38:46 +0100 Subject: [PATCH 13/17] Error with dbase password --- terraform/environments/cdpt-ifs/database.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/database.tf b/terraform/environments/cdpt-ifs/database.tf index 8a9b1b47eff..3d2c8810a22 100644 --- a/terraform/environments/cdpt-ifs/database.tf +++ b/terraform/environments/cdpt-ifs/database.tf @@ -10,7 +10,7 @@ resource "aws_db_instance" "database" { instance_class = local.application_data.accounts[local.environment].db_instance_class identifier = local.application_data.accounts[local.environment].db_instance_identifier username = local.application_data.accounts[local.environment].db_user - password = aws_secretsmanager_secret_version.db_password.secret_string + password = aws_secretsmanager_secret_version.dbase_password.secret_string vpc_security_group_ids = [aws_security_group.db.id] depends_on = [aws_security_group.db] snapshot_identifier = local.application_data.accounts[local.environment].db_snapshot_identifier From b7477eab691c24f327f2bf51692d4965da9dde72 Mon Sep 17 00:00:00 2001 From: tom-ogle-moj <142220790+tom-ogle-moj@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:43:10 +0100 Subject: [PATCH 14/17] DPR2-893: Glue connection and placeholder operational datastore secret (#6670) * DPR2-893: Glue connection and placeholder operational datastore secrets for use in datahub jobs. * DPR2-893: Add data source for operational_datastore secret. * DPR2-893: Fix typo and add in initial attempt at glue connection security group rules * DPR2-893: Fix name * DPR2-893: Fix reference * DPR2-893: Fix to ports in security group * DPR2-893: try removing allow all traffic to any IP. * Revert "DPR2-893: try removing allow all traffic to any IP." This reverts commit 3c5ce9abf3c389eaf193b2de71192d53b13e775f. * DPR2-893: Remove egress allowed via security group * DPR2-893: Switch to all tcp traffic ingress allowed from same SG * DPR2-893: restrict glue connection and operational datastore secret related resources to development environment only for now. --- .../digital-prison-reporting/data.tf | 15 +++++++ .../digital-prison-reporting/locals.tf | 6 +++ .../operational_datastore.tf | 45 +++++++++++++++++++ .../digital-prison-reporting/secrets.tf | 25 +++++++++++ 4 files changed, 91 insertions(+) create mode 100644 terraform/environments/digital-prison-reporting/operational_datastore.tf diff --git a/terraform/environments/digital-prison-reporting/data.tf b/terraform/environments/digital-prison-reporting/data.tf index 0dbde46d914..89d824dc2eb 100644 --- a/terraform/environments/digital-prison-reporting/data.tf +++ b/terraform/environments/digital-prison-reporting/data.tf @@ -29,6 +29,21 @@ data "aws_secretsmanager_secret_version" "datamart" { depends_on = [aws_secretsmanager_secret.redshift] } +# Operational DataStore Secrets for use in DataHub +data "aws_secretsmanager_secret" "operational_datastore" { + count = (local.environment == "development" ? 1 : 0) + name = aws_secretsmanager_secret.operational_datastore[0].id + + depends_on = [aws_secretsmanager_secret_version.operational_datastore[0]] +} + +data "aws_secretsmanager_secret_version" "operational_datastore" { + count = (local.environment == "development" ? 1 : 0) + secret_id = data.aws_secretsmanager_secret.operational_datastore[0].id + + depends_on = [aws_secretsmanager_secret.operational_datastore[0]] +} + # AWS _IAM_ Policy data "aws_iam_policy" "rds_full_access" { diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf index d8aa3ac5dca..2b1ccee2145 100644 --- a/terraform/environments/digital-prison-reporting/locals.tf +++ b/terraform/environments/digital-prison-reporting/locals.tf @@ -323,6 +323,12 @@ locals { port = "5432" } + # Operational DataStore Secrets PlaceHolder + operational_datastore_secrets_placeholder = { + username = "placeholder" + password = "placeholder" + } + # biprws Secrets Placeholder enable_biprws_secrets = local.application_data.accounts[local.environment].biprws.enable biprws_secrets_placeholder = { diff --git a/terraform/environments/digital-prison-reporting/operational_datastore.tf b/terraform/environments/digital-prison-reporting/operational_datastore.tf new file mode 100644 index 00000000000..3c8bd6a18cc --- /dev/null +++ b/terraform/environments/digital-prison-reporting/operational_datastore.tf @@ -0,0 +1,45 @@ +resource "aws_glue_connection" "glue_operational_datastore_connection" { + count = (local.environment == "development" ? 1 : 0) + name = "${local.project}-operational-datastore-connection" + connection_type = "JDBC" + + connection_properties = { + # This will be replaced by the details for the real Operational Data Store + JDBC_CONNECTION_URL = "jdbc:postgresql://dpr2-834-instance-1.cja8lnnvvipo.eu-west-2.rds.amazonaws.com:5432/postgres" + SECRET_ID = data.aws_secretsmanager_secret.operational_datastore[0].name + } + + physical_connection_requirements { + availability_zone = data.aws_subnet.private_subnets_a.availability_zone + security_group_id_list = [aws_security_group.glue_operational_datastore_connection_sg[0].id] + subnet_id = data.aws_subnet.private_subnets_a.id + } +} + +resource aws_security_group "glue_operational_datastore_connection_sg" { + count = (local.environment == "development" ? 1 : 0) + name = "${local.project}-operational-datastore-connection_sg" + description = "Security group to allow glue access to Operational Datastore via JDBC Connection" + vpc_id = data.aws_vpc.shared.id + + # This SG is attached to the Glue connection and should also be attached to the Operational Datastore RDS + # See https://docs.aws.amazon.com/glue/latest/dg/setup-vpc-for-glue-access.html + + # A self-referencing inbound rule for all TCP ports to enable AWS Glue to communicate between its components + ingress { + from_port = 0 + to_port = 65535 + protocol = "TCP" + self = true + description = "Security Group can Ingress to itself on all ports - required for Glue to communicate with itself" + } + + # Allow all traffic out + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + description = "Allow all traffic out from this Security Group" + } +} \ No newline at end of file diff --git a/terraform/environments/digital-prison-reporting/secrets.tf b/terraform/environments/digital-prison-reporting/secrets.tf index 76a6976c217..efdc8642f26 100644 --- a/terraform/environments/digital-prison-reporting/secrets.tf +++ b/terraform/environments/digital-prison-reporting/secrets.tf @@ -53,6 +53,31 @@ resource "aws_secretsmanager_secret_version" "dps" { } } +# Operational DataStore Secrets for use in DataHub +# PlaceHolder Secrets +resource "aws_secretsmanager_secret" "operational_datastore" { + count = (local.environment == "development" ? 1 : 0) + name = "external/operational_data_store" + + tags = merge( + local.all_tags, + { + Name = "external/operational_data_store" + Resource_Type = "Secrets" + } + ) +} + +resource "aws_secretsmanager_secret_version" "operational_datastore" { + count = (local.environment == "development" ? 1 : 0) + secret_id = aws_secretsmanager_secret.operational_datastore[0].id + secret_string = jsonencode(local.operational_datastore_secrets_placeholder) + + lifecycle { + ignore_changes = [secret_string,] + } +} + # Redshift Access Secrets resource "aws_secretsmanager_secret" "redshift" { name = "dpr-redshift-sqlworkbench-${local.env}" From 7b5accabc8d032400cb3a75822392e801d5e6161 Mon Sep 17 00:00:00 2001 From: Dominic Robinson <65237317+drobinson-moj@users.noreply.github.com> Date: Fri, 21 Jun 2024 00:01:18 +0100 Subject: [PATCH 15/17] nomis: DSOS-2862: prod weblogic and iops changes (#6685) * drop weblogic count to 6 * update iops --- .../environments/nomis/locals_production.tf | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/terraform/environments/nomis/locals_production.tf b/terraform/environments/nomis/locals_production.tf index a1feef97e1c..bd9891cfa13 100644 --- a/terraform/environments/nomis/locals_production.tf +++ b/terraform/environments/nomis/locals_production.tf @@ -86,7 +86,7 @@ locals { # ACTIVE (green deployment) prod-nomis-web-b = merge(local.ec2_autoscaling_groups.web, { autoscaling_group = merge(module.baseline_presets.ec2_autoscaling_group.default_with_ready_hook_and_warm_pool, { - desired_capacity = 8 + desired_capacity = 6 max_size = 8 # instance_refresh = { @@ -168,8 +168,8 @@ locals { "/dev/sdc" = { label = "app", size = 1000 } # /u02 }) ebs_volume_config = merge(local.ec2_instances.db.ebs_volume_config, { - data = { total_size = 4000, iops = 12000, throughput = 750 } - flash = { total_size = 1000, iops = 5000, throughput = 500 } + data = { total_size = 4000, iops = 9000, throughput = 250 } + flash = { total_size = 1000, iops = 3000, throughput = 250 } }) instance = merge(local.ec2_instances.db.instance, { disable_api_termination = true @@ -200,8 +200,8 @@ locals { "/dev/sdc" = { label = "app", size = 500 } }) ebs_volume_config = merge(local.ec2_instances.db.ebs_volume_config, { - data = { total_size = 4000, iops = 12000, throughput = 750 } - flash = { total_size = 1000, iops = 5000, throughput = 500 } + data = { total_size = 4000, iops = 9000, throughput = 250 } + flash = { total_size = 1000, iops = 3000, throughput = 125 } }) instance = merge(local.ec2_instances.db.instance, { disable_api_termination = true @@ -233,8 +233,8 @@ locals { "/dev/sdc" = { label = "app", size = 1000 } # /u02 }) ebs_volume_config = merge(local.ec2_instances.db.ebs_volume_config, { - data = { total_size = 6000, iops = 12000, throughput = 750 } - flash = { total_size = 1000, iops = 5000, throughput = 500 } + data = { total_size = 6000, iops = 9000, throughput = 250 } + flash = { total_size = 1000, iops = 3000, throughput = 250 } }) instance = merge(local.ec2_instances.db.instance, { disable_api_termination = true @@ -267,8 +267,10 @@ locals { "/dev/sdc" = { label = "app", size = 500 } }) ebs_volume_config = merge(local.ec2_instances.db.ebs_volume_config, { - data = { total_size = 6000, iops = 12000, throughput = 750 } - flash = { total_size = 1000, iops = 5000, throughput = 500 } + data = { total_size = 6000, iops = 3000, throughput = 125 } + flash = { total_size = 1000, iops = 3000, throughput = 125 } + # data = { total_size = 6000, iops = 9000, throughput = 250 } # replace above with this on failover + # flash = { total_size = 1000, iops = 3000, throughput = 250 } # replace above with this on failover }) instance = merge(local.ec2_instances.db.instance, { disable_api_termination = true From e10e8c30be167bcd6d15ba122ba53bdc2efaa79f Mon Sep 17 00:00:00 2001 From: Fawaz Shafaat Date: Fri, 21 Jun 2024 09:46:26 +0100 Subject: [PATCH 16/17] Ami changed --- terraform/environments/cdpt-ifs/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/cdpt-ifs/application_variables.json b/terraform/environments/cdpt-ifs/application_variables.json index e787aeafec0..6d5b6629295 100644 --- a/terraform/environments/cdpt-ifs/application_variables.json +++ b/terraform/environments/cdpt-ifs/application_variables.json @@ -4,7 +4,7 @@ "environment_name": "development", "container_port": 80, "client_id": "7ee6af8d-ea3c-4349-8765-644f2a1edf3b", - "ami_image_id": "ami-0bb2bf9a00240bf36", + "ami_image_id": "ami-084d79c0ad854f80b", "instance_type": "t3.xlarge", "app_count": 1, "ec2_desired_capacity": 1, From 5017e3f6677753ae56fd5839d59fc40c86dd913d Mon Sep 17 00:00:00 2001 From: pavmoj <142988272+pavmoj@users.noreply.github.com> Date: Fri, 21 Jun 2024 10:11:26 +0100 Subject: [PATCH 17/17] Update SSL policy (#6688) --- terraform/environments/planetfm/locals_preproduction.tf | 2 +- terraform/environments/planetfm/locals_production.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/planetfm/locals_preproduction.tf b/terraform/environments/planetfm/locals_preproduction.tf index 6dc77ef4186..66705a6ac86 100644 --- a/terraform/environments/planetfm/locals_preproduction.tf +++ b/terraform/environments/planetfm/locals_preproduction.tf @@ -192,7 +192,7 @@ locals { certificate_names_or_arns = ["planetfm_wildcard_cert"] port = 443 protocol = "HTTPS" - ssl_policy = "ELBSecurityPolicy-2016-08" + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" default_action = { type = "fixed-response" diff --git a/terraform/environments/planetfm/locals_production.tf b/terraform/environments/planetfm/locals_production.tf index 885665534c7..8be0dc1c4b6 100644 --- a/terraform/environments/planetfm/locals_production.tf +++ b/terraform/environments/planetfm/locals_production.tf @@ -346,7 +346,7 @@ locals { certificate_names_or_arns = ["planetfm_wildcard_cert"] port = 443 protocol = "HTTPS" - ssl_policy = "ELBSecurityPolicy-2016-08" + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" default_action = { type = "fixed-response"