From 1ab576ef3b4e7ec82e0b7da669cea85e64b3150e Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Thu, 13 Jun 2024 10:18:24 +0000 Subject: [PATCH 01/46] add a standard s3 bucket with short versioning policies for athena query result default location, named also by the default --- .../electronic-monitoring-data/s3_main.tf | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/terraform/environments/electronic-monitoring-data/s3_main.tf b/terraform/environments/electronic-monitoring-data/s3_main.tf index 7dee0b6e27a..797eb7738e5 100644 --- a/terraform/environments/electronic-monitoring-data/s3_main.tf +++ b/terraform/environments/electronic-monitoring-data/s3_main.tf @@ -79,4 +79,75 @@ resource "aws_s3_bucket_notification" "send_metadata_to_ap" { } depends_on = [aws_lambda_permission.send_metadata_to_ap] -} \ No newline at end of file +} + +# ---------------------------------- +# Athena Query result storage bucket +# ---------------------------------- +module "athena-s3-bucket" { + source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=cadab51" + + bucket_name = join("-", ["aws-athena-query-results", local.env_account_id, local.region]) + versioning_enabled = true + + # to disable ACLs in preference of BucketOwnership controls as per https://aws.amazon.com/blogs/aws/heads-up-amazon-s3-security-changes-are-coming-in-april-of-2023/ set: + ownership_controls = "BucketOwnerEnforced" + acl = "private" + + # Refer to the below section "Replication" before enabling replication + replication_enabled = false + # Below variable and providers configuration is only relevant if 'replication_enabled' is set to true + # replication_region = "eu-west-2" + providers = { + # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the + # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation + # Platform team to add a new provider for the additional Region. + # Leave this provider block in even if you are not using replication + aws.bucket-replication = aws + } + + lifecycle_rule = [ + { + id = "main" + enabled = "Enabled" + prefix = "" + + tags = { + rule = "log" + autoclean = "true" + } + + transition = [ + { + days = 14 + storage_class = "STANDARD_IA" + }, { + days = 365 + storage_class = "GLACIER" + } + ] + + expiration = { + days = 14 + } + + noncurrent_version_transition = [ + { + days = 14 + storage_class = "STANDARD_IA" + }, { + days = 365 + storage_class = "GLACIER" + } + ] + + noncurrent_version_expiration = { + days = 14 + } + } + ] + + version_enabled = true + + tags = local.tags +} From 9b6a0a3c31af967c90c08016e6eaa6f937b91930 Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Thu, 13 Jun 2024 14:28:44 +0000 Subject: [PATCH 02/46] removed duplicate variable --- terraform/environments/electronic-monitoring-data/s3_main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/terraform/environments/electronic-monitoring-data/s3_main.tf b/terraform/environments/electronic-monitoring-data/s3_main.tf index 797eb7738e5..feea74ccbf9 100644 --- a/terraform/environments/electronic-monitoring-data/s3_main.tf +++ b/terraform/environments/electronic-monitoring-data/s3_main.tf @@ -147,7 +147,5 @@ module "athena-s3-bucket" { } ] - version_enabled = true - tags = local.tags } From bd2833f8a652111614f8a12713b19729daa4399e Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Thu, 13 Jun 2024 14:34:29 +0000 Subject: [PATCH 03/46] hard coded region to pass plan failure --- terraform/environments/electronic-monitoring-data/s3_main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/electronic-monitoring-data/s3_main.tf b/terraform/environments/electronic-monitoring-data/s3_main.tf index feea74ccbf9..dfe598d0c25 100644 --- a/terraform/environments/electronic-monitoring-data/s3_main.tf +++ b/terraform/environments/electronic-monitoring-data/s3_main.tf @@ -87,7 +87,7 @@ resource "aws_s3_bucket_notification" "send_metadata_to_ap" { module "athena-s3-bucket" { source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=cadab51" - bucket_name = join("-", ["aws-athena-query-results", local.env_account_id, local.region]) + bucket_name = join("-", ["aws-athena-query-results", local.env_account_id, "eu-west-2"]) versioning_enabled = true # to disable ACLs in preference of BucketOwnership controls as per https://aws.amazon.com/blogs/aws/heads-up-amazon-s3-security-changes-are-coming-in-april-of-2023/ set: From 588e7fb863a32843204ef4b8c3f62f36ce8be879 Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Fri, 14 Jun 2024 09:01:41 +0000 Subject: [PATCH 04/46] fixed days to STANDARD_IA to be a minimum of 30 as log failures state. --- terraform/environments/electronic-monitoring-data/s3_main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/electronic-monitoring-data/s3_main.tf b/terraform/environments/electronic-monitoring-data/s3_main.tf index dfe598d0c25..c8a3803ac96 100644 --- a/terraform/environments/electronic-monitoring-data/s3_main.tf +++ b/terraform/environments/electronic-monitoring-data/s3_main.tf @@ -119,7 +119,7 @@ module "athena-s3-bucket" { transition = [ { - days = 14 + days = 30 storage_class = "STANDARD_IA" }, { days = 365 @@ -133,7 +133,7 @@ module "athena-s3-bucket" { noncurrent_version_transition = [ { - days = 14 + days = 30 storage_class = "STANDARD_IA" }, { days = 365 From 2e32ce57c4f90880d237eeff497f45bfffce1432 Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Fri, 14 Jun 2024 09:13:31 +0000 Subject: [PATCH 05/46] fixed transition period expiration needing to be greater than days --- terraform/environments/electronic-monitoring-data/s3_main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/electronic-monitoring-data/s3_main.tf b/terraform/environments/electronic-monitoring-data/s3_main.tf index c8a3803ac96..10409647647 100644 --- a/terraform/environments/electronic-monitoring-data/s3_main.tf +++ b/terraform/environments/electronic-monitoring-data/s3_main.tf @@ -128,7 +128,7 @@ module "athena-s3-bucket" { ] expiration = { - days = 14 + days = 31 } noncurrent_version_transition = [ @@ -142,7 +142,7 @@ module "athena-s3-bucket" { ] noncurrent_version_expiration = { - days = 14 + days = 31 } } ] From 3cece589d9bf2fda825eeaa6f64414966840a0d5 Mon Sep 17 00:00:00 2001 From: Luke Williams Date: Fri, 14 Jun 2024 09:30:37 +0000 Subject: [PATCH 06/46] changed the days around again to fix errors --- .../environments/electronic-monitoring-data/s3_main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/environments/electronic-monitoring-data/s3_main.tf b/terraform/environments/electronic-monitoring-data/s3_main.tf index 10409647647..84dabda8fec 100644 --- a/terraform/environments/electronic-monitoring-data/s3_main.tf +++ b/terraform/environments/electronic-monitoring-data/s3_main.tf @@ -122,13 +122,13 @@ module "athena-s3-bucket" { days = 30 storage_class = "STANDARD_IA" }, { - days = 365 + days = 90 storage_class = "GLACIER" } ] expiration = { - days = 31 + days = 365 } noncurrent_version_transition = [ @@ -136,13 +136,13 @@ module "athena-s3-bucket" { days = 30 storage_class = "STANDARD_IA" }, { - days = 365 + days = 90 storage_class = "GLACIER" } ] noncurrent_version_expiration = { - days = 31 + days = 365 } } ] From b3dad5faa37176e5bba3340d8e2b7befc071962e Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 11:13:31 +0100 Subject: [PATCH 07/46] LAWS-3669 Set instance_warmup_period for cap provider --- terraform/environments/apex/application_variables.json | 1 + terraform/environments/apex/ecs.tf | 1 + terraform/environments/apex/modules/ecs/main.tf | 1 + terraform/environments/apex/modules/ecs/variables.tf | 5 +++++ 4 files changed, 8 insertions(+) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index b87ecaf2d6f..eaf5ade4723 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -28,6 +28,7 @@ "appscaling_max_capacity": 2, "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, + "ec2_instance_warmup_period": 300, "container_memory_allocation": 1000, "region": "eu-west-2", "docker_image_tag": "development", diff --git a/terraform/environments/apex/ecs.tf b/terraform/environments/apex/ecs.tf index 5c742519db3..66376cc4f03 100644 --- a/terraform/environments/apex/ecs.tf +++ b/terraform/environments/apex/ecs.tf @@ -31,6 +31,7 @@ module "apex-ecs" { ecs_scaling_cpu_threshold = local.application_data.accounts[local.environment].ecs_scaling_cpu_threshold ecs_scaling_mem_threshold = local.application_data.accounts[local.environment].ecs_scaling_mem_threshold ecs_target_capacity = local.ecs_target_capacity + ec2_instance_warmup_period = local.application_data.accounts[local.environment].ec2_instance_warmup_period log_group_kms_key = aws_kms_key.cloudwatch_logs_key.arn environment = local.environment diff --git a/terraform/environments/apex/modules/ecs/main.tf b/terraform/environments/apex/modules/ecs/main.tf index 40d14715611..cca23a2f09d 100644 --- a/terraform/environments/apex/modules/ecs/main.tf +++ b/terraform/environments/apex/modules/ecs/main.tf @@ -531,6 +531,7 @@ resource "aws_ecs_capacity_provider" "apex" { # minimum_scaling_step_size = 1 status = "ENABLED" target_capacity = var.ecs_target_capacity + instance_warmup_period = var.ec2_instance_warmup_period } managed_draining = "ENABLED" } diff --git a/terraform/environments/apex/modules/ecs/variables.tf b/terraform/environments/apex/modules/ecs/variables.tf index 696ef0dd13e..090c588e1b8 100644 --- a/terraform/environments/apex/modules/ecs/variables.tf +++ b/terraform/environments/apex/modules/ecs/variables.tf @@ -142,6 +142,11 @@ variable "ecs_target_capacity" { description = "The target value for the CloudWatch metric used in the Amazon ECS-managed target tracking scaling policy. For example, a value of 100 will result in the Amazon EC2 instances in your Auto Scaling group being completely utilized." } +variable "ec2_instance_warmup_period" { + type = string + description = "Period of time, in seconds, after a newly launched Amazon EC2 instance can contribute to CloudWatch metrics for Auto Scaling group for the Capacity Provider." +} + variable "lb_tg_arn" { type = string description = "Load balancer target group ARN used by ECS service" From dc8c223db520398e2bbd9ca18c23c14f3199a04a Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 11:21:46 +0100 Subject: [PATCH 08/46] LAWS-3669 Trigger new task rotation --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index eaf5ade4723..45818e73e2c 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -29,7 +29,7 @@ "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, - "container_memory_allocation": 1000, + "container_memory_allocation": 1024, "region": "eu-west-2", "docker_image_tag": "development", "app_debug_enabled": "false", From 1dec1338dfbb623e1cfcbcfcfcb27cb31537d177 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 11:25:21 +0100 Subject: [PATCH 09/46] LAWS-3669 Set higher max task and ec2 for scaling for new deployment --- terraform/environments/apex/application_variables.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 45818e73e2c..9636b65599e 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -19,13 +19,13 @@ "instance_type": "t3a.medium", "key_name": "", "ec2_desired_capacity": 1, - "ec2_max_size": 2, + "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", "network_mode": "bridge", "app_count": 1, "appscaling_min_capacity": 1, - "appscaling_max_capacity": 2, + "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, From f7eda69772aa86f00e840d03dfe7487e676823d6 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 11:28:45 +0100 Subject: [PATCH 10/46] LAWS-3669 Increase task memory allocation to ensure one instance on task --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 9636b65599e..32e7bf83281 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -29,7 +29,7 @@ "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, - "container_memory_allocation": 1024, + "container_memory_allocation": 2000, "region": "eu-west-2", "docker_image_tag": "development", "app_debug_enabled": "false", From 961577c8141e03914d12d03c991c7a99fbc71171 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 11:35:45 +0100 Subject: [PATCH 11/46] LAWS-3669 Increase task memory allocation to ensure one instance on task 2 --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 32e7bf83281..bb0d44d97ac 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -29,7 +29,7 @@ "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, - "container_memory_allocation": 2000, + "container_memory_allocation": 2024, "region": "eu-west-2", "docker_image_tag": "development", "app_debug_enabled": "false", From 8d80988a37b3a9571d83dc6885a13eab115396de Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 12:31:20 +0100 Subject: [PATCH 12/46] Revery testing changes --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index bb0d44d97ac..9d72b30c94a 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -29,7 +29,7 @@ "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, - "container_memory_allocation": 2024, + "container_memory_allocation": 1000, "region": "eu-west-2", "docker_image_tag": "development", "app_debug_enabled": "false", From a4603f20cc7150cd472081d2f8bb26796c25d30b Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 12:35:47 +0100 Subject: [PATCH 13/46] Revery testing changes 2 --- terraform/environments/apex/application_variables.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 9d72b30c94a..eaf5ade4723 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -19,13 +19,13 @@ "instance_type": "t3a.medium", "key_name": "", "ec2_desired_capacity": 1, - "ec2_max_size": 5, + "ec2_max_size": 2, "ec2_min_size": 1, "task_definition_volume": "upload_volume", "network_mode": "bridge", "app_count": 1, "appscaling_min_capacity": 1, - "appscaling_max_capacity": 5, + "appscaling_max_capacity": 2, "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, From 82fe73ec4eb70629f4af15b21f33eedbbe6dae7e Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 12:52:52 +0100 Subject: [PATCH 14/46] Testing new deployment again --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index eaf5ade4723..45818e73e2c 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -29,7 +29,7 @@ "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, - "container_memory_allocation": 1000, + "container_memory_allocation": 1024, "region": "eu-west-2", "docker_image_tag": "development", "app_debug_enabled": "false", From 39cbd0b109fedbc49b9766cd1c1f80f7b5959c0c Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 12:58:27 +0100 Subject: [PATCH 15/46] Testing new deployment again 2 --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 45818e73e2c..eaf5ade4723 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -29,7 +29,7 @@ "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, - "container_memory_allocation": 1024, + "container_memory_allocation": 1000, "region": "eu-west-2", "docker_image_tag": "development", "app_debug_enabled": "false", From d8489394beabceeed19071857c6a2e3d92f74eb0 Mon Sep 17 00:00:00 2001 From: Madhu Kadiri Date: Fri, 14 Jun 2024 14:55:14 +0100 Subject: [PATCH 16/46] RDS_Table_D_Comments_Transformation_Scripts_Added --- .../D_Comments_V2_DDL.sql | 59 +++++++++++++++++++ .../D_Comments_V2_Insert_Into_Select.sql | 27 +++++++++ .../dms_g4s_cap_dw_task_transformations.json | 50 ---------------- 3 files changed, 86 insertions(+), 50 deletions(-) create mode 100644 terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_DDL.sql create mode 100644 terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql delete mode 100644 terraform/environments/electronic-monitoring-data/modules/dms/dms_g4s_cap_dw_task_transformations.json diff --git a/terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_DDL.sql b/terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_DDL.sql new file mode 100644 index 00000000000..4919f587c5a --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_DDL.sql @@ -0,0 +1,59 @@ +SET ANSI_NULLS ON +GO +SET QUOTED_IDENTIFIER ON +GO +CREATE TABLE [dbo].[D_Comments_V2]( + [CommentSID] [int] IDENTITY(1,1) NOT NULL, + [VisitID] [int] NULL, + [ActivityID] [uniqueidentifier] NULL, + [Comments] [varchar](4200) NULL, + [CommentType] [varchar](50) NULL +) ON [PRIMARY] +GO +CREATE CLUSTERED INDEX [PK_D_Comments_V2] ON [dbo].[D_Comments_V2] +( + [CommentSID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO +SET ANSI_PADDING ON +GO +CREATE NONCLUSTERED INDEX [I1_D_Comments_V2] ON [dbo].[D_Comments_V2] +( + [VisitID] ASC, + [CommentType] ASC +) +INCLUDE([CommentSID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO +SET ANSI_PADDING ON +GO +CREATE NONCLUSTERED INDEX [I2_D_Comments_V2] ON [dbo].[D_Comments_V2] +( + [CommentType] ASC +) +INCLUDE([CommentSID],[ActivityID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO +CREATE NONCLUSTERED INDEX [I3_D_Comments_V2] ON [dbo].[D_Comments_V2] +( + [VisitID] ASC, + [ActivityID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO +SET ANSI_PADDING ON +GO +CREATE NONCLUSTERED INDEX [I4_D_Comments_V2] ON [dbo].[D_Comments_V2] +( + [ActivityID] ASC, + [CommentType] ASC +) +INCLUDE([CommentSID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +GO +EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'CommentSID' +GO +EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'VisitID' +GO +EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'ActivityID' +GO +EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Free text comments summarising either a phone call or a visit adding detail that isnt captured else where ' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'Comments' +GO +EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'identifies if the comment belongs to either a phone call or a visit' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'CommentType' +GO diff --git a/terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql b/terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql new file mode 100644 index 00000000000..fec30bc9475 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql @@ -0,0 +1,27 @@ +SET IDENTITY_INSERT g4s_cap_dw.dbo.D_Comments_V2 ON +; + +INSERT INTO g4s_cap_dw.dbo.D_Comments_V2 (CommentSID, VisitID, ActivityID, Comments, CommentType) + SELECT CommentSID, VisitID, ActivityID, trim(replace(Comments, char(141), '')) AS Comments, CommentType + FROM g4s_cap_dw.dbo.D_Comments +; + +SET IDENTITY_INSERT g4s_cap_dw.dbo.D_Comments_V2 OFF; + + +-- TESTING QUERIES -- +-- +-- SELECT COUNT(*) FROM g4s_cap_dw.dbo.D_Comments_V2; -- 49695569 +-- SELECT COUNT(*) FROM g4s_cap_dw.dbo.D_Comments; -- 49695569 +-- + + +-- +-- SELECT 'D_Comments' AS TableName, Comments +-- FROM g4s_cap_dw.dbo.D_Comments +-- WHERE CommentSID = 26837791 +-- UNION +-- SELECT 'D_Comments_V2' AS TableName, Comments +-- FROM g4s_cap_dw.dbo.D_Comments_V2 +-- WHERE CommentSID = 26837791 +-- \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/modules/dms/dms_g4s_cap_dw_task_transformations.json b/terraform/environments/electronic-monitoring-data/modules/dms/dms_g4s_cap_dw_task_transformations.json deleted file mode 100644 index 2ec59045ccb..00000000000 --- a/terraform/environments/electronic-monitoring-data/modules/dms/dms_g4s_cap_dw_task_transformations.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "rules": [ - { - "rule-type": "selection", - "rule-id": "01", - "rule-name": "select-table", - "object-locator": { - "schema-name": "dbo", - "table-name": "D_Comments" - }, - "rule-action": "include", - "load-order": "1", - "filters": [], - "parallel-load": null, - "isAutoSegmentationChecked": false - }, - { - "rule-type": "transformation", - "rule-id": "11", - "rule-name": "add_column_011", - "rule-target": "column", - "object-locator": { - "schema-name": "dbo", - "table-name": "D_Comments" - }, - "rule-action": "add-column", - "expression": "trim(replace($Comments, char(141), ''))", - "value": "Comments_V2", - "data-type": { - "type": "string", - "length": 4200 - } - }, - { - "rule-type": "transformation", - "rule-id": "12", - "rule-name": "remove_column_012", - "rule-target": "column", - "object-locator": { - "schema-name": "dbo", - "table-name": "D_Comments", - "column-name": "Comments" - }, - "parallel-load": null, - "rule-action": "remove-column", - "value": null, - "old-value": null - } - ] -} From c7cecce0bf53f4da352b3d37eb689429ce29d00e Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 16:12:25 +0100 Subject: [PATCH 17/46] LAWS-3669 Add cooldown for scaling of ECS tasks --- terraform/environments/apex/modules/ecs/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terraform/environments/apex/modules/ecs/main.tf b/terraform/environments/apex/modules/ecs/main.tf index cca23a2f09d..16e4958fbdf 100644 --- a/terraform/environments/apex/modules/ecs/main.tf +++ b/terraform/environments/apex/modules/ecs/main.tf @@ -502,6 +502,8 @@ resource "aws_appautoscaling_policy" "ecs_target_cpu" { predefined_metric_type = "ECSServiceAverageCPUUtilization" } target_value = var.ecs_scaling_cpu_threshold + scale_in_cooldown = 60 + scale_out_cooldown = 60 } } @@ -516,6 +518,8 @@ resource "aws_appautoscaling_policy" "ecs_target_memory" { predefined_metric_type = "ECSServiceAverageMemoryUtilization" } target_value = var.ecs_scaling_mem_threshold + scale_in_cooldown = 60 + scale_out_cooldown = 60 } } From 148ceb05e148582c09691550ff689262eda4539e Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 16:26:18 +0100 Subject: [PATCH 18/46] Test new cooldown for ecs tasks --- terraform/environments/apex/application_variables.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index eaf5ade4723..9d72b30c94a 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -19,13 +19,13 @@ "instance_type": "t3a.medium", "key_name": "", "ec2_desired_capacity": 1, - "ec2_max_size": 2, + "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", "network_mode": "bridge", "app_count": 1, "appscaling_min_capacity": 1, - "appscaling_max_capacity": 2, + "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, From edb55c2c1ba0c8e3f08cbc7e40fe88a594b9ce33 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 16:44:44 +0100 Subject: [PATCH 19/46] Test new cooldown for ecs tasks 2 --- terraform/environments/apex/application_variables.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 9d72b30c94a..780efd8ba27 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -18,7 +18,7 @@ "ami_image_id": "ami-0c1789b266d504ff3", "instance_type": "t3a.medium", "key_name": "", - "ec2_desired_capacity": 1, + "ec2_desired_capacity": 2, "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", @@ -29,7 +29,7 @@ "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, - "container_memory_allocation": 1000, + "container_memory_allocation": 2000, "region": "eu-west-2", "docker_image_tag": "development", "app_debug_enabled": "false", From 0bdfa8fec98e01d68cafaa159794ce89c863c276 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 16:50:41 +0100 Subject: [PATCH 20/46] Test new cooldown for ecs tasks 3 --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 780efd8ba27..62947163875 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -27,7 +27,7 @@ "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, - "ecs_scaling_mem_threshold": 80, + "ecs_scaling_mem_threshold": 0.3, "ec2_instance_warmup_period": 300, "container_memory_allocation": 2000, "region": "eu-west-2", From 85282dff65111dcfec3f42eae0dbc3df17327b94 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 16:52:35 +0100 Subject: [PATCH 21/46] Test new cooldown for ecs tasks 4 --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 62947163875..4baa3a17443 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -18,7 +18,7 @@ "ami_image_id": "ami-0c1789b266d504ff3", "instance_type": "t3a.medium", "key_name": "", - "ec2_desired_capacity": 2, + "ec2_desired_capacity": 1, "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", From 22de3a0f2a4af4ae896bbead495c177989bb89f8 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Fri, 14 Jun 2024 17:01:19 +0100 Subject: [PATCH 22/46] Test new cooldown for ecs tasks 5 --- terraform/environments/apex/application_variables.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 4baa3a17443..c4024fc7255 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -18,16 +18,16 @@ "ami_image_id": "ami-0c1789b266d504ff3", "instance_type": "t3a.medium", "key_name": "", - "ec2_desired_capacity": 1, + "ec2_desired_capacity": 5, "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", "network_mode": "bridge", - "app_count": 1, + "app_count": 5, "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, - "ecs_scaling_mem_threshold": 0.3, + "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, "container_memory_allocation": 2000, "region": "eu-west-2", From 6cf98a975172582f90f3751a051408900784a2e6 Mon Sep 17 00:00:00 2001 From: lalithanagarur Date: Fri, 14 Jun 2024 17:57:17 +0100 Subject: [PATCH 23/46] added essex police details to prod --- .../environment-configuration.tf | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-ingestion/environment-configuration.tf b/terraform/environments/analytical-platform-ingestion/environment-configuration.tf index 20cc86d0172..322ed285bf7 100644 --- a/terraform/environments/analytical-platform-ingestion/environment-configuration.tf +++ b/terraform/environments/analytical-platform-ingestion/environment-configuration.tf @@ -54,7 +54,14 @@ locals { /* Transfer Server */ transfer_server_hostname = "sftp.ingestion.analytical-platform.service.justice.gov.uk" transfer_server_sftp_users = {} - transfer_server_sftp_users_with_egress = {} + transfer_server_sftp_users_with_egress = { + "essex-police" = { + ssh_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpNyYzfbqmy4QkViLRyASRqxrEK7G34o3Bc6Jdp8vK555/oBuAUXUxfavtenZnxuxdxsrBYBSCBFcU4+igeXN/nN2kVfaUlt1xBZBCRUaajinhmt+3CLbr8bWmHR/5vL/DhxHH+j/+gDH5A244XN/ybZQvCGX/ilgKiae8s0tiOZD2hmX0fhRTCohQFG/DIu06gqKIyxUQoHBoBJxjzaDvjqioJgqmD9893DN+Gx1KozmaQWHM+0f7iK1UFp8BkdeFBVkj8TOfx60o/EmAjWQ/U+WSHblaXo0nI+LQKZYkW52uTEnfSkbkyvs/vj8E8+vagwYi0noyTVmb5qReSuk1kyuqEP2ycKIaWKt+Z4LnwxHm7KO51SMMeBgpiFHaUTQWXZHYuU2aXVfFIgJkCtHdEjG7Qe2P8K5XU5rG+CrQ/Y9PxPrKQHk+2nox9dLfCWo2Eho1N85z9/rA7A0oNwsHkjWAl3k87lWdpg7y3VNLzqsMNF4M4HjpQV60MH73dUU= essex-police@kpvmshift04app.netr.ecis.police.uk" + cidr_blocks = ["194.74.29.178/32"] + egress_bucket = module.bold_egress_bucket.s3_bucket_id + egress_bucket_kms_key = module.s3_bold_egress_kms.key_arn + } + } } } } From b7542e4dcc3f5213a1c2ebe03d49e9ecd87bba3c Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 09:19:12 +0100 Subject: [PATCH 24/46] Test scaling with task cooldown 1.0 --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index c4024fc7255..1c280f2913f 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -27,7 +27,7 @@ "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, - "ecs_scaling_mem_threshold": 80, + "ecs_scaling_mem_threshold": 0.3, "ec2_instance_warmup_period": 300, "container_memory_allocation": 2000, "region": "eu-west-2", From 2a0e1b87132f34f5cfc1b202fc01828e8d5015bf Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 09:24:00 +0100 Subject: [PATCH 25/46] Test scaling with task cooldown 1.1 --- terraform/environments/apex/application_variables.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 1c280f2913f..4baa3a17443 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -18,12 +18,12 @@ "ami_image_id": "ami-0c1789b266d504ff3", "instance_type": "t3a.medium", "key_name": "", - "ec2_desired_capacity": 5, + "ec2_desired_capacity": 1, "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", "network_mode": "bridge", - "app_count": 5, + "app_count": 1, "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, From 7d5505b41e45ad6003e66e14d37a77a3fde031e9 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 17 Jun 2024 08:24:33 +0000 Subject: [PATCH 26/46] Add CaDeT DPR runner Signed-off-by: Jacob Woffenden --- .../helm-charts-actions-runners.tf | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf b/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf index ee5b68790d5..9ce61048d51 100644 --- a/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf +++ b/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf @@ -27,3 +27,27 @@ resource "helm_release" "actions_runner_mojas_create_a_derived_table" { ) ] } + +resource "helm_release" "actions_runner_mojas_create_a_derived_table_dpr" { + count = terraform.workspace == "analytical-platform-compute-production" ? 1 : 0 + + /* https://github.com/ministryofjustice/analytical-platform-actions-runner */ + name = "actions-runner-mojas-create-a-derived-table-dpr" + repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts" + version = "2.317.0" + chart = "actions-runner" + namespace = kubernetes_namespace.actions_runners[0].metadata[0].name + values = [ + templatefile( + "${path.module}/src/helm/values/actions-runners/create-a-derived-table/values.yml.tftpl", + { + replicaCount = 1 + github_organisation = "moj-analytical-services" + github_repository = "create-a-derived-table" + github_token = data.aws_secretsmanager_secret_version.actions_runners_create_a_derived_table[0].secret_string + github_runner_labels = "analytical-platform,digital-prison-reporting" + eks_role_arn = "arn:aws:iam::972272129531:role/dpr-data-api-cross-account-role" + } + ) + ] +} From 16ce97009a9c43b8fd74511b0a33be9c32220004 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 17 Jun 2024 08:25:01 +0000 Subject: [PATCH 27/46] update label Signed-off-by: Jacob Woffenden --- .../analytical-platform-compute/helm-charts-actions-runners.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf b/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf index 9ce61048d51..41f1fda5e5b 100644 --- a/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf +++ b/terraform/environments/analytical-platform-compute/helm-charts-actions-runners.tf @@ -45,7 +45,7 @@ resource "helm_release" "actions_runner_mojas_create_a_derived_table_dpr" { github_organisation = "moj-analytical-services" github_repository = "create-a-derived-table" github_token = data.aws_secretsmanager_secret_version.actions_runners_create_a_derived_table[0].secret_string - github_runner_labels = "analytical-platform,digital-prison-reporting" + github_runner_labels = "digital-prison-reporting" eks_role_arn = "arn:aws:iam::972272129531:role/dpr-data-api-cross-account-role" } ) From fef0517257adb09a15d1ce8c5bb59ceeeb08b476 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 09:38:02 +0100 Subject: [PATCH 28/46] Test scaling with task cooldown 1.2 --- terraform/environments/apex/application_variables.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 4baa3a17443..d5e69a0e6dc 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -18,7 +18,7 @@ "ami_image_id": "ami-0c1789b266d504ff3", "instance_type": "t3a.medium", "key_name": "", - "ec2_desired_capacity": 1, + "ec2_desired_capacity": 5, "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", @@ -27,7 +27,7 @@ "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, - "ecs_scaling_mem_threshold": 0.3, + "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, "container_memory_allocation": 2000, "region": "eu-west-2", From c3aabbce98d344ef327f4acb3e64c2468d38855d Mon Sep 17 00:00:00 2001 From: Rich Green Date: Mon, 17 Jun 2024 09:47:13 +0100 Subject: [PATCH 29/46] remove `types` from `pull_request:` --- .github/workflows/cooker.yml | 1 - .github/workflows/example.yml | 1 - .github/workflows/sprinkler.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/cooker.yml b/.github/workflows/cooker.yml index f544318d731..52e133755b4 100644 --- a/.github/workflows/cooker.yml +++ b/.github/workflows/cooker.yml @@ -11,7 +11,6 @@ on: pull_request: branches: - main - types: [opened, edited, reopened, synchronize] paths: - 'terraform/environments/cooker/**' - '.github/workflows/cooker.yml' diff --git a/.github/workflows/example.yml b/.github/workflows/example.yml index 840cefe6b5e..64bf2c6d472 100644 --- a/.github/workflows/example.yml +++ b/.github/workflows/example.yml @@ -11,7 +11,6 @@ on: pull_request: branches: - main - types: [opened, edited, reopened, synchronize] paths: - 'terraform/environments/example/**' - '.github/workflows/example.yml' diff --git a/.github/workflows/sprinkler.yml b/.github/workflows/sprinkler.yml index 96c308be523..c47449fa079 100644 --- a/.github/workflows/sprinkler.yml +++ b/.github/workflows/sprinkler.yml @@ -11,7 +11,6 @@ on: pull_request: branches: - main - types: [opened, edited, reopened, synchronize] paths: - 'terraform/environments/sprinkler/**' - '.github/workflows/sprinkler.yml' From 88cf6d191011f153b2721a246c52cc915bdf7cd0 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 09:56:37 +0100 Subject: [PATCH 30/46] Revert to default ecs cooldown of 300s --- terraform/environments/apex/application_variables.json | 2 +- terraform/environments/apex/modules/ecs/main.tf | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index d5e69a0e6dc..ca067d7fd25 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -27,7 +27,7 @@ "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, - "ecs_scaling_mem_threshold": 80, + "ecs_scaling_mem_threshold": 20, "ec2_instance_warmup_period": 300, "container_memory_allocation": 2000, "region": "eu-west-2", diff --git a/terraform/environments/apex/modules/ecs/main.tf b/terraform/environments/apex/modules/ecs/main.tf index 16e4958fbdf..4c2f5a60ef2 100644 --- a/terraform/environments/apex/modules/ecs/main.tf +++ b/terraform/environments/apex/modules/ecs/main.tf @@ -502,8 +502,8 @@ resource "aws_appautoscaling_policy" "ecs_target_cpu" { predefined_metric_type = "ECSServiceAverageCPUUtilization" } target_value = var.ecs_scaling_cpu_threshold - scale_in_cooldown = 60 - scale_out_cooldown = 60 + # scale_in_cooldown = 300 + # scale_out_cooldown = 300 } } @@ -518,8 +518,8 @@ resource "aws_appautoscaling_policy" "ecs_target_memory" { predefined_metric_type = "ECSServiceAverageMemoryUtilization" } target_value = var.ecs_scaling_mem_threshold - scale_in_cooldown = 60 - scale_out_cooldown = 60 + # scale_in_cooldown = 300 + # scale_out_cooldown = 300 } } From 59fbceb2645a706743a848f13faa1889c74d448e Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 09:59:18 +0100 Subject: [PATCH 31/46] Force set ecs cooldown to 300s --- terraform/environments/apex/application_variables.json | 2 +- terraform/environments/apex/modules/ecs/main.tf | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index ca067d7fd25..5c8c945a3c2 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -18,7 +18,7 @@ "ami_image_id": "ami-0c1789b266d504ff3", "instance_type": "t3a.medium", "key_name": "", - "ec2_desired_capacity": 5, + "ec2_desired_capacity": 1, "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", diff --git a/terraform/environments/apex/modules/ecs/main.tf b/terraform/environments/apex/modules/ecs/main.tf index 4c2f5a60ef2..6078c72c837 100644 --- a/terraform/environments/apex/modules/ecs/main.tf +++ b/terraform/environments/apex/modules/ecs/main.tf @@ -502,8 +502,8 @@ resource "aws_appautoscaling_policy" "ecs_target_cpu" { predefined_metric_type = "ECSServiceAverageCPUUtilization" } target_value = var.ecs_scaling_cpu_threshold - # scale_in_cooldown = 300 - # scale_out_cooldown = 300 + scale_in_cooldown = 300 + scale_out_cooldown = 300 } } @@ -518,8 +518,8 @@ resource "aws_appautoscaling_policy" "ecs_target_memory" { predefined_metric_type = "ECSServiceAverageMemoryUtilization" } target_value = var.ecs_scaling_mem_threshold - # scale_in_cooldown = 300 - # scale_out_cooldown = 300 + scale_in_cooldown = 300 + scale_out_cooldown = 300 } } From 804338e464a7808d8f2a6f581baef01fe0ba6cb8 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 10:10:27 +0100 Subject: [PATCH 32/46] Force set ecs cooldown to 300s v2 --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 5c8c945a3c2..4baa3a17443 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -27,7 +27,7 @@ "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, - "ecs_scaling_mem_threshold": 20, + "ecs_scaling_mem_threshold": 0.3, "ec2_instance_warmup_period": 300, "container_memory_allocation": 2000, "region": "eu-west-2", From 00af8b233bb6b0425034c4f481066e1897d81d94 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 10:12:28 +0100 Subject: [PATCH 33/46] Force set ecs cooldown to 300s v3 --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 4baa3a17443..754f866bf95 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -18,7 +18,7 @@ "ami_image_id": "ami-0c1789b266d504ff3", "instance_type": "t3a.medium", "key_name": "", - "ec2_desired_capacity": 1, + "ec2_desired_capacity": 3, "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", From 1b111c6648d4579f5c558ac082202f2eabbea77a Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 10:26:50 +0100 Subject: [PATCH 34/46] Force set ecs cooldown to 300s v4 --- terraform/environments/apex/application_variables.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 754f866bf95..00ba2201ac9 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -18,7 +18,7 @@ "ami_image_id": "ami-0c1789b266d504ff3", "instance_type": "t3a.medium", "key_name": "", - "ec2_desired_capacity": 3, + "ec2_desired_capacity": 4, "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", @@ -27,7 +27,7 @@ "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, - "ecs_scaling_mem_threshold": 0.3, + "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, "container_memory_allocation": 2000, "region": "eu-west-2", From 5e136368df7a199efcafe8f7643c6737e57be179 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 10:31:15 +0100 Subject: [PATCH 35/46] Force set ecs cooldown to 300s v5 --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 00ba2201ac9..29c4f0d86bb 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -27,7 +27,7 @@ "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, - "ecs_scaling_mem_threshold": 80, + "ecs_scaling_mem_threshold": 0.3, "ec2_instance_warmup_period": 300, "container_memory_allocation": 2000, "region": "eu-west-2", From 4ddb5a481b2ae3ea328a385fa9d3060124ada4c8 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 10:43:15 +0100 Subject: [PATCH 36/46] Force set ecs cooldown to 300s v6 --- terraform/environments/apex/application_variables.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 29c4f0d86bb..c4024fc7255 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -18,16 +18,16 @@ "ami_image_id": "ami-0c1789b266d504ff3", "instance_type": "t3a.medium", "key_name": "", - "ec2_desired_capacity": 4, + "ec2_desired_capacity": 5, "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", "network_mode": "bridge", - "app_count": 1, + "app_count": 5, "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, - "ecs_scaling_mem_threshold": 0.3, + "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, "container_memory_allocation": 2000, "region": "eu-west-2", From 718ffed9ea843eeff24bc0977cf37e6ddacac001 Mon Sep 17 00:00:00 2001 From: David Sibley Date: Mon, 17 Jun 2024 10:55:54 +0100 Subject: [PATCH 37/46] updated ssl_policy for https listener --- terraform/environments/performance-hub/loadbalancer.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/environments/performance-hub/loadbalancer.tf b/terraform/environments/performance-hub/loadbalancer.tf index e89ee4de7b6..92280419e67 100644 --- a/terraform/environments/performance-hub/loadbalancer.tf +++ b/terraform/environments/performance-hub/loadbalancer.tf @@ -76,6 +76,7 @@ resource "aws_lb_listener" "https_listener" { port = "443" protocol = "HTTPS" certificate_arn = format("arn:aws:acm:eu-west-2:%s:certificate/%s", data.aws_caller_identity.current.account_id, local.app_data.accounts[local.environment].cert_arn) + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" default_action { target_group_arn = aws_lb_target_group.target_group.id From 7dda040f651f915d01373363b7043a1b994d3dbf Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 11:07:49 +0100 Subject: [PATCH 38/46] Testing deployment for final time 1 --- terraform/environments/apex/application_variables.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index c4024fc7255..bb0d44d97ac 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -18,18 +18,18 @@ "ami_image_id": "ami-0c1789b266d504ff3", "instance_type": "t3a.medium", "key_name": "", - "ec2_desired_capacity": 5, + "ec2_desired_capacity": 1, "ec2_max_size": 5, "ec2_min_size": 1, "task_definition_volume": "upload_volume", "network_mode": "bridge", - "app_count": 5, + "app_count": 1, "appscaling_min_capacity": 1, "appscaling_max_capacity": 5, "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, - "container_memory_allocation": 2000, + "container_memory_allocation": 2024, "region": "eu-west-2", "docker_image_tag": "development", "app_debug_enabled": "false", From 49ecea203695d7fc042d07e8a82d21a716f73721 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 12:02:12 +0100 Subject: [PATCH 39/46] LAWS-3669 Revert all changes except ones being kept --- terraform/environments/apex/application_variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index bb0d44d97ac..9d72b30c94a 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -29,7 +29,7 @@ "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, - "container_memory_allocation": 2024, + "container_memory_allocation": 1000, "region": "eu-west-2", "docker_image_tag": "development", "app_debug_enabled": "false", From 665e6c0b3b63372e9914e9072ba7a7fc6e580f19 Mon Sep 17 00:00:00 2001 From: "Vincent.Cheung" Date: Mon, 17 Jun 2024 12:03:52 +0100 Subject: [PATCH 40/46] LAWS-3669 Revert all changes except ones being kept 2 --- terraform/environments/apex/application_variables.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json index 9d72b30c94a..eaf5ade4723 100644 --- a/terraform/environments/apex/application_variables.json +++ b/terraform/environments/apex/application_variables.json @@ -19,13 +19,13 @@ "instance_type": "t3a.medium", "key_name": "", "ec2_desired_capacity": 1, - "ec2_max_size": 5, + "ec2_max_size": 2, "ec2_min_size": 1, "task_definition_volume": "upload_volume", "network_mode": "bridge", "app_count": 1, "appscaling_min_capacity": 1, - "appscaling_max_capacity": 5, + "appscaling_max_capacity": 2, "ecs_scaling_cpu_threshold": 80, "ecs_scaling_mem_threshold": 80, "ec2_instance_warmup_period": 300, From 3bdfdece334972495e363859573e9635f94558f9 Mon Sep 17 00:00:00 2001 From: Dominic Robinson <65237317+drobinson-moj@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:01:32 +0100 Subject: [PATCH 41/46] nomis: DSOS-2229: reenable session loggin (#6620) --- terraform/environments/nomis/locals.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/environments/nomis/locals.tf b/terraform/environments/nomis/locals.tf index 13fb16e86f5..a608bd69a17 100644 --- a/terraform/environments/nomis/locals.tf +++ b/terraform/environments/nomis/locals.tf @@ -38,12 +38,12 @@ locals { enable_ec2_cloud_watch_agent = true enable_ec2_oracle_enterprise_managed_server = true enable_ec2_self_provision = true - # enable_ec2_session_manager_cloudwatch_logs = true - enable_ec2_user_keypair = true - enable_image_builder = true - enable_hmpps_domain = true # Syscon users are collaborators so need domain creds to access nomis-client EC2s - iam_policies_filter = ["ImageBuilderS3BucketWriteAndDeleteAccessPolicy"] - iam_policies_ec2_default = ["EC2S3BucketWriteAndDeleteAccessPolicy", "ImageBuilderS3BucketWriteAndDeleteAccessPolicy"] + enable_ec2_session_manager_cloudwatch_logs = true + enable_ec2_user_keypair = true + enable_image_builder = true + enable_hmpps_domain = true # Syscon users are collaborators so need domain creds to access nomis-client EC2s + iam_policies_filter = ["ImageBuilderS3BucketWriteAndDeleteAccessPolicy"] + iam_policies_ec2_default = ["EC2S3BucketWriteAndDeleteAccessPolicy", "ImageBuilderS3BucketWriteAndDeleteAccessPolicy"] route53_resolver_rules = { outbound-data-and-private-subnets = ["azure-fixngo-domain"] } From 06a31b9bb2a5bd65fd6cd078ab2ea817015fada0 Mon Sep 17 00:00:00 2001 From: Madhu Kadiri Date: Mon, 17 Jun 2024 13:24:25 +0100 Subject: [PATCH 42/46] char(129) replacement transformation added --- .../dms_g4s_cap_dw_task_tables_selection.json | 2 +- .../D_Comments_V2_Insert_Into_Select.sql | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/terraform/environments/electronic-monitoring-data/dms_g4s_cap_dw_task_tables_selection.json b/terraform/environments/electronic-monitoring-data/dms_g4s_cap_dw_task_tables_selection.json index 0b1848207a3..dfb55110f5f 100644 --- a/terraform/environments/electronic-monitoring-data/dms_g4s_cap_dw_task_tables_selection.json +++ b/terraform/environments/electronic-monitoring-data/dms_g4s_cap_dw_task_tables_selection.json @@ -5,7 +5,7 @@ "rule-id": "01", "rule-name": "all", "object-locator": { - "schema-name": "dbo", + "schema-name": "%", "table-name": "%" }, "rule-action": "include", diff --git a/terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql b/terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql index fec30bc9475..9d7179f2078 100644 --- a/terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql +++ b/terraform/environments/electronic-monitoring-data/modules/dms/RDS_MSSQLServer_DB_Scripts/D_Comments_V2_Insert_Into_Select.sql @@ -1,8 +1,12 @@ SET IDENTITY_INSERT g4s_cap_dw.dbo.D_Comments_V2 ON ; +truncate table [g4s_cap_dw].[dbo].[D_Comments_V2]; + INSERT INTO g4s_cap_dw.dbo.D_Comments_V2 (CommentSID, VisitID, ActivityID, Comments, CommentType) - SELECT CommentSID, VisitID, ActivityID, trim(replace(Comments, char(141), '')) AS Comments, CommentType + SELECT CommentSID, VisitID, ActivityID, + trim(replace(replace(Comments, char(141), ''), char(129), '')) AS Comments, + CommentType FROM g4s_cap_dw.dbo.D_Comments ; From 2c58057916c8fa1dcaffeb08552805fc338aa99b Mon Sep 17 00:00:00 2001 From: W Date: Mon, 17 Jun 2024 15:31:12 +0100 Subject: [PATCH 43/46] onr db alarms to dbas onr db alarms to dbas --- .../environments/oasys/locals_defaults.tf | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/terraform/environments/oasys/locals_defaults.tf b/terraform/environments/oasys/locals_defaults.tf index dfca54c1903..91ca5e67375 100644 --- a/terraform/environments/oasys/locals_defaults.tf +++ b/terraform/environments/oasys/locals_defaults.tf @@ -173,24 +173,10 @@ locals { database_onr_a = { cloudwatch_metric_alarms = merge( - module.baseline_presets.cloudwatch_metric_alarms.ec2, - module.baseline_presets.cloudwatch_metric_alarms.ec2_cwagent_linux, - module.baseline_presets.cloudwatch_metric_alarms.ec2_instance_cwagent_collectd_service_status_os, - module.baseline_presets.cloudwatch_metric_alarms.ec2_instance_cwagent_collectd_service_status_app, - { - cpu-utilization-high = { - comparison_operator = "GreaterThanOrEqualToThreshold" - evaluation_periods = "120" - datapoints_to_alarm = "120" - metric_name = "CPUUtilization" - namespace = "AWS/EC2" - period = "60" - statistic = "Maximum" - threshold = "95" - alarm_description = "Triggers if the average cpu remains at 95% utilization or above for 2 hours on an oasys-db instance" - alarm_actions = ["dso_pagerduty"] - } - }, + module.baseline_presets.cloudwatch_metric_alarms_by_sns_topic["dba_pagerduty"].ec2, + module.baseline_presets.cloudwatch_metric_alarms_by_sns_topic["dba_pagerduty"].ec2_cwagent_linux, + module.baseline_presets.cloudwatch_metric_alarms_by_sns_topic["dso_pagerduty"].ec2_instance_cwagent_collectd_service_status_os, + module.baseline_presets.cloudwatch_metric_alarms_by_sns_topic["dba_pagerduty"].ec2_instance_cwagent_collectd_service_status_app, { low-inodes = { comparison_operator = "GreaterThanOrEqualToThreshold" @@ -204,7 +190,25 @@ locals { alarm_description = "Triggers if free inodes falls below the threshold for an hour" alarm_actions = ["dso_pagerduty"] } - } + }, + local.environment == "production" ? {} : { + cpu-utilization-high = merge(module.baseline_presets.cloudwatch_metric_alarms_by_sns_topic["dba_pagerduty"].ec2["cpu-utilization-high"], { + evaluation_periods = "480" + datapoints_to_alarm = "480" + threshold = "95" + alarm_description = "Triggers if the average cpu remains at 95% utilization or above for 8 hours to allow for DB refreshes. See https://dsdmoj.atlassian.net/wiki/spaces/DSTT/pages/4326064583" + }) + cpu-iowait-high = merge(module.baseline_presets.cloudwatch_metric_alarms_by_sns_topic["dba_pagerduty"].ec2_cwagent_linux["cpu-iowait-high"], { + evaluation_periods = "480" + datapoints_to_alarm = "480" + threshold = "40" + alarm_description = "Triggers if the amount of CPU time spent waiting for I/O to complete is continually high for 8 hours allowing for DB refreshes. See https://dsdmoj.atlassian.net/wiki/spaces/DSTT/pages/4325900634" + }) + }, + # DBAs have slack integration via OEM for this so don't include pagerduty integration + module.baseline_presets.cloudwatch_metric_alarms.ec2_instance_cwagent_collectd_oracle_db_connected, + # db_backup + module.baseline_presets.cloudwatch_metric_alarms_by_sns_topic["dba_pagerduty"].ec2_instance_cwagent_collectd_oracle_db_backup, ) config = merge(module.baseline_presets.ec2_instance.config.db, { ami_name = "base_rhel_7_9_2024-01-01T00-00-06.493Z" From 2d573ebeeba93d0da9960ad4c1f0658544ea4836 Mon Sep 17 00:00:00 2001 From: Dominic Robinson <65237317+drobinson-moj@users.noreply.github.com> Date: Mon, 17 Jun 2024 15:53:09 +0100 Subject: [PATCH 44/46] Nomis: DSOS-2229: enable update of ssm agent (#6623) * DSOS-2229: add ssm agent association * add ssm-associations * add SSM associations * test s3 bucket location * fix * remove unused file --- terraform/environments/nomis/ec2_common.tf | 19 -------- terraform/environments/nomis/locals.tf | 1 + .../environments/nomis/locals_development.tf | 2 + .../nomis/locals_ec2_autoscaling_groups.tf | 19 ++++---- .../nomis/locals_ec2_instances.tf | 15 ++++--- .../nomis/locals_preproduction.tf | 2 + .../environments/nomis/locals_production.tf | 2 + terraform/environments/nomis/locals_test.tf | 1 + terraform/environments/nomis/main.tf | 6 +++ terraform/modules/baseline/ssm.tf | 30 +++++++++++++ terraform/modules/baseline/variables.tf | 43 +++++++++++++----- terraform/modules/baseline_presets/outputs.tf | 7 +++ terraform/modules/baseline_presets/ssm.tf | 45 +++++++++++++++++++ .../modules/baseline_presets/variables.tf | 1 + 14 files changed, 149 insertions(+), 44 deletions(-) delete mode 100644 terraform/environments/nomis/ec2_common.tf diff --git a/terraform/environments/nomis/ec2_common.tf b/terraform/environments/nomis/ec2_common.tf deleted file mode 100644 index e109524b2ac..00000000000 --- a/terraform/environments/nomis/ec2_common.tf +++ /dev/null @@ -1,19 +0,0 @@ -#------------------------------------------------------------------------------ -# SSM Agent - update Systems Manager Agent -#------------------------------------------------------------------------------ - -#resource "aws_ssm_association" "update_ssm_agent" { -# name = "AWS-UpdateSSMAgent" # this is an AWS provided document -# association_name = "update-ssm-agent" -# parameters = { -# allowDowngrade = "false" -# } -# targets { -# # we could just target all instances, but this would also include the bastion, which gets rebuilt everyday -# key = "tag:os_type" -# values = ["Linux", "Windows"] -# } -# apply_only_at_cron_interval = false -# schedule_expression = "cron(30 7 ? * TUE *)" -#} - diff --git a/terraform/environments/nomis/locals.tf b/terraform/environments/nomis/locals.tf index a608bd69a17..33dbc5a7174 100644 --- a/terraform/environments/nomis/locals.tf +++ b/terraform/environments/nomis/locals.tf @@ -39,6 +39,7 @@ locals { enable_ec2_oracle_enterprise_managed_server = true enable_ec2_self_provision = true enable_ec2_session_manager_cloudwatch_logs = true + enable_ec2_ssm_agent_update = true enable_ec2_user_keypair = true enable_image_builder = true enable_hmpps_domain = true # Syscon users are collaborators so need domain creds to access nomis-client EC2s diff --git a/terraform/environments/nomis/locals_development.tf b/terraform/environments/nomis/locals_development.tf index 2b25456b140..35878e6e6f5 100644 --- a/terraform/environments/nomis/locals_development.tf +++ b/terraform/environments/nomis/locals_development.tf @@ -132,6 +132,7 @@ locals { tags = merge(local.ec2_instances.build.tags, { description = "Syscon build and release server" instance-scheduling = "skip-scheduling" + update-ssm-agent = "patchgroup2" }) }) @@ -159,6 +160,7 @@ locals { instance-scheduling = "skip-scheduling" nomis-environment = "dev" oracle-sids = "" + update-ssm-agent = "patchgroup2" }) }) diff --git a/terraform/environments/nomis/locals_ec2_autoscaling_groups.tf b/terraform/environments/nomis/locals_ec2_autoscaling_groups.tf index 3062f1e467a..d235c324d41 100644 --- a/terraform/environments/nomis/locals_ec2_autoscaling_groups.tf +++ b/terraform/environments/nomis/locals_ec2_autoscaling_groups.tf @@ -41,9 +41,10 @@ locals { ] } tags = { - backup = "false" - component = "test" - os-type = "Linux" + backup = "false" + component = "test" + os-type = "Linux" + update-ssm-agent = "patchgroup1" } } @@ -96,6 +97,7 @@ locals { instance-access-policy = "full" os-type = "Windows" server-type = "NomisClient" + update-ssm-agent = "patchgroup1" } } @@ -184,11 +186,12 @@ locals { } tags = { # ami = "base_ol_8_5" # commented out to ensure harden role does not re-run - backup = "false" - component = "web" - description = "For testing nomis weblogic 19c image" - os-type = "Linux" - server-type = "nomis-web19c" + backup = "false" + component = "web" + description = "For testing nomis weblogic 19c image" + os-type = "Linux" + server-type = "nomis-web19c" + update-ssm-agent = "patchgroup1" } } } diff --git a/terraform/environments/nomis/locals_ec2_instances.tf b/terraform/environments/nomis/locals_ec2_instances.tf index bbb2a6dce9d..8b1a0cac0e2 100644 --- a/terraform/environments/nomis/locals_ec2_instances.tf +++ b/terraform/environments/nomis/locals_ec2_instances.tf @@ -44,11 +44,12 @@ locals { ] } tags = { - ami = "base_rhel_7_9" - backup = "false" - component = "build" - os-type = "Linux" - server-type = "nomis-build" + ami = "base_rhel_7_9" + backup = "false" + component = "build" + os-type = "Linux" + server-type = "nomis-build" + update-ssm-agent = "patchgroup1" } } @@ -123,6 +124,7 @@ locals { os-type = "Linux" os-version = "RHEL 7.9" server-type = "nomis-db" + update-ssm-agent = "patchgroup1" } } @@ -197,6 +199,7 @@ locals { os-type = "Linux" os-version = "OL 8.5" server-type = "nomis-db19c" + update-ssm-agent = "patchgroup1" } } @@ -242,6 +245,7 @@ locals { instance-access-policy = "limited" os-type = "Linux" server-type = "nomis-web" + #update-ssm-agent = "patchgroup1" # not supported on RHEL6, don't include } } @@ -281,6 +285,7 @@ locals { instance-access-policy = "limited" os-type = "Linux" server-type = "nomis-xtag" + update-ssm-agent = "patchgroup1" } } } diff --git a/terraform/environments/nomis/locals_preproduction.tf b/terraform/environments/nomis/locals_preproduction.tf index 6fb29a511f2..ef1a2fff028 100644 --- a/terraform/environments/nomis/locals_preproduction.tf +++ b/terraform/environments/nomis/locals_preproduction.tf @@ -200,6 +200,7 @@ locals { instance-scheduling = "skip-scheduling" nomis-environment = "preprod" oracle-sids = "PPCNOM PPNDH PPTRDAT" + update-ssm-agent = "patchgroup2" }) }) @@ -265,6 +266,7 @@ locals { misload-dbname = "PPMIS" nomis-environment = "preprod" oracle-sids = "PPMIS PPCNMAUD" + update-ssm-agent = "patchgroup2" }) }) diff --git a/terraform/environments/nomis/locals_production.tf b/terraform/environments/nomis/locals_production.tf index 9b006b96c4f..a1feef97e1c 100644 --- a/terraform/environments/nomis/locals_production.tf +++ b/terraform/environments/nomis/locals_production.tf @@ -179,6 +179,7 @@ locals { description = "Production databases for CNOM and NDH" nomis-environment = "prod" oracle-sids = "PDCNOM PDNDH PDTRDAT" + update-ssm-agent = "patchgroup2" }) }) @@ -244,6 +245,7 @@ locals { misload-dbname = "PDMIS" nomis-environment = "prod" oracle-sids = "PDCNMAUD PDMIS" + update-ssm-agent = "patchgroup2" }) }) diff --git a/terraform/environments/nomis/locals_test.tf b/terraform/environments/nomis/locals_test.tf index c51cfdbc1ba..5bc09c47c86 100644 --- a/terraform/environments/nomis/locals_test.tf +++ b/terraform/environments/nomis/locals_test.tf @@ -379,6 +379,7 @@ locals { instance-scheduling = "skip-scheduling" nomis-environment = "t3" oracle-sids = "T3CNOM" + update-ssm-agent = "patchgroup2" }) }) } diff --git a/terraform/environments/nomis/main.tf b/terraform/environments/nomis/main.tf index de28fdf87bf..a23f7e6d41b 100644 --- a/terraform/environments/nomis/main.tf +++ b/terraform/environments/nomis/main.tf @@ -194,6 +194,12 @@ module "baseline" { lookup(local.baseline_environment_specific, "sns_topics", {}), ) + ssm_associations = merge( + module.baseline_presets.ssm_associations, + lookup(local.baseline_all_environments, "ssm_associations", {}), + lookup(local.baseline_environment_specific, "ssm_associations", {}), + ) + ssm_documents = merge( module.baseline_presets.ssm_documents, lookup(local.baseline_all_environments, "ssm_documents", {}), diff --git a/terraform/modules/baseline/ssm.tf b/terraform/modules/baseline/ssm.tf index 0b47a023849..588f02e66a2 100644 --- a/terraform/modules/baseline/ssm.tf +++ b/terraform/modules/baseline/ssm.tf @@ -54,6 +54,36 @@ locals { } } +resource "aws_ssm_association" "this" { + for_each = var.ssm_associations + + apply_only_at_cron_interval = each.value.apply_only_at_cron_interval + association_name = each.key + name = each.value.name + max_concurrency = each.value.max_concurrency + max_errors = each.value.max_errors + schedule_expression = each.value.schedule_expression + + dynamic "output_location" { + for_each = each.value.output_location != null ? [each.value.output_location] : [] + content { + s3_bucket_name = try(module.s3_bucket[output_location.value.s3_bucket_name].bucket.bucket, output_location.value.s3_bucket_name) + s3_key_prefix = output_location.value.s3_key_prefix + s3_region = var.environment.region + } + } + + dynamic "targets" { + for_each = each.value.targets + content { + key = targets.value.key + values = [ + for value in targets.value.values : try(module.ec2_instance[value].aws_instance.id, value) + ] + } + } +} + resource "aws_ssm_document" "this" { for_each = var.ssm_documents diff --git a/terraform/modules/baseline/variables.tf b/terraform/modules/baseline/variables.tf index 2a599f4a356..4032be3e404 100644 --- a/terraform/modules/baseline/variables.tf +++ b/terraform/modules/baseline/variables.tf @@ -1122,6 +1122,37 @@ variable "sns_topics" { default = {} } +variable "ssm_associations" { + description = "A map of ssm associations to create where map key is the association name" + type = map(object({ + apply_only_at_cron_interval = optional(bool) + name = string + max_concurrency = optional(number) + max_errors = optional(number) + schedule_expression = optional(string) + output_location = optional(object({ + s3_bucket_name = string # or s3_buckets map key + s3_key_prefix = optional(string) + })) + targets = optional(list(object({ + key = string # 'tag:my_tag_name' or 'InstanceIds' + values = list(string) # [my_tag_value] or [ec2_instance map key] + })), []) + })) + default = {} +} + +variable "ssm_documents" { + description = "A map of ssm documents to create where map key is the document name" + type = map(object({ + content = string + document_format = optional(string) + document_type = string + tags = optional(map(string), {}) + })) + default = {} +} + variable "ssm_parameters" { # Example usage: # my_ec2_params = { @@ -1163,20 +1194,8 @@ variable "ssm_parameters" { default = {} } -variable "ssm_documents" { - description = "A map of ssm documents to create where map key is the document name" - type = map(object({ - content = string - document_format = optional(string) - document_type = string - tags = optional(map(string), {}) - })) - default = {} -} - variable "tags" { description = "Any additional tags to apply to all resources, in addition to those provided by environment module" type = map(string) default = {} } - diff --git a/terraform/modules/baseline_presets/outputs.tf b/terraform/modules/baseline_presets/outputs.tf index 16796bd9837..0f2cecfb5b3 100644 --- a/terraform/modules/baseline_presets/outputs.tf +++ b/terraform/modules/baseline_presets/outputs.tf @@ -172,6 +172,13 @@ output "secretsmanager_secrets" { } } +output "ssm_associations" { + description = "Map of common ssm associations to create" + value = { + for key, value in local.ssm_associations : key => value if contains(local.ssm_associations_filter, key) + } +} + output "ssm_documents" { description = "Map of common ssm documents to create" value = { diff --git a/terraform/modules/baseline_presets/ssm.tf b/terraform/modules/baseline_presets/ssm.tf index ade1700f657..bd4eadd5367 100644 --- a/terraform/modules/baseline_presets/ssm.tf +++ b/terraform/modules/baseline_presets/ssm.tf @@ -1,5 +1,24 @@ locals { + ssm_schedule_expressions = { + development = { + update-ssm-agent-patchgroup1 = "cron(30 7 ? * MON *)" + update-ssm-agent-patchgroup2 = "cron(30 7 ? * WED *)" + } + test = { + update-ssm-agent-patchgroup1 = "cron(30 7 ? * MON *)" + update-ssm-agent-patchgroup2 = "cron(30 7 ? * WED *)" + } + preproduction = { + update-ssm-agent-patchgroup1 = "cron(00 12 ? * TUE *)" + update-ssm-agent-patchgroup2 = "cron(00 12 ? * WED *)" + } + production = { + update-ssm-agent-patchgroup1 = "cron(00 12 ? * WED *)" + update-ssm-agent-patchgroup2 = "cron(00 12 ? * THU *)" + } + } + # the modernisation platform secret 'environment_management' can not be # accessed from EC2s. Create a copy as an SSM parameter with just # the relevant account ids. @@ -14,6 +33,32 @@ locals { # add a cloud watch windows SSM param if the file is present cloud_watch_windows_filename = "./templates/cloud_watch_windows.json" + ssm_associations_filter = flatten([ + var.options.enable_ec2_ssm_agent_update ? ["AWS-UpdateSSMAgent-patchgroup1"] : [], + var.options.enable_ec2_ssm_agent_update ? ["AWS-UpdateSSMAgent-patchgroup2"] : [], + ]) + + ssm_associations = { + AWS-UpdateSSMAgent-patchgroup1 = { + apply_only_at_cron_interval = true + name = "AWS-UpdateSSMAgent" + schedule_expression = local.ssm_schedule_expressions[var.environment.environment].update-ssm-agent-patchgroup1 + targets = [{ + key = "tag:update-ssm-agent" + values = ["patchgroup1"] + }] + } + AWS-UpdateSSMAgent-patchgroup2 = { + apply_only_at_cron_interval = true + name = "AWS-UpdateSSMAgent" + schedule_expression = local.ssm_schedule_expressions[var.environment.environment].update-ssm-agent-patchgroup2 + targets = [{ + key = "tag:update-ssm-agent" + values = ["patchgroup2"] + }] + } + } + ssm_documents_filter = flatten([ var.options.enable_hmpps_domain ? ["ec2-ad-join-windows"] : [], var.options.enable_hmpps_domain ? ["ec2-ad-leave-windows"] : [], diff --git a/terraform/modules/baseline_presets/variables.tf b/terraform/modules/baseline_presets/variables.tf index 6687201bc06..e5e2c6878ae 100644 --- a/terraform/modules/baseline_presets/variables.tf +++ b/terraform/modules/baseline_presets/variables.tf @@ -33,6 +33,7 @@ variable "options" { enable_ec2_reduced_ssm_policy = optional(bool, false) # create standard AWS SSM policy minus ssm:GetParameter enable_ec2_oracle_enterprise_managed_server = optional(bool, false) # create role for accessing secrets in hmpps-oem accounts enable_ec2_session_manager_cloudwatch_logs = optional(bool, false) # create SSM doc and log group for session manager logs + enable_ec2_ssm_agent_update = optional(bool, false) # create SSM association for auto-update of SSM agent. update-ssm-agent tag needs to be set on EC2s also enable_ec2_user_keypair = optional(bool, false) # create secret and key-pair for ec2-user enable_shared_s3 = optional(bool, false) # create devtest and preprodprod S3 bucket enable_observability_platform_monitoring = optional(bool, false) # create role for observability platform monitroing From 5f71a5bfb3586b3988d5e4e09ba5ec405d2de22d Mon Sep 17 00:00:00 2001 From: Bill Buchan Date: Mon, 17 Jun 2024 17:59:47 +0100 Subject: [PATCH 45/46] Create new Secrets for DMS Endpoint Configuration --- .../oracle_db_shared/dms_secrets.tf | 80 +++++++++++++++++++ .../components/oracle_db_shared/iam.tf | 53 ++++++++++++ .../components/oracle_db_shared/locals.tf | 6 ++ 3 files changed, 139 insertions(+) create mode 100644 terraform/environments/delius-core/modules/components/oracle_db_shared/dms_secrets.tf diff --git a/terraform/environments/delius-core/modules/components/oracle_db_shared/dms_secrets.tf b/terraform/environments/delius-core/modules/components/oracle_db_shared/dms_secrets.tf new file mode 100644 index 00000000000..358c7d5f871 --- /dev/null +++ b/terraform/environments/delius-core/modules/components/oracle_db_shared/dms_secrets.tf @@ -0,0 +1,80 @@ +# Database Read Access +resource "aws_secretsmanager_secret" "dms_audit_endpoint_source" { + name = local.dms_audit_endpoint_source_secret_name + description = "Database Endpoint for Reading Audited Interaction Replication Data" + kms_key_id = var.account_config.kms_keys.general_shared + tags = var.tags +} + +data "aws_iam_policy_document" "dms_audit_endpoint_source" { + statement { + sid = "DMSRoleToReadTheSecret" + effect = "Allow" + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${local.delius_account_id}:role/DMSSecretsManagerAccessRole"] + } + actions = ["secretsmanager:GetSecretValue"] + resources = [aws_secretsmanager_secret.dms_audit_endpoint_source.arn] + } +} + +resource "aws_secretsmanager_secret_policy" "dms_audit_endpoint_source" { + secret_arn = aws_secretsmanager_secret.dms_audit_endpoint_source.arn + policy = data.aws_iam_policy_document.dms_audit_endpoint_source.json +} + + +# ASM Read Access +resource "aws_secretsmanager_secret" "dms_asm_endpoint_source" { + name = local.dms_asm_endpoint_source_secret_name + description = "ASM Endpoint" + kms_key_id = var.account_config.kms_keys.general_shared + tags = var.tags +} + +data "aws_iam_policy_document" "dms_asm_endpoint_source" { + statement { + sid = "DMSRoleToReadTheSecret" + effect = "Allow" + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${local.delius_account_id}:role/DMSSecretsManagerAccessRole"] + } + actions = ["secretsmanager:GetSecretValue"] + resources = [aws_secretsmanager_secret.dms_asm_endpoint_source.arn] + } +} + +resource "aws_secretsmanager_secret_policy" "dms_asm_endpoint_source" { + secret_arn = aws_secretsmanager_secret.dms_asm_endpoint_source.arn + policy = data.aws_iam_policy_document.dms_asm_endpoint_source.json +} + + +# Database Write Access +resource "aws_secretsmanager_secret" "dms_audit_endpoint_target" { + name = local.dms_audit_endpoint_source_secret_name + description = "Database Endpoint for Writing Audited Interaction Replication Data" + kms_key_id = var.account_config.kms_keys.general_shared + tags = var.tags +} + +data "aws_iam_policy_document" "dms_audit_endpoint_target" { + statement { + sid = "DMSRoleToReadTheSecret" + effect = "Allow" + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${local.delius_account_id}:role/DMSSecretsManagerAccessRole"] + } + actions = ["secretsmanager:GetSecretValue"] + resources = [aws_secretsmanager_secret.dms_audit_endpoint_target.arn] + } +} + +resource "aws_secretsmanager_secret_policy" "dms_audit_endpoint_target" { + secret_arn = aws_secretsmanager_secret.dms_audit_endpoint_target.arn + policy = data.aws_iam_policy_document.dms_audit_endpoint_target.json +} + diff --git a/terraform/environments/delius-core/modules/components/oracle_db_shared/iam.tf b/terraform/environments/delius-core/modules/components/oracle_db_shared/iam.tf index 59774d6743b..bb30726fcfc 100644 --- a/terraform/environments/delius-core/modules/components/oracle_db_shared/iam.tf +++ b/terraform/environments/delius-core/modules/components/oracle_db_shared/iam.tf @@ -297,3 +297,56 @@ resource "aws_iam_role_policy_attachment" "OracleEnterpriseManagementSecretsPoli role = aws_iam_role.EC2OracleEnterpriseManagementSecretsRole.name policy_arn = aws_iam_policy.OracleEnterpriseManagementSecretsPolicy.arn } + + + +# new IAM role OEM setup to allow DMS to access secrets manager and kms keys +resource "aws_iam_role" "DMSSecretsManagerAccessRole" { + name = "DMSSecretsManagerAccessRole" + + assume_role_policy = < Date: Tue, 18 Jun 2024 09:31:26 +0100 Subject: [PATCH 46/46] add healthcheck path --- terraform/environments/cdpt-ifs/loadbalancer.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/environments/cdpt-ifs/loadbalancer.tf b/terraform/environments/cdpt-ifs/loadbalancer.tf index cd81031ef78..51683cd9b6f 100644 --- a/terraform/environments/cdpt-ifs/loadbalancer.tf +++ b/terraform/environments/cdpt-ifs/loadbalancer.tf @@ -46,6 +46,7 @@ resource "aws_lb_target_group" "ifs_target_group" { unhealthy_threshold = "5" matcher = "200-499" timeout = "10" + path = "/health" } }