From c2ab4b7a4f74878bb4c9f33b0203d9c62a79693e Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 19 Dec 2023 14:39:16 +0000
Subject: [PATCH 01/51] create a fault injection simulator template the ecs app

---
 terraform/environment/global/iam_fis.tf       | 23 +++++
 terraform/environment/global/outputs.tf       |  1 +
 terraform/environment/refactor.tf             |  6 ++
 terraform/environment/region/ecs.tf           |  1 +
 .../modules/app/fault_injection_simulator.tf  | 83 +++++++++++++++++++
 .../region/modules/app/variables.tf           |  5 ++
 terraform/environment/region/variables.tf     |  1 +
 terraform/environment/regions.tf              |  2 +
 8 files changed, 122 insertions(+)
 create mode 100644 terraform/environment/global/iam_fis.tf
 create mode 100644 terraform/environment/region/modules/app/fault_injection_simulator.tf

diff --git a/terraform/environment/global/iam_fis.tf b/terraform/environment/global/iam_fis.tf
new file mode 100644
index 0000000000..44859bb3c8
--- /dev/null
+++ b/terraform/environment/global/iam_fis.tf
@@ -0,0 +1,23 @@
+resource "aws_iam_role" "fis" {
+  name               = "fis-${data.aws_default_tags.current.tags.environment-name}"
+  assume_role_policy = data.aws_iam_policy_document.lambda_assume.json
+  provider           = aws.global
+}
+
+data "aws_iam_policy_document" "fis_assume" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["fis.amazonaws.com"]
+    }
+  }
+  provider = aws.global
+}
+
+resource "aws_iam_role_policy_attachment" "fis" {
+  role       = aws_iam_role.fis.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSFaultInjectionSimulatorECSAccess"
+  provider   = aws.global
+}
diff --git a/terraform/environment/global/outputs.tf b/terraform/environment/global/outputs.tf
index 70303b1306..538a5abee7 100644
--- a/terraform/environment/global/outputs.tf
+++ b/terraform/environment/global/outputs.tf
@@ -8,5 +8,6 @@ output "iam_roles" {
     app_ecs_task_role  = aws_iam_role.app_task_role,
     s3_antivirus       = aws_iam_role.s3_antivirus,
     cross_account_put  = aws_iam_role.cross_account_put,
+    fis                = aws_iam_role.fis,
   }
 }
diff --git a/terraform/environment/refactor.tf b/terraform/environment/refactor.tf
index 96bd9fc35e..af2171b4df 100644
--- a/terraform/environment/refactor.tf
+++ b/terraform/environment/refactor.tf
@@ -67,3 +67,9 @@ moved {
   from = module.eu_west_1[0].module.events.aws_cloudwatch_event_archive.reduced_fees
   to   = module.eu_west_1[0].module.events.aws_cloudwatch_event_archive.main
 }
+
+import {
+  to       = aws_fis_experiment_template.template
+  id       = "EXT28aThorqMvmJp"
+  provider = aws.eu_west_1
+}
diff --git a/terraform/environment/region/ecs.tf b/terraform/environment/region/ecs.tf
index 90e957a9ae..0b595e3cb2 100644
--- a/terraform/environment/region/ecs.tf
+++ b/terraform/environment/region/ecs.tf
@@ -34,6 +34,7 @@ module "app" {
   app_allowed_api_arns            = concat(var.uid_service.api_arns, var.lpa_store_service.api_arns)
   ingress_allow_list_cidr         = concat(var.ingress_allow_list_cidr, split(",", data.aws_ssm_parameter.additional_allowed_ingress_cidrs.value))
   alb_deletion_protection_enabled = var.alb_deletion_protection_enabled
+  fis_role_arn                    = var.iam_roles.fis.arn
   lpas_table                      = var.lpas_table
   container_port                  = 8080
   public_access_enabled           = var.public_access_enabled
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
new file mode 100644
index 0000000000..2c11f7f319
--- /dev/null
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -0,0 +1,83 @@
+resource "aws_fis_experiment_template" "ecs_app" {
+  provider    = aws.region
+  description = "${data.aws_default_tags.current.tags.environment-name} - APP ECS Task Experiments"
+  role_arn    = var.fis_role_arn
+
+  action {
+    action_id   = "aws:ecs:stop-task"
+    description = null
+    name        = "stop_task"
+    start_after = ["wait_before_stop_task"]
+    target {
+      key   = "Tasks"
+      value = "${data.aws_default_tags.current.tags.environment-name}-app-ecs-tasks"
+    }
+  }
+
+  action {
+    action_id   = "aws:ecs:task-cpu-stress"
+    description = null
+    name        = "cpu_stress_100_percent"
+    start_after = ["wait"]
+    parameter {
+      key   = "duration"
+      value = "PT5M"
+    }
+    target {
+      key   = "Tasks"
+      value = "${data.aws_default_tags.current.tags.environment-name}-app-ecs-tasks"
+    }
+  }
+
+  action {
+    action_id   = "aws:ecs:task-network-latency"
+    description = null
+    name        = "ecs_network_latency"
+    start_after = []
+    parameter {
+      key   = "duration"
+      value = "PT5M"
+    }
+    target {
+      key   = "Tasks"
+      value = "${data.aws_default_tags.current.tags.environment-name}-app-ecs-tasks"
+    }
+  }
+
+  action {
+    action_id   = "aws:fis:wait"
+    description = null
+    name        = "wait"
+    start_after = ["stop_task"]
+    parameter {
+      key   = "duration"
+      value = "PT5M"
+    }
+  }
+
+  action {
+    action_id   = "aws:fis:wait"
+    description = null
+    name        = "wait_before_stop_task"
+    start_after = ["ecs_network_latency"]
+    parameter {
+      key   = "duration"
+      value = "PT5M"
+    }
+  }
+
+  stop_condition {
+    source = "none"
+    value  = null
+  }
+
+  target {
+    name = "${data.aws_default_tags.current.tags.environment-name}-app-ecs-tasks"
+    parameters = {
+      "cluster" : var.ecs_cluster,
+      "service" : aws_ecs_service.app.name,
+    }
+    resource_type  = "aws:ecs:task"
+    selection_mode = "ALL"
+  }
+}
diff --git a/terraform/environment/region/modules/app/variables.tf b/terraform/environment/region/modules/app/variables.tf
index 41fc5d69a2..902c148058 100644
--- a/terraform/environment/region/modules/app/variables.tf
+++ b/terraform/environment/region/modules/app/variables.tf
@@ -15,6 +15,11 @@ variable "ecs_task_role" {
   description = "ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services."
 }
 
+variable "fis_role_arn" {
+  type        = any
+  description = "ARN of IAM role that allows AWS FIS to make calls to other AWS services."
+}
+
 variable "ecs_cluster" {
   type        = string
   description = "ARN of an ECS cluster."
diff --git a/terraform/environment/region/variables.tf b/terraform/environment/region/variables.tf
index 6481693694..fced6786e9 100644
--- a/terraform/environment/region/variables.tf
+++ b/terraform/environment/region/variables.tf
@@ -8,6 +8,7 @@ variable "iam_roles" {
     app_ecs_task_role  = any
     s3_antivirus       = any
     cross_account_put  = any
+    fis                = any
   })
   description = "ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services."
 }
diff --git a/terraform/environment/regions.tf b/terraform/environment/regions.tf
index 1fe2ac0da9..fb490e106b 100644
--- a/terraform/environment/regions.tf
+++ b/terraform/environment/regions.tf
@@ -20,6 +20,7 @@ module "eu_west_1" {
     app_ecs_task_role  = module.global.iam_roles.app_ecs_task_role
     s3_antivirus       = module.global.iam_roles.s3_antivirus
     cross_account_put  = module.global.iam_roles.cross_account_put
+    fis                = module.global.iam_roles.fis
   }
   application_log_retention_days          = local.environment.cloudwatch_log_groups.application_log_retention_days
   ecs_capacity_provider                   = local.ecs_capacity_provider
@@ -76,6 +77,7 @@ module "eu_west_2" {
     app_ecs_task_role  = module.global.iam_roles.app_ecs_task_role
     s3_antivirus       = module.global.iam_roles.s3_antivirus
     cross_account_put  = module.global.iam_roles.cross_account_put
+    fis                = module.global.iam_roles.fis
   }
   application_log_retention_days          = local.environment.cloudwatch_log_groups.application_log_retention_days
   ecs_capacity_provider                   = local.ecs_capacity_provider

From 8be64711298ed6483089e4261f6a1f3ad48ad342 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 19 Dec 2023 14:45:02 +0000
Subject: [PATCH 02/51] don't import existing one anymore

---
 terraform/environment/refactor.tf | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/terraform/environment/refactor.tf b/terraform/environment/refactor.tf
index af2171b4df..96bd9fc35e 100644
--- a/terraform/environment/refactor.tf
+++ b/terraform/environment/refactor.tf
@@ -67,9 +67,3 @@ moved {
   from = module.eu_west_1[0].module.events.aws_cloudwatch_event_archive.reduced_fees
   to   = module.eu_west_1[0].module.events.aws_cloudwatch_event_archive.main
 }
-
-import {
-  to       = aws_fis_experiment_template.template
-  id       = "EXT28aThorqMvmJp"
-  provider = aws.eu_west_1
-}

From ba940cc6f1cc8b87785d0f648081eb73c0e88255 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 19 Dec 2023 14:49:38 +0000
Subject: [PATCH 03/51] expand abberv

---
 ...is.tf => iam_fault_injection_simulator.tf} | 10 +++---
 terraform/environment/global/outputs.tf       | 10 +++---
 terraform/environment/region/ecs.tf           | 34 +++++++++----------
 .../modules/app/fault_injection_simulator.tf  |  2 +-
 .../region/modules/app/variables.tf           |  2 +-
 terraform/environment/region/variables.tf     | 10 +++---
 terraform/environment/regions.tf              | 20 +++++------
 7 files changed, 44 insertions(+), 44 deletions(-)
 rename terraform/environment/global/{iam_fis.tf => iam_fault_injection_simulator.tf} (52%)

diff --git a/terraform/environment/global/iam_fis.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
similarity index 52%
rename from terraform/environment/global/iam_fis.tf
rename to terraform/environment/global/iam_fault_injection_simulator.tf
index 44859bb3c8..35d9d17ab8 100644
--- a/terraform/environment/global/iam_fis.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -1,10 +1,10 @@
-resource "aws_iam_role" "fis" {
-  name               = "fis-${data.aws_default_tags.current.tags.environment-name}"
+resource "aws_iam_role" "fault_injection_simulator" {
+  name               = "fault-injection-simulator-${data.aws_default_tags.current.tags.environment-name}"
   assume_role_policy = data.aws_iam_policy_document.lambda_assume.json
   provider           = aws.global
 }
 
-data "aws_iam_policy_document" "fis_assume" {
+data "aws_iam_policy_document" "fault_injection_simulator_assume" {
   statement {
     actions = ["sts:AssumeRole"]
 
@@ -16,8 +16,8 @@ data "aws_iam_policy_document" "fis_assume" {
   provider = aws.global
 }
 
-resource "aws_iam_role_policy_attachment" "fis" {
-  role       = aws_iam_role.fis.name
+resource "aws_iam_role_policy_attachment" "fault_injection_simulator" {
+  role       = aws_iam_role.fault_injection_simulator.name
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSFaultInjectionSimulatorECSAccess"
   provider   = aws.global
 }
diff --git a/terraform/environment/global/outputs.tf b/terraform/environment/global/outputs.tf
index 538a5abee7..7cafcd989c 100644
--- a/terraform/environment/global/outputs.tf
+++ b/terraform/environment/global/outputs.tf
@@ -4,10 +4,10 @@ output "resource_group_arn" {
 
 output "iam_roles" {
   value = {
-    ecs_execution_role = aws_iam_role.execution_role,
-    app_ecs_task_role  = aws_iam_role.app_task_role,
-    s3_antivirus       = aws_iam_role.s3_antivirus,
-    cross_account_put  = aws_iam_role.cross_account_put,
-    fis                = aws_iam_role.fis,
+    ecs_execution_role        = aws_iam_role.execution_role,
+    app_ecs_task_role         = aws_iam_role.app_task_role,
+    s3_antivirus              = aws_iam_role.s3_antivirus,
+    cross_account_put         = aws_iam_role.cross_account_put,
+    fault_injection_simulator = aws_iam_role.fault_injection_simulator,
   }
 }
diff --git a/terraform/environment/region/ecs.tf b/terraform/environment/region/ecs.tf
index 0b595e3cb2..45384705ad 100644
--- a/terraform/environment/region/ecs.tf
+++ b/terraform/environment/region/ecs.tf
@@ -21,23 +21,23 @@ data "aws_ssm_parameter" "additional_allowed_ingress_cidrs" {
 }
 
 module "app" {
-  source                          = "./modules/app"
-  ecs_cluster                     = aws_ecs_cluster.main.id
-  ecs_execution_role              = var.iam_roles.ecs_execution_role
-  ecs_task_role                   = var.iam_roles.app_ecs_task_role
-  ecs_service_desired_count       = 1
-  ecs_application_log_group_name  = module.application_logs.cloudwatch_log_group.name
-  ecs_capacity_provider           = var.ecs_capacity_provider
-  app_env_vars                    = var.app_env_vars
-  app_service_repository_url      = var.app_service_repository_url
-  app_service_container_version   = var.app_service_container_version
-  app_allowed_api_arns            = concat(var.uid_service.api_arns, var.lpa_store_service.api_arns)
-  ingress_allow_list_cidr         = concat(var.ingress_allow_list_cidr, split(",", data.aws_ssm_parameter.additional_allowed_ingress_cidrs.value))
-  alb_deletion_protection_enabled = var.alb_deletion_protection_enabled
-  fis_role_arn                    = var.iam_roles.fis.arn
-  lpas_table                      = var.lpas_table
-  container_port                  = 8080
-  public_access_enabled           = var.public_access_enabled
+  source                             = "./modules/app"
+  ecs_cluster                        = aws_ecs_cluster.main.id
+  ecs_execution_role                 = var.iam_roles.ecs_execution_role
+  ecs_task_role                      = var.iam_roles.app_ecs_task_role
+  ecs_service_desired_count          = 1
+  ecs_application_log_group_name     = module.application_logs.cloudwatch_log_group.name
+  ecs_capacity_provider              = var.ecs_capacity_provider
+  app_env_vars                       = var.app_env_vars
+  app_service_repository_url         = var.app_service_repository_url
+  app_service_container_version      = var.app_service_container_version
+  app_allowed_api_arns               = concat(var.uid_service.api_arns, var.lpa_store_service.api_arns)
+  ingress_allow_list_cidr            = concat(var.ingress_allow_list_cidr, split(",", data.aws_ssm_parameter.additional_allowed_ingress_cidrs.value))
+  alb_deletion_protection_enabled    = var.alb_deletion_protection_enabled
+  fault_injection_simulator_role_arn = var.iam_roles.fault_injection_simulator.arn
+  lpas_table                         = var.lpas_table
+  container_port                     = 8080
+  public_access_enabled              = var.public_access_enabled
   network = {
     vpc_id              = data.aws_vpc.main.id
     application_subnets = data.aws_subnet.application[*].id
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 2c11f7f319..7ae8f3c0c8 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -1,7 +1,7 @@
 resource "aws_fis_experiment_template" "ecs_app" {
   provider    = aws.region
   description = "${data.aws_default_tags.current.tags.environment-name} - APP ECS Task Experiments"
-  role_arn    = var.fis_role_arn
+  role_arn    = var.fault_injection_simulator_role_arn
 
   action {
     action_id   = "aws:ecs:stop-task"
diff --git a/terraform/environment/region/modules/app/variables.tf b/terraform/environment/region/modules/app/variables.tf
index 902c148058..be35c14146 100644
--- a/terraform/environment/region/modules/app/variables.tf
+++ b/terraform/environment/region/modules/app/variables.tf
@@ -15,7 +15,7 @@ variable "ecs_task_role" {
   description = "ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services."
 }
 
-variable "fis_role_arn" {
+variable "fault_injection_simulator_role_arn" {
   type        = any
   description = "ARN of IAM role that allows AWS FIS to make calls to other AWS services."
 }
diff --git a/terraform/environment/region/variables.tf b/terraform/environment/region/variables.tf
index fced6786e9..e719745134 100644
--- a/terraform/environment/region/variables.tf
+++ b/terraform/environment/region/variables.tf
@@ -4,11 +4,11 @@ locals {
 
 variable "iam_roles" {
   type = object({
-    ecs_execution_role = any
-    app_ecs_task_role  = any
-    s3_antivirus       = any
-    cross_account_put  = any
-    fis                = any
+    ecs_execution_role        = any
+    app_ecs_task_role         = any
+    s3_antivirus              = any
+    cross_account_put         = any
+    fault_injection_simulator = any
   })
   description = "ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services."
 }
diff --git a/terraform/environment/regions.tf b/terraform/environment/regions.tf
index fb490e106b..c2845bf33e 100644
--- a/terraform/environment/regions.tf
+++ b/terraform/environment/regions.tf
@@ -16,11 +16,11 @@ module "eu_west_1" {
   source = "./region"
   count  = contains(local.environment.regions, "eu-west-1") ? 1 : 0
   iam_roles = {
-    ecs_execution_role = module.global.iam_roles.ecs_execution_role
-    app_ecs_task_role  = module.global.iam_roles.app_ecs_task_role
-    s3_antivirus       = module.global.iam_roles.s3_antivirus
-    cross_account_put  = module.global.iam_roles.cross_account_put
-    fis                = module.global.iam_roles.fis
+    ecs_execution_role        = module.global.iam_roles.ecs_execution_role
+    app_ecs_task_role         = module.global.iam_roles.app_ecs_task_role
+    s3_antivirus              = module.global.iam_roles.s3_antivirus
+    cross_account_put         = module.global.iam_roles.cross_account_put
+    fault_injection_simulator = module.global.iam_roles.fault_injection_simulator
   }
   application_log_retention_days          = local.environment.cloudwatch_log_groups.application_log_retention_days
   ecs_capacity_provider                   = local.ecs_capacity_provider
@@ -73,11 +73,11 @@ module "eu_west_2" {
   source = "./region"
   count  = contains(local.environment.regions, "eu-west-2") ? 1 : 0
   iam_roles = {
-    ecs_execution_role = module.global.iam_roles.ecs_execution_role
-    app_ecs_task_role  = module.global.iam_roles.app_ecs_task_role
-    s3_antivirus       = module.global.iam_roles.s3_antivirus
-    cross_account_put  = module.global.iam_roles.cross_account_put
-    fis                = module.global.iam_roles.fis
+    ecs_execution_role        = module.global.iam_roles.ecs_execution_role
+    app_ecs_task_role         = module.global.iam_roles.app_ecs_task_role
+    s3_antivirus              = module.global.iam_roles.s3_antivirus
+    cross_account_put         = module.global.iam_roles.cross_account_put
+    fault_injection_simulator = module.global.iam_roles.fault_injection_simulator
   }
   application_log_retention_days          = local.environment.cloudwatch_log_groups.application_log_retention_days
   ecs_capacity_provider                   = local.ecs_capacity_provider

From 3e37c33fab35b5a306874ade55559a3068cd3821 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 19 Dec 2023 15:01:51 +0000
Subject: [PATCH 04/51] target names must start with a letter

---
 .../region/modules/app/fault_injection_simulator.tf  | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 7ae8f3c0c8..decccdda9e 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -10,7 +10,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
     start_after = ["wait_before_stop_task"]
     target {
       key   = "Tasks"
-      value = "${data.aws_default_tags.current.tags.environment-name}-app-ecs-tasks"
+      value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
     }
   }
 
@@ -25,7 +25,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
     }
     target {
       key   = "Tasks"
-      value = "${data.aws_default_tags.current.tags.environment-name}-app-ecs-tasks"
+      value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
     }
   }
 
@@ -40,7 +40,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
     }
     target {
       key   = "Tasks"
-      value = "${data.aws_default_tags.current.tags.environment-name}-app-ecs-tasks"
+      value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
     }
   }
 
@@ -72,11 +72,15 @@ resource "aws_fis_experiment_template" "ecs_app" {
   }
 
   target {
-    name = "${data.aws_default_tags.current.tags.environment-name}-app-ecs-tasks"
+    name = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
     parameters = {
       "cluster" : var.ecs_cluster,
       "service" : aws_ecs_service.app.name,
     }
+    resource_tag {
+      key   = "aws:ecs:cluster-name"
+      value = var.ecs_cluster
+    }
     resource_type  = "aws:ecs:task"
     selection_mode = "ALL"
   }

From e66e85abffe56514b895e9226377fa6bcda2eaa2 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 19 Dec 2023 15:03:45 +0000
Subject: [PATCH 05/51] add name

---
 .../region/modules/app/fault_injection_simulator.tf          | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index decccdda9e..236816fe0a 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -1,7 +1,10 @@
 resource "aws_fis_experiment_template" "ecs_app" {
   provider    = aws.region
-  description = "${data.aws_default_tags.current.tags.environment-name} - APP ECS Task Experiments"
+  description = "Run ECS task experiments for the app service"
   role_arn    = var.fault_injection_simulator_role_arn
+  tags = {
+    Name = "${data.aws_default_tags.current.tags.environment-name} - APP ECS Task Experiments"
+  }
 
   action {
     action_id   = "aws:ecs:stop-task"

From 3832a4b14d49c62bc2c9d6c579450438b2a3a5cd Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 19 Dec 2023 15:08:12 +0000
Subject: [PATCH 06/51] never create experiments for production

---
 .../environment/region/modules/app/fault_injection_simulator.tf  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 236816fe0a..859cd59c05 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -1,4 +1,5 @@
 resource "aws_fis_experiment_template" "ecs_app" {
+  count       = data.aws_default_tags.current.tags.environment-name == "production" ? 0 : 1
   provider    = aws.region
   description = "Run ECS task experiments for the app service"
   role_arn    = var.fault_injection_simulator_role_arn

From 1108a7db86542415654829a49c886dbc904f5ba0 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 19 Dec 2023 15:22:40 +0000
Subject: [PATCH 07/51] add logging

---
 .../modules/app/fault_injection_simulator.tf  | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 859cd59c05..dfdcacfdc8 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -1,3 +1,15 @@
+data "aws_kms_alias" "cloudwatch_application_logs_encryption" {
+  name     = "alias/${data.aws_default_tags.current.tags.application}_cloudwatch_application_logs_encryption"
+  provider = aws.region
+}
+
+resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
+  name              = "/aws/fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}"
+  retention_in_days = 7
+  kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
+  provider          = aws.region
+}
+
 resource "aws_fis_experiment_template" "ecs_app" {
   count       = data.aws_default_tags.current.tags.environment-name == "production" ? 0 : 1
   provider    = aws.region
@@ -69,6 +81,13 @@ resource "aws_fis_experiment_template" "ecs_app" {
       value = "PT5M"
     }
   }
+  log_configuration {
+    log_schema_version = 2
+
+    cloudwatch_logs_configuration {
+      log_group_arn = "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
+    }
+  }
 
   stop_condition {
     source = "none"

From 37a7c33a27badc928372edf781806098ef70aa0e Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 19 Dec 2023 15:36:26 +0000
Subject: [PATCH 08/51] fix assume role

---
 terraform/environment/global/iam_fault_injection_simulator.tf   | 2 +-
 .../environment/region/modules/app/fault_injection_simulator.tf | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index 35d9d17ab8..bb5d2e7c8d 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -1,6 +1,6 @@
 resource "aws_iam_role" "fault_injection_simulator" {
   name               = "fault-injection-simulator-${data.aws_default_tags.current.tags.environment-name}"
-  assume_role_policy = data.aws_iam_policy_document.lambda_assume.json
+  assume_role_policy = data.aws_iam_policy_document.fault_injection_simulator_assume.json
   provider           = aws.global
 }
 
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index dfdcacfdc8..c42ce38b1c 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -81,6 +81,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
       value = "PT5M"
     }
   }
+
   log_configuration {
     log_schema_version = 2
 

From d12692bc5f795ce83acee345f71fe538c870ac54 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 20 Dec 2023 12:37:31 +0000
Subject: [PATCH 09/51] add some permissions, remove logs and log encryption

---
 .../global/iam_fault_injection_simulator.tf   | 34 ++++++++
 .../environment/region/modules/app/ecs.tf     |  1 +
 .../modules/app/fault_injection_simulator.tf  | 81 ++++---------------
 3 files changed, 49 insertions(+), 67 deletions(-)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index bb5d2e7c8d..474b1cb12c 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -21,3 +21,37 @@ resource "aws_iam_role_policy_attachment" "fault_injection_simulator" {
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSFaultInjectionSimulatorECSAccess"
   provider   = aws.global
 }
+
+resource "aws_iam_role_policy" "additional_permissions" {
+  name     = "additional-permissions"
+  role     = aws_iam_role.fault_injection_simulator.name
+  policy   = data.aws_iam_policy_document.additional_permissions.json
+  provider = aws.global
+}
+
+data "aws_iam_policy_document" "additional_permissions" {
+
+  statement {
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "iam:CreateServiceLinkedRole",
+    ]
+    condition {
+      test     = "StringLike"
+      variable = "iam:AWSServiceName"
+      values   = ["fis.amazonaws.com"]
+    }
+  }
+
+  statement {
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "logs:CreateLogDelivery",
+      "logs:DescribeLogGroups",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+    ]
+  }
+}
diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index 30f5ed7996..b4577dd071 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -32,6 +32,7 @@ resource "aws_ecs_service" "app" {
     create = "7m"
     update = "4m"
   }
+
   provider = aws.region
 }
 
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index c42ce38b1c..96111a8f6a 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -6,8 +6,8 @@ data "aws_kms_alias" "cloudwatch_application_logs_encryption" {
 resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
   name              = "/aws/fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}"
   retention_in_days = 7
-  kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
-  provider          = aws.region
+  # kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
+  provider = aws.region
 }
 
 resource "aws_fis_experiment_template" "ecs_app" {
@@ -19,37 +19,10 @@ resource "aws_fis_experiment_template" "ecs_app" {
     Name = "${data.aws_default_tags.current.tags.environment-name} - APP ECS Task Experiments"
   }
 
-  action {
-    action_id   = "aws:ecs:stop-task"
-    description = null
-    name        = "stop_task"
-    start_after = ["wait_before_stop_task"]
-    target {
-      key   = "Tasks"
-      value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
-    }
-  }
-
   action {
     action_id   = "aws:ecs:task-cpu-stress"
     description = null
     name        = "cpu_stress_100_percent"
-    start_after = ["wait"]
-    parameter {
-      key   = "duration"
-      value = "PT5M"
-    }
-    target {
-      key   = "Tasks"
-      value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
-    }
-  }
-
-  action {
-    action_id   = "aws:ecs:task-network-latency"
-    description = null
-    name        = "ecs_network_latency"
-    start_after = []
     parameter {
       key   = "duration"
       value = "PT5M"
@@ -60,36 +33,6 @@ resource "aws_fis_experiment_template" "ecs_app" {
     }
   }
 
-  action {
-    action_id   = "aws:fis:wait"
-    description = null
-    name        = "wait"
-    start_after = ["stop_task"]
-    parameter {
-      key   = "duration"
-      value = "PT5M"
-    }
-  }
-
-  action {
-    action_id   = "aws:fis:wait"
-    description = null
-    name        = "wait_before_stop_task"
-    start_after = ["ecs_network_latency"]
-    parameter {
-      key   = "duration"
-      value = "PT5M"
-    }
-  }
-
-  log_configuration {
-    log_schema_version = 2
-
-    cloudwatch_logs_configuration {
-      log_group_arn = "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
-    }
-  }
-
   stop_condition {
     source = "none"
     value  = null
@@ -97,14 +40,18 @@ resource "aws_fis_experiment_template" "ecs_app" {
 
   target {
     name = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
-    parameters = {
-      "cluster" : var.ecs_cluster,
-      "service" : aws_ecs_service.app.name,
-    }
-    resource_tag {
-      key   = "aws:ecs:cluster-name"
-      value = var.ecs_cluster
-    }
+    resource_arns = [
+      "arn:aws:ecs:eu-west-1:653761790766:task/936mlpab157-eu-west-1/1b1aedd43f94458d9ef3475479e19169",
+      # aws_ecs_task_definition.mock_onelogin.arn,
+    ]
+    # parameters = {
+    #   "cluster" : var.ecs_cluster,
+    #   "service" : aws_ecs_service.app.name,
+    # }
+    # resource_tag {
+    #   key   = "aws:ecs:service"
+    #   value = aws_ecs_service.app.name
+    # }
     resource_type  = "aws:ecs:task"
     selection_mode = "ALL"
   }

From 5a0625607362d6a6ac4f5d37d2e31c237ca101ec Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 20 Dec 2023 13:58:54 +0000
Subject: [PATCH 10/51] add ssm agent container

---
 .../environment/region/modules/app/ecs.tf     |  2 +-
 .../modules/app/fault_injection_simulator.tf  | 36 +++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index b4577dd071..2ea4bf6f4b 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -84,7 +84,7 @@ resource "aws_ecs_task_definition" "app" {
     operating_system_family = "LINUX"
     cpu_architecture        = "X86_64"
   }
-  container_definitions = "[${local.app}, ${local.aws_otel_collector}]"
+  container_definitions = "[${local.app}, ${local.aws_otel_collector}, ${local.amazon_ssm_agent}]"
   task_role_arn         = var.ecs_task_role.arn
   execution_role_arn    = var.ecs_execution_role.arn
   provider              = aws.region
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 96111a8f6a..4a99b31096 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -56,3 +56,39 @@ resource "aws_fis_experiment_template" "ecs_app" {
     selection_mode = "ALL"
   }
 }
+
+locals {
+  amazon_ssm_agent = jsonencode(
+    {
+      name         = "amazon-ssm-agent",
+      image        = "public.ecr.aws/amazon-ssm-agent/amazon-ssm-agent:latest",
+      cpu          = 0,
+      links        = [],
+      portMappings = [],
+      essential    = false,
+      entryPoint   = [],
+      command = [
+        "/bin/bash",
+        "-c",
+        "set -e; yum upgrade -y; yum install jq procps awscli -y; term_handler() { echo \"Deleting SSM activation $ACTIVATION_ID\"; if ! aws ssm delete-activation --activation-id $ACTIVATION_ID --region $ECS_TASK_REGION; then echo \"SSM activation $ACTIVATION_ID failed to be deleted\" 1>&2; fi; MANAGED_INSTANCE_ID=$(jq -e -r .ManagedInstanceID /var/lib/amazon/ssm/registration); echo \"Deregistering SSM Managed Instance $MANAGED_INSTANCE_ID\"; if ! aws ssm deregister-managed-instance --instance-id $MANAGED_INSTANCE_ID --region $ECS_TASK_REGION; then echo \"SSM Managed Instance $MANAGED_INSTANCE_ID failed to be deregistered\" 1>&2; fi; kill -SIGTERM $SSM_AGENT_PID; }; trap term_handler SIGTERM SIGINT; if [[ -z $MANAGED_INSTANCE_ROLE_NAME ]]; then echo \"Environment variable MANAGED_INSTANCE_ROLE_NAME not set, exiting\" 1>&2; exit 1; fi; if ! ps ax | grep amazon-ssm-agent | grep -v grep > /dev/null; then if [[ -n $ECS_CONTAINER_METADATA_URI_V4 ]] ; then echo \"Found ECS Container Metadata, running activation with metadata\"; TASK_METADATA=$(curl \"${ECS_CONTAINER_METADATA_URI_V4}/task\"); ECS_TASK_AVAILABILITY_ZONE=$(echo $TASK_METADATA | jq -e -r '.AvailabilityZone'); ECS_TASK_ARN=$(echo $TASK_METADATA | jq -e -r '.TaskARN'); ECS_TASK_REGION=$(echo $ECS_TASK_AVAILABILITY_ZONE | sed 's/.$//'); ECS_TASK_AVAILABILITY_ZONE_REGEX='^(af|ap|ca|cn|eu|me|sa|us|us-gov)-(central|north|(north(east|west))|south|south(east|west)|east|west)-[0-9]{1}[a-z]{1}$'; if ! [[ $ECS_TASK_AVAILABILITY_ZONE =~ $ECS_TASK_AVAILABILITY_ZONE_REGEX ]]; then echo \"Error extracting Availability Zone from ECS Container Metadata, exiting\" 1>&2; exit 1; fi; ECS_TASK_ARN_REGEX='^arn:(aws|aws-cn|aws-us-gov):ecs:[a-z0-9-]+:[0-9]{12}:task/[a-zA-Z0-9_-]+/[a-zA-Z0-9]+$'; if ! [[ $ECS_TASK_ARN =~ $ECS_TASK_ARN_REGEX ]]; then echo \"Error extracting Task ARN from ECS Container Metadata, exiting\" 1>&2; exit 1; fi; CREATE_ACTIVATION_OUTPUT=$(aws ssm create-activation --iam-role $MANAGED_INSTANCE_ROLE_NAME --tags Key=ECS_TASK_AVAILABILITY_ZONE,Value=$ECS_TASK_AVAILABILITY_ZONE Key=ECS_TASK_ARN,Value=$ECS_TASK_ARN Key=FAULT_INJECTION_SIDECAR,Value=true --region $ECS_TASK_REGION); ACTIVATION_CODE=$(echo $CREATE_ACTIVATION_OUTPUT | jq -e -r .ActivationCode); ACTIVATION_ID=$(echo $CREATE_ACTIVATION_OUTPUT | jq -e -r .ActivationId); if ! amazon-ssm-agent -register -code $ACTIVATION_CODE -id $ACTIVATION_ID -region $ECS_TASK_REGION; then echo \"Failed to register with AWS Systems Manager (SSM), exiting\" 1>&2; exit 1; fi; amazon-ssm-agent & SSM_AGENT_PID=$!; wait $SSM_AGENT_PID; else echo \"ECS Container Metadata not found, exiting\" 1>&2; exit 1; fi; else echo \"SSM agent is already running, exiting\" 1>&2; exit 1; fi"
+      ],
+      environment = [
+        {
+          name  = "MANAGED_INSTANCE_ROLE_NAME",
+          value = "SSMManagedInstanceRole"
+        }
+      ],
+      environmentFiles      = [],
+      mountPoints           = [],
+      volumesFrom           = [],
+      secrets               = [],
+      dnsServers            = [],
+      dnsSearchDomains      = [],
+      extraHosts            = [],
+      dockerSecurityOptions = [],
+      dockerLabels          = {},
+      ulimits               = [],
+      logConfiguration      = {},
+      systemControls        = []
+  })
+}

From 201be00cd05a532519cb1e7eed3cd3638895cc15 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 20 Dec 2023 16:29:29 +0000
Subject: [PATCH 11/51] add ssm agent to task

---
 terraform/environment/region/modules/app/ecs.tf       |  1 +
 .../region/modules/app/fault_injection_simulator.tf   | 11 +++++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index 2ea4bf6f4b..9022a85b9c 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -84,6 +84,7 @@ resource "aws_ecs_task_definition" "app" {
     operating_system_family = "LINUX"
     cpu_architecture        = "X86_64"
   }
+  # conditionally add ssm agent container
   container_definitions = "[${local.app}, ${local.aws_otel_collector}, ${local.amazon_ssm_agent}]"
   task_role_arn         = var.ecs_task_role.arn
   execution_role_arn    = var.ecs_execution_role.arn
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 4a99b31096..7a73fc23f2 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -70,7 +70,7 @@ locals {
       command = [
         "/bin/bash",
         "-c",
-        "set -e; yum upgrade -y; yum install jq procps awscli -y; term_handler() { echo \"Deleting SSM activation $ACTIVATION_ID\"; if ! aws ssm delete-activation --activation-id $ACTIVATION_ID --region $ECS_TASK_REGION; then echo \"SSM activation $ACTIVATION_ID failed to be deleted\" 1>&2; fi; MANAGED_INSTANCE_ID=$(jq -e -r .ManagedInstanceID /var/lib/amazon/ssm/registration); echo \"Deregistering SSM Managed Instance $MANAGED_INSTANCE_ID\"; if ! aws ssm deregister-managed-instance --instance-id $MANAGED_INSTANCE_ID --region $ECS_TASK_REGION; then echo \"SSM Managed Instance $MANAGED_INSTANCE_ID failed to be deregistered\" 1>&2; fi; kill -SIGTERM $SSM_AGENT_PID; }; trap term_handler SIGTERM SIGINT; if [[ -z $MANAGED_INSTANCE_ROLE_NAME ]]; then echo \"Environment variable MANAGED_INSTANCE_ROLE_NAME not set, exiting\" 1>&2; exit 1; fi; if ! ps ax | grep amazon-ssm-agent | grep -v grep > /dev/null; then if [[ -n $ECS_CONTAINER_METADATA_URI_V4 ]] ; then echo \"Found ECS Container Metadata, running activation with metadata\"; TASK_METADATA=$(curl \"${ECS_CONTAINER_METADATA_URI_V4}/task\"); ECS_TASK_AVAILABILITY_ZONE=$(echo $TASK_METADATA | jq -e -r '.AvailabilityZone'); ECS_TASK_ARN=$(echo $TASK_METADATA | jq -e -r '.TaskARN'); ECS_TASK_REGION=$(echo $ECS_TASK_AVAILABILITY_ZONE | sed 's/.$//'); ECS_TASK_AVAILABILITY_ZONE_REGEX='^(af|ap|ca|cn|eu|me|sa|us|us-gov)-(central|north|(north(east|west))|south|south(east|west)|east|west)-[0-9]{1}[a-z]{1}$'; if ! [[ $ECS_TASK_AVAILABILITY_ZONE =~ $ECS_TASK_AVAILABILITY_ZONE_REGEX ]]; then echo \"Error extracting Availability Zone from ECS Container Metadata, exiting\" 1>&2; exit 1; fi; ECS_TASK_ARN_REGEX='^arn:(aws|aws-cn|aws-us-gov):ecs:[a-z0-9-]+:[0-9]{12}:task/[a-zA-Z0-9_-]+/[a-zA-Z0-9]+$'; if ! [[ $ECS_TASK_ARN =~ $ECS_TASK_ARN_REGEX ]]; then echo \"Error extracting Task ARN from ECS Container Metadata, exiting\" 1>&2; exit 1; fi; CREATE_ACTIVATION_OUTPUT=$(aws ssm create-activation --iam-role $MANAGED_INSTANCE_ROLE_NAME --tags Key=ECS_TASK_AVAILABILITY_ZONE,Value=$ECS_TASK_AVAILABILITY_ZONE Key=ECS_TASK_ARN,Value=$ECS_TASK_ARN Key=FAULT_INJECTION_SIDECAR,Value=true --region $ECS_TASK_REGION); ACTIVATION_CODE=$(echo $CREATE_ACTIVATION_OUTPUT | jq -e -r .ActivationCode); ACTIVATION_ID=$(echo $CREATE_ACTIVATION_OUTPUT | jq -e -r .ActivationId); if ! amazon-ssm-agent -register -code $ACTIVATION_CODE -id $ACTIVATION_ID -region $ECS_TASK_REGION; then echo \"Failed to register with AWS Systems Manager (SSM), exiting\" 1>&2; exit 1; fi; amazon-ssm-agent & SSM_AGENT_PID=$!; wait $SSM_AGENT_PID; else echo \"ECS Container Metadata not found, exiting\" 1>&2; exit 1; fi; else echo \"SSM agent is already running, exiting\" 1>&2; exit 1; fi"
+        "set -e; yum upgrade -y; yum install jq procps awscli -y; term_handler() { echo \"Deleting SSM activation $ACTIVATION_ID\"; if ! aws ssm delete-activation --activation-id $ACTIVATION_ID --region $ECS_TASK_REGION; then echo \"SSM activation $ACTIVATION_ID failed to be deleted\" 1>&2; fi; MANAGED_INSTANCE_ID=$(jq -e -r .ManagedInstanceID /var/lib/amazon/ssm/registration); echo \"Deregistering SSM Managed Instance $MANAGED_INSTANCE_ID\"; if ! aws ssm deregister-managed-instance --instance-id $MANAGED_INSTANCE_ID --region $ECS_TASK_REGION; then echo \"SSM Managed Instance $MANAGED_INSTANCE_ID failed to be deregistered\" 1>&2; fi; kill -SIGTERM $SSM_AGENT_PID; }; trap term_handler SIGTERM SIGINT; if [[ -z $MANAGED_INSTANCE_ROLE_NAME ]]; then echo \"Environment variable MANAGED_INSTANCE_ROLE_NAME not set, exiting\" 1>&2; exit 1; fi; if ! ps ax | grep amazon-ssm-agent | grep -v grep > /dev/null; then if [[ -n $ECS_CONTAINER_METADATA_URI_V4 ]] ; then echo \"Found ECS Container Metadata, running activation with metadata\"; TASK_METADATA=$(curl \"$${ECS_CONTAINER_METADATA_URI_V4}/task\"); ECS_TASK_AVAILABILITY_ZONE=$(echo $TASK_METADATA | jq -e -r '.AvailabilityZone'); ECS_TASK_ARN=$(echo $TASK_METADATA | jq -e -r '.TaskARN'); ECS_TASK_REGION=$(echo $ECS_TASK_AVAILABILITY_ZONE | sed 's/.$//'); ECS_TASK_AVAILABILITY_ZONE_REGEX='^(af|ap|ca|cn|eu|me|sa|us|us-gov)-(central|north|(north(east|west))|south|south(east|west)|east|west)-[0-9]{1}[a-z]{1}$'; if ! [[ $ECS_TASK_AVAILABILITY_ZONE =~ $ECS_TASK_AVAILABILITY_ZONE_REGEX ]]; then echo \"Error extracting Availability Zone from ECS Container Metadata, exiting\" 1>&2; exit 1; fi; ECS_TASK_ARN_REGEX='^arn:(aws|aws-cn|aws-us-gov):ecs:[a-z0-9-]+:[0-9]{12}:task/[a-zA-Z0-9_-]+/[a-zA-Z0-9]+$'; if ! [[ $ECS_TASK_ARN =~ $ECS_TASK_ARN_REGEX ]]; then echo \"Error extracting Task ARN from ECS Container Metadata, exiting\" 1>&2; exit 1; fi; CREATE_ACTIVATION_OUTPUT=$(aws ssm create-activation --iam-role $MANAGED_INSTANCE_ROLE_NAME --tags Key=ECS_TASK_AVAILABILITY_ZONE,Value=$ECS_TASK_AVAILABILITY_ZONE Key=ECS_TASK_ARN,Value=$ECS_TASK_ARN Key=FAULT_INJECTION_SIDECAR,Value=true --region $ECS_TASK_REGION); ACTIVATION_CODE=$(echo $CREATE_ACTIVATION_OUTPUT | jq -e -r .ActivationCode); ACTIVATION_ID=$(echo $CREATE_ACTIVATION_OUTPUT | jq -e -r .ActivationId); if ! amazon-ssm-agent -register -code $ACTIVATION_CODE -id $ACTIVATION_ID -region $ECS_TASK_REGION; then echo \"Failed to register with AWS Systems Manager (SSM), exiting\" 1>&2; exit 1; fi; amazon-ssm-agent & SSM_AGENT_PID=$!; wait $SSM_AGENT_PID; else echo \"ECS Container Metadata not found, exiting\" 1>&2; exit 1; fi; else echo \"SSM agent is already running, exiting\" 1>&2; exit 1; fi"
       ],
       environment = [
         {
@@ -78,6 +78,14 @@ locals {
           value = "SSMManagedInstanceRole"
         }
       ],
+      logConfiguration = {
+        logDriver = "awslogs",
+        options = {
+          awslogs-group         = var.ecs_application_log_group_name,
+          awslogs-region        = data.aws_region.current.name,
+          awslogs-stream-prefix = "${data.aws_default_tags.current.tags.environment-name}.otel.app"
+        }
+      },
       environmentFiles      = [],
       mountPoints           = [],
       volumesFrom           = [],
@@ -88,7 +96,6 @@ locals {
       dockerSecurityOptions = [],
       dockerLabels          = {},
       ulimits               = [],
-      logConfiguration      = {},
       systemControls        = []
   })
 }

From 4650fd7cc71fe33002db4832f7c7063cc1d035d6 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 20 Dec 2023 16:39:39 +0000
Subject: [PATCH 12/51] add iam pre-reqs for fis

---
 .../global/iam_fault_injection_simulator.tf   | 12 +++++++++++
 .../environment/region/modules/app/ecs.tf     |  8 +++++--
 .../modules/app/fault_injection_simulator.tf  | 21 +++++++++++++++++++
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index 474b1cb12c..a1112f8ae2 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -32,6 +32,7 @@ resource "aws_iam_role_policy" "additional_permissions" {
 data "aws_iam_policy_document" "additional_permissions" {
 
   statement {
+    sid       = "AllowServiceLinkedRole"
     effect    = "Allow"
     resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
     actions = [
@@ -45,6 +46,7 @@ data "aws_iam_policy_document" "additional_permissions" {
   }
 
   statement {
+    sid       = "AllowCloudWatchLogs"
     effect    = "Allow"
     resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
     actions = [
@@ -54,4 +56,14 @@ data "aws_iam_policy_document" "additional_permissions" {
       "logs:PutLogEvents",
     ]
   }
+  statement {
+    sid       = "AllowSSMCommands"
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "ssm:SendCommand",
+      "ssm:ListCommands",
+      "ssm:CancelCommand",
+    ]
+  }
 }
diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index 9022a85b9c..0e5702a8ba 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -92,8 +92,12 @@ resource "aws_ecs_task_definition" "app" {
 }
 
 resource "aws_iam_role_policy" "app_task_role" {
-  name     = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}-app-task-role"
-  policy   = data.aws_iam_policy_document.task_role_access_policy.json
+  name = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}-app-task-role"
+  policy = concat(
+    data.aws_iam_policy_document.task_role_access_policy.json,
+    #make fis policy conditional
+    data.aws_iam_policy_document.fis_allow.json
+  )
   role     = var.ecs_task_role.name
   provider = aws.region
 }
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 7a73fc23f2..e8b1ccc33c 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -99,3 +99,24 @@ locals {
       systemControls        = []
   })
 }
+
+data "aws_iam_policy_document" "fis_allow" {
+  statement {
+    sid       = "AllowSSMCommands"
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "ssm:CreateActivation",
+      "ssm:AddTagsToResource",
+    ]
+  }
+
+  statement {
+    sid       = "AllowPassRole"
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "iam:PassRole",
+    ]
+  }
+}

From 8332de5bc99e46b76bfd5a2b27906442261913d7 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 22 Dec 2023 10:54:28 +0000
Subject: [PATCH 13/51] Setup FIS permissions Prepare for FIS ECS Task
 permissions

---
 .../global/iam_fault_injection_simulator.tf   | 72 ++++++++++++++++++-
 .../environment/region/modules/app/ecs.tf     | 30 ++++++--
 2 files changed, 93 insertions(+), 9 deletions(-)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index a1112f8ae2..e9f4d37c77 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -22,14 +22,22 @@ resource "aws_iam_role_policy_attachment" "fault_injection_simulator" {
   provider   = aws.global
 }
 
-resource "aws_iam_role_policy" "additional_permissions" {
+resource "aws_iam_role_policy" "fault_injection_simulator_additional_permissions" {
   name     = "additional-permissions"
   role     = aws_iam_role.fault_injection_simulator.name
-  policy   = data.aws_iam_policy_document.additional_permissions.json
+  policy   = data.aws_iam_policy_document.fault_injection_simulator_combined.json
   provider = aws.global
 }
 
-data "aws_iam_policy_document" "additional_permissions" {
+#TODO: consolidate this into a single document if possible
+data "aws_iam_policy_document" "fault_injection_simulator_combined" {
+  source_policy_documents = [
+    data.aws_iam_policy_document.fault_injection_simulator_additional_permissions.json,
+    data.aws_iam_policy_document.fis_autocreated_role.json
+  ]
+}
+
+data "aws_iam_policy_document" "fault_injection_simulator_additional_permissions" {
 
   statement {
     sid       = "AllowServiceLinkedRole"
@@ -67,3 +75,61 @@ data "aws_iam_policy_document" "additional_permissions" {
     ]
   }
 }
+
+data "aws_iam_policy_document" "fis_autocreated_role" {
+  # example taken from role created in console to be attached to the FIS role
+  version = "2012-10-17"
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "ecs:DescribeClusters",
+      "ecs:ListContainerInstances"
+    ]
+    resources = [
+      "arn:aws:ecs:*:*:cluster/*"
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "ecs:DescribeTasks",
+      "ecs:StopTask"
+    ]
+    resources = [
+      "arn:aws:ecs:*:*:task/*/*"
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "ecs:ListTasks",
+      "ecs:UpdateContainerInstancesState"
+    ]
+    resources = [
+      "arn:aws:ecs:*:*:container-instance/*/*"
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "ssm:SendCommand"
+    ]
+    resources = [
+      "arn:aws:ssm:*:*:managed-instance/*",
+      "arn:aws:ssm:*:*:document/*"
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "ssm:ListCommands",
+      "ssm:CancelCommand"
+    ]
+    resources = ["*"]
+  }
+}
diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index 0e5702a8ba..d28ef82605 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -84,7 +84,7 @@ resource "aws_ecs_task_definition" "app" {
     operating_system_family = "LINUX"
     cpu_architecture        = "X86_64"
   }
-  # conditionally add ssm agent container
+  #TODO: conditionally add ssm agent container
   container_definitions = "[${local.app}, ${local.aws_otel_collector}, ${local.amazon_ssm_agent}]"
   task_role_arn         = var.ecs_task_role.arn
   execution_role_arn    = var.ecs_execution_role.arn
@@ -93,15 +93,33 @@ resource "aws_ecs_task_definition" "app" {
 
 resource "aws_iam_role_policy" "app_task_role" {
   name = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}-app-task-role"
-  policy = concat(
-    data.aws_iam_policy_document.task_role_access_policy.json,
-    #make fis policy conditional
-    data.aws_iam_policy_document.fis_allow.json
-  )
+  #TODO: make fis policy conditional using the combined policy document
+  policy   = data.aws_iam_policy_document.task_role_access_policy.json
   role     = var.ecs_task_role.name
   provider = aws.region
 }
 
+data "aws_iam_policy_document" "combined" {
+  source_policy_documents = [
+    data.aws_iam_policy_document.task_role_access_policy.json,
+    data.aws_iam_policy_document.fis_related_task_permissions.json
+  ]
+}
+
+data "aws_iam_policy_document" "fis_related_task_permissions" {
+  policy_id = "${local.policy_region_prefix}_fis_related_task_permissions"
+  statement {
+    sid    = local.policy_region_prefix
+    effect = "Allow"
+
+    actions = [
+      "xray:PutTraceSegments",
+    ]
+
+    resources = ["*"]
+  }
+}
+
 data "aws_kms_alias" "secrets_manager_secret_encryption_key" {
   name     = "alias/${data.aws_default_tags.current.tags.application}_secrets_manager_secret_encryption_key"
   provider = aws.region

From 00701fce4b4a55a24140db05b35a3cf3c7147b8d Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 22 Dec 2023 11:16:32 +0000
Subject: [PATCH 14/51] add task permissions for fis

---
 .../global/iam_fault_injection_simulator.tf   | 19 ++++++++++---------
 .../environment/region/modules/app/ecs.tf     |  6 ++++--
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index e9f4d37c77..f041fab5ec 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -38,7 +38,7 @@ data "aws_iam_policy_document" "fault_injection_simulator_combined" {
 }
 
 data "aws_iam_policy_document" "fault_injection_simulator_additional_permissions" {
-
+  policy_id = "fix experiment permissions"
   statement {
     sid       = "AllowServiceLinkedRole"
     effect    = "Allow"
@@ -124,12 +124,13 @@ data "aws_iam_policy_document" "fis_autocreated_role" {
     ]
   }
 
-  statement {
-    effect = "Allow"
-    actions = [
-      "ssm:ListCommands",
-      "ssm:CancelCommand"
-    ]
-    resources = ["*"]
-  }
+  #duplicate of above
+  # statement {
+  #   effect = "Allow"
+  #   actions = [
+  #     "ssm:ListCommands",
+  #     "ssm:CancelCommand"
+  #   ]
+  #   resources = ["*"]
+  # }
 }
diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index d28ef82605..798bfb3b14 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -109,11 +109,13 @@ data "aws_iam_policy_document" "combined" {
 data "aws_iam_policy_document" "fis_related_task_permissions" {
   policy_id = "${local.policy_region_prefix}_fis_related_task_permissions"
   statement {
-    sid    = local.policy_region_prefix
+    sid    = "${local.policy_region_prefix}_fis_ecs_task_actions"
     effect = "Allow"
 
     actions = [
-      "xray:PutTraceSegments",
+      "ssm:CreateActivation",
+      "ssm:AddTagsToResource",
+      "iam:PassRole",
     ]
 
     resources = ["*"]

From 221d4e4ea8a782d1a6725c0232befa3b04dc07dc Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 22 Dec 2023 11:19:37 +0000
Subject: [PATCH 15/51] consolidate

---
 terraform/environment/region/modules/app/ecs.tf  | 16 ----------------
 .../modules/app/fault_injection_simulator.tf     |  3 ++-
 2 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index 798bfb3b14..f17a500f30 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -106,22 +106,6 @@ data "aws_iam_policy_document" "combined" {
   ]
 }
 
-data "aws_iam_policy_document" "fis_related_task_permissions" {
-  policy_id = "${local.policy_region_prefix}_fis_related_task_permissions"
-  statement {
-    sid    = "${local.policy_region_prefix}_fis_ecs_task_actions"
-    effect = "Allow"
-
-    actions = [
-      "ssm:CreateActivation",
-      "ssm:AddTagsToResource",
-      "iam:PassRole",
-    ]
-
-    resources = ["*"]
-  }
-}
-
 data "aws_kms_alias" "secrets_manager_secret_encryption_key" {
   name     = "alias/${data.aws_default_tags.current.tags.application}_secrets_manager_secret_encryption_key"
   provider = aws.region
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index e8b1ccc33c..25e7c0740b 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -100,7 +100,8 @@ locals {
   })
 }
 
-data "aws_iam_policy_document" "fis_allow" {
+data "aws_iam_policy_document" "fis_related_task_permissions" {
+  policy_id = "${local.policy_region_prefix}_fis_ecs_task_actions"
   statement {
     sid       = "AllowSSMCommands"
     effect    = "Allow"

From cef711e7570176d859aff3b97e73350ee971c57e Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 22 Dec 2023 12:23:33 +0000
Subject: [PATCH 16/51] add managed instance perms

---
 .../region/modules/app/fault_injection_simulator.tf | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 25e7c0740b..2a478a623a 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -75,7 +75,7 @@ locals {
       environment = [
         {
           name  = "MANAGED_INSTANCE_ROLE_NAME",
-          value = "SSMManagedInstanceRole"
+          value = "${data.aws_default_tags.current.tags.environment-name}-app-task-role"
         }
       ],
       logConfiguration = {
@@ -112,6 +112,17 @@ data "aws_iam_policy_document" "fis_related_task_permissions" {
     ]
   }
 
+  statement {
+    sid       = "ManagedInstancePermissions"
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "ssm:DeleteActivation",
+      "ssm:DeregisterManagedInstance",
+      "ssm:CreateActivation",
+    ]
+  }
+
   statement {
     sid       = "AllowPassRole"
     effect    = "Allow"

From d77b3f663bc5d0fc3b83d6e47910305a69e73088 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 19 Jan 2024 10:04:35 +0000
Subject: [PATCH 17/51] use managed policies for fis role

---
 .../environment/global/iam_ecs_task_roles.tf  |   1 +
 .../global/iam_fault_injection_simulator.tf   | 102 ++++--------------
 2 files changed, 24 insertions(+), 79 deletions(-)

diff --git a/terraform/environment/global/iam_ecs_task_roles.tf b/terraform/environment/global/iam_ecs_task_roles.tf
index 7ece0ae8db..1c72dad062 100644
--- a/terraform/environment/global/iam_ecs_task_roles.tf
+++ b/terraform/environment/global/iam_ecs_task_roles.tf
@@ -16,3 +16,4 @@ data "aws_iam_policy_document" "task_role_assume_policy" {
   }
   provider = aws.global
 }
+
diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index f041fab5ec..f996c5059e 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -1,3 +1,5 @@
+# Create role for running experiments
+
 resource "aws_iam_role" "fault_injection_simulator" {
   name               = "fault-injection-simulator-${data.aws_default_tags.current.tags.environment-name}"
   assume_role_policy = data.aws_iam_policy_document.fault_injection_simulator_assume.json
@@ -16,27 +18,27 @@ data "aws_iam_policy_document" "fault_injection_simulator_assume" {
   provider = aws.global
 }
 
-resource "aws_iam_role_policy_attachment" "fault_injection_simulator" {
+# Add permissions for FIS to run experiments (ECS, Logging, SSM)
+
+resource "aws_iam_role_policy_attachment" "fault_injection_simulator_ecs_access" {
   role       = aws_iam_role.fault_injection_simulator.name
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSFaultInjectionSimulatorECSAccess"
   provider   = aws.global
 }
 
+resource "aws_iam_role_policy_attachment" "fault_injection_simulator_ssm_access" {
+  role       = aws_iam_role.fault_injection_simulator.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSFaultInjectionSimulatorSSMAccess"
+  provider   = aws.global
+}
+
 resource "aws_iam_role_policy" "fault_injection_simulator_additional_permissions" {
   name     = "additional-permissions"
   role     = aws_iam_role.fault_injection_simulator.name
-  policy   = data.aws_iam_policy_document.fault_injection_simulator_combined.json
+  policy   = data.aws_iam_policy_document.fault_injection_simulator_additional_permissions.json
   provider = aws.global
 }
 
-#TODO: consolidate this into a single document if possible
-data "aws_iam_policy_document" "fault_injection_simulator_combined" {
-  source_policy_documents = [
-    data.aws_iam_policy_document.fault_injection_simulator_additional_permissions.json,
-    data.aws_iam_policy_document.fis_autocreated_role.json
-  ]
-}
-
 data "aws_iam_policy_document" "fault_injection_simulator_additional_permissions" {
   policy_id = "fix experiment permissions"
   statement {
@@ -51,6 +53,16 @@ data "aws_iam_policy_document" "fault_injection_simulator_additional_permissions
       variable = "iam:AWSServiceName"
       values   = ["fis.amazonaws.com"]
     }
+    condition {
+      test     = "StringEquals"
+      variable = "aws:SourceAccount"
+      values   = [data.aws_caller_identity.global.account_id]
+    }
+    condition {
+      test     = "ArnLike"
+      variable = "aws:SourceArn"
+      values   = ["arn:aws:fis:${data.aws_region.global.name}:${data.aws_caller_identity.global.account_id}:experiment/*"]
+    }
   }
 
   statement {
@@ -62,75 +74,7 @@ data "aws_iam_policy_document" "fault_injection_simulator_additional_permissions
       "logs:DescribeLogGroups",
       "logs:CreateLogStream",
       "logs:PutLogEvents",
+      "logs:DescribeResourcePolicies",
     ]
   }
-  statement {
-    sid       = "AllowSSMCommands"
-    effect    = "Allow"
-    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
-    actions = [
-      "ssm:SendCommand",
-      "ssm:ListCommands",
-      "ssm:CancelCommand",
-    ]
-  }
-}
-
-data "aws_iam_policy_document" "fis_autocreated_role" {
-  # example taken from role created in console to be attached to the FIS role
-  version = "2012-10-17"
-
-  statement {
-    effect = "Allow"
-    actions = [
-      "ecs:DescribeClusters",
-      "ecs:ListContainerInstances"
-    ]
-    resources = [
-      "arn:aws:ecs:*:*:cluster/*"
-    ]
-  }
-
-  statement {
-    effect = "Allow"
-    actions = [
-      "ecs:DescribeTasks",
-      "ecs:StopTask"
-    ]
-    resources = [
-      "arn:aws:ecs:*:*:task/*/*"
-    ]
-  }
-
-  statement {
-    effect = "Allow"
-    actions = [
-      "ecs:ListTasks",
-      "ecs:UpdateContainerInstancesState"
-    ]
-    resources = [
-      "arn:aws:ecs:*:*:container-instance/*/*"
-    ]
-  }
-
-  statement {
-    effect = "Allow"
-    actions = [
-      "ssm:SendCommand"
-    ]
-    resources = [
-      "arn:aws:ssm:*:*:managed-instance/*",
-      "arn:aws:ssm:*:*:document/*"
-    ]
-  }
-
-  #duplicate of above
-  # statement {
-  #   effect = "Allow"
-  #   actions = [
-  #     "ssm:ListCommands",
-  #     "ssm:CancelCommand"
-  #   ]
-  #   resources = ["*"]
-  # }
 }

From c99156552470af95465ea631489ca7994158074d Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 19 Jan 2024 10:05:40 +0000
Subject: [PATCH 18/51] configure logging (not working in practice) use managed
 role for instance registration

---
 .../environment/region/modules/app/ecs.tf     |  5 ++--
 .../modules/app/fault_injection_simulator.tf  | 23 +++++++++++++++----
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index f17a500f30..18506a002a 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -94,7 +94,8 @@ resource "aws_ecs_task_definition" "app" {
 resource "aws_iam_role_policy" "app_task_role" {
   name = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}-app-task-role"
   #TODO: make fis policy conditional using the combined policy document
-  policy   = data.aws_iam_policy_document.task_role_access_policy.json
+  # policy   = data.aws_iam_policy_document.task_role_access_policy.json
+  policy   = data.aws_iam_policy_document.combined.json
   role     = var.ecs_task_role.name
   provider = aws.region
 }
@@ -102,7 +103,7 @@ resource "aws_iam_role_policy" "app_task_role" {
 data "aws_iam_policy_document" "combined" {
   source_policy_documents = [
     data.aws_iam_policy_document.task_role_access_policy.json,
-    data.aws_iam_policy_document.fis_related_task_permissions.json
+    data.aws_iam_policy_document.ecs_task_role_fis_related_task_permissions.json
   ]
 }
 
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 2a478a623a..d63f81bdad 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -1,3 +1,5 @@
+# Create encrypted logging for fault injection experiments
+
 data "aws_kms_alias" "cloudwatch_application_logs_encryption" {
   name     = "alias/${data.aws_default_tags.current.tags.application}_cloudwatch_application_logs_encryption"
   provider = aws.region
@@ -10,6 +12,8 @@ resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
   provider = aws.region
 }
 
+# Create experiment template for ECS tasks
+
 resource "aws_fis_experiment_template" "ecs_app" {
   count       = data.aws_default_tags.current.tags.environment-name == "production" ? 0 : 1
   provider    = aws.region
@@ -38,10 +42,18 @@ resource "aws_fis_experiment_template" "ecs_app" {
     value  = null
   }
 
+  log_configuration {
+    log_schema_version = 2
+
+    cloudwatch_logs_configuration {
+      log_group_arn = "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*" # tfsec:ignore:aws-cloudwatch-log-group-wildcard
+    }
+  }
+
   target {
     name = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
     resource_arns = [
-      "arn:aws:ecs:eu-west-1:653761790766:task/936mlpab157-eu-west-1/1b1aedd43f94458d9ef3475479e19169",
+      "arn:aws:ecs:eu-west-1:653761790766:task/936mlpab157-eu-west-1/0d3c81896ebb4f90aba341b512f90c4e",
       # aws_ecs_task_definition.mock_onelogin.arn,
     ]
     # parameters = {
@@ -57,6 +69,8 @@ resource "aws_fis_experiment_template" "ecs_app" {
   }
 }
 
+# Create ECS task definition for ssm agent, used to run experiments
+
 locals {
   amazon_ssm_agent = jsonencode(
     {
@@ -75,7 +89,7 @@ locals {
       environment = [
         {
           name  = "MANAGED_INSTANCE_ROLE_NAME",
-          value = "${data.aws_default_tags.current.tags.environment-name}-app-task-role"
+          value = "service-role/AmazonEC2RunCommandRoleForManagedInstances"
         }
       ],
       logConfiguration = {
@@ -100,7 +114,9 @@ locals {
   })
 }
 
-data "aws_iam_policy_document" "fis_related_task_permissions" {
+# Additional permissions for the ECS task role to run experiments
+
+data "aws_iam_policy_document" "ecs_task_role_fis_related_task_permissions" {
   policy_id = "${local.policy_region_prefix}_fis_ecs_task_actions"
   statement {
     sid       = "AllowSSMCommands"
@@ -119,7 +135,6 @@ data "aws_iam_policy_document" "fis_related_task_permissions" {
     actions = [
       "ssm:DeleteActivation",
       "ssm:DeregisterManagedInstance",
-      "ssm:CreateActivation",
     ]
   }
 

From 505eca6c9621ad67828769a8e8019401426bc853 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 19 Jan 2024 10:07:18 +0000
Subject: [PATCH 19/51] update lock

---
 terraform/environment/.terraform.lock.hcl | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/terraform/environment/.terraform.lock.hcl b/terraform/environment/.terraform.lock.hcl
index 2cb66e3111..f37a04b7e0 100644
--- a/terraform/environment/.terraform.lock.hcl
+++ b/terraform/environment/.terraform.lock.hcl
@@ -45,7 +45,11 @@ provider "registry.terraform.io/hashicorp/local" {
 
 provider "registry.terraform.io/pagerduty/pagerduty" {
   version     = "3.4.1"
+<<<<<<< HEAD
   constraints = "3.4.1"
+=======
+  constraints = ">= 2.16.0, ~> 3.4.0"
+>>>>>>> 1c4530c2 (update lock)
   hashes = [
     "h1:ntsWamEgQsmFukTV3vtgj6NFowleLE7V3l6U4dW2nOo=",
     "zh:0ac31f1a07ed501a1d14025b3d1196cfe06f9c96da010da50bc360c5186a514f",

From 73ec64ae3297e2355e428e8694e77a480d5c6f7e Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 19 Jan 2024 11:20:28 +0000
Subject: [PATCH 20/51] add role to allow registration of instance

---
 .../global/iam_fault_injection_simulator.tf   | 51 +++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index f996c5059e..6462ceff40 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -78,3 +78,54 @@ data "aws_iam_policy_document" "fault_injection_simulator_additional_permissions
     ]
   }
 }
+
+
+# Create role for registering instance
+
+resource "aws_iam_role" "ssm_register_instance" {
+  name               = "ssm-register-instance-${data.aws_default_tags.current.tags.environment-name}"
+  assume_role_policy = data.aws_iam_policy_document.ssm_register_instance_assume.json
+  provider           = aws.global
+}
+
+data "aws_iam_policy_document" "ssm_register_instance_assume" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["ssm.amazonaws.com"]
+    }
+  }
+  provider = aws.global
+}
+
+resource "aws_iam_role_policy" "ssm_register_instance_permissions" {
+  name     = "ssm-register-instance-permissions"
+  role     = aws_iam_role.fault_injection_simulator.name
+  policy   = data.aws_iam_policy_document.ssm_register_instance_permissions.json
+  provider = aws.global
+}
+
+data "aws_iam_policy_document" "ssm_register_instance_permissions" {
+  policy_id = "ssm instance activation permissions"
+  statement {
+    sid       = "AllowSSMCommands"
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "ssm:CreateActivation",
+      "ssm:AddTagsToResource",
+    ]
+  }
+
+  statement {
+    sid       = "ManagedInstancePermissions"
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "ssm:DeleteActivation",
+      "ssm:DeregisterManagedInstance",
+    ]
+  }
+}

From 8aff44d741a624fedd87e6a5b45848c387a2f0ba Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 19 Jan 2024 11:21:06 +0000
Subject: [PATCH 21/51] add resource policy for log group

---
 .../modules/app/fault_injection_simulator.tf  | 31 +++++++++++++++++--
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index d63f81bdad..7a6d7a3399 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -6,11 +6,36 @@ data "aws_kms_alias" "cloudwatch_application_logs_encryption" {
 }
 
 resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
-  name              = "/aws/fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}"
+  name              = "fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}"
   retention_in_days = 7
   # kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
   provider = aws.region
 }
+data "aws_iam_policy_document" "fis_app_ecs_tasks" {
+  provider  = aws.region
+  policy_id = "fis_app_ecs_tasks"
+  statement {
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "logs:PutLogEventsBatch",
+    ]
+
+    resources = ["arn:aws:logs:*:*:log-group:/fis/*"]
+
+    principals {
+      identifiers = ["fis.amazonaws.com"]
+      type        = "Service"
+    }
+  }
+}
+
+resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
+  provider        = aws.region
+  policy_document = data.aws_iam_policy_document.fis_app_ecs_tasks.json
+  policy_name     = "fis_app_ecs_tasks"
+}
+
 
 # Create experiment template for ECS tasks
 
@@ -53,7 +78,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
   target {
     name = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
     resource_arns = [
-      "arn:aws:ecs:eu-west-1:653761790766:task/936mlpab157-eu-west-1/0d3c81896ebb4f90aba341b512f90c4e",
+      "arn:aws:ecs:eu-west-1:653761790766:task/936mlpab157-eu-west-1/6abd833098024a9280dd4f392a2659b6",
       # aws_ecs_task_definition.mock_onelogin.arn,
     ]
     # parameters = {
@@ -89,7 +114,7 @@ locals {
       environment = [
         {
           name  = "MANAGED_INSTANCE_ROLE_NAME",
-          value = "service-role/AmazonEC2RunCommandRoleForManagedInstances"
+          value = "ssm-register-instance-${data.aws_default_tags.current.tags.environment-name}"
         }
       ],
       logConfiguration = {

From 980b6c0c1d99dd843810884060bf7a9ee5dcdd79 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 19 Jan 2024 13:44:19 +0000
Subject: [PATCH 22/51] select targets using tags

---
 .../modules/app/fault_injection_simulator.tf  | 82 +++++++++----------
 1 file changed, 40 insertions(+), 42 deletions(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 7a6d7a3399..86c2e7c8a6 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -11,31 +11,37 @@ resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
   # kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
   provider = aws.region
 }
-data "aws_iam_policy_document" "fis_app_ecs_tasks" {
-  provider  = aws.region
-  policy_id = "fis_app_ecs_tasks"
-  statement {
-    actions = [
-      "logs:CreateLogStream",
-      "logs:PutLogEvents",
-      "logs:PutLogEventsBatch",
-    ]
-
-    resources = ["arn:aws:logs:*:*:log-group:/fis/*"]
-
-    principals {
-      identifiers = ["fis.amazonaws.com"]
-      type        = "Service"
-    }
-  }
-}
-
-resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
-  provider        = aws.region
-  policy_document = data.aws_iam_policy_document.fis_app_ecs_tasks.json
-  policy_name     = "fis_app_ecs_tasks"
-}
 
+# Add resource policy to allow FIS or the FIS role to write logs - not working
+# data "aws_iam_policy_document" "fis_app_ecs_tasks" {
+#   provider  = aws.region
+#   policy_id = "fis_app_ecs_tasks"
+#   statement {
+#     actions = [
+#       "logs:CreateLogDelivery",
+#       "logs:DescribeLogGroups",
+#       "logs:CreateLogStream",
+#       "logs:PutLogEvents",
+#       "logs:DescribeResourcePolicies",
+#     ]
+
+#     resources = [
+#       "arn:aws:logs:*:*:log-group:/fis/*",
+#       "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:fis/app-ecs-tasks-experiment-936mlpab157:*"
+#     ]
+
+#     principals {
+#       identifiers = [data.aws_caller_identity.current.account_id]
+#       type        = "AWS"
+#     }
+#   }
+# }
+
+# resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
+#   provider        = aws.region
+#   policy_document = data.aws_iam_policy_document.fis_app_ecs_tasks.json
+#   policy_name     = "fis_app_ecs_tasks"
+# }
 
 # Create experiment template for ECS tasks
 
@@ -67,28 +73,20 @@ resource "aws_fis_experiment_template" "ecs_app" {
     value  = null
   }
 
-  log_configuration {
-    log_schema_version = 2
+  # log_configuration {
+  #   log_schema_version = 2
 
-    cloudwatch_logs_configuration {
-      log_group_arn = "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*" # tfsec:ignore:aws-cloudwatch-log-group-wildcard
-    }
-  }
+  #   cloudwatch_logs_configuration {
+  #     log_group_arn = "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*" # tfsec:ignore:aws-cloudwatch-log-group-wildcard
+  #   }
+  # }
 
   target {
     name = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
-    resource_arns = [
-      "arn:aws:ecs:eu-west-1:653761790766:task/936mlpab157-eu-west-1/6abd833098024a9280dd4f392a2659b6",
-      # aws_ecs_task_definition.mock_onelogin.arn,
-    ]
-    # parameters = {
-    #   "cluster" : var.ecs_cluster,
-    #   "service" : aws_ecs_service.app.name,
-    # }
-    # resource_tag {
-    #   key   = "aws:ecs:service"
-    #   value = aws_ecs_service.app.name
-    # }
+    resource_tag {
+      key   = "environment-name"
+      value = data.aws_default_tags.current.tags.environment-name
+    }
     resource_type  = "aws:ecs:task"
     selection_mode = "ALL"
   }

From 624add78431ffe978166a4f229d998db283f7e95 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 19 Jan 2024 14:10:43 +0000
Subject: [PATCH 23/51] test logging configuration

---
 .../modules/app/fault_injection_simulator.tf  | 60 +++++++++----------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 86c2e7c8a6..a067d50fbe 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -13,35 +13,35 @@ resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
 }
 
 # Add resource policy to allow FIS or the FIS role to write logs - not working
-# data "aws_iam_policy_document" "fis_app_ecs_tasks" {
-#   provider  = aws.region
-#   policy_id = "fis_app_ecs_tasks"
-#   statement {
-#     actions = [
-#       "logs:CreateLogDelivery",
-#       "logs:DescribeLogGroups",
-#       "logs:CreateLogStream",
-#       "logs:PutLogEvents",
-#       "logs:DescribeResourcePolicies",
-#     ]
-
-#     resources = [
-#       "arn:aws:logs:*:*:log-group:/fis/*",
-#       "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:fis/app-ecs-tasks-experiment-936mlpab157:*"
-#     ]
-
-#     principals {
-#       identifiers = [data.aws_caller_identity.current.account_id]
-#       type        = "AWS"
-#     }
-#   }
-# }
-
-# resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
-#   provider        = aws.region
-#   policy_document = data.aws_iam_policy_document.fis_app_ecs_tasks.json
-#   policy_name     = "fis_app_ecs_tasks"
-# }
+data "aws_iam_policy_document" "fis_app_ecs_tasks" {
+  provider  = aws.region
+  policy_id = "fis_app_ecs_tasks"
+  statement {
+    actions = [
+      "logs:CreateLogDelivery",
+      "logs:DescribeLogGroups",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "logs:DescribeResourcePolicies",
+    ]
+
+    resources = [
+      "arn:aws:logs:*:*:log-group:/fis/*",
+      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:fis/app-ecs-tasks-experiment-936mlpab157:*"
+    ]
+
+    principals {
+      identifiers = [data.aws_caller_identity.current.account_id]
+      type        = "AWS"
+    }
+  }
+}
+
+resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
+  provider        = aws.region
+  policy_document = data.aws_iam_policy_document.fis_app_ecs_tasks.json
+  policy_name     = "fis_app_ecs_tasks"
+}
 
 # Create experiment template for ECS tasks
 
@@ -74,7 +74,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
   }
 
   # log_configuration {
-  #   log_schema_version = 2
+  #   log_schema_version = 1
 
   #   cloudwatch_logs_configuration {
   #     log_group_arn = "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*" # tfsec:ignore:aws-cloudwatch-log-group-wildcard

From fe160f3b480a65e13d5f422bd8aabb492276acb7 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 19 Jan 2024 14:22:44 +0000
Subject: [PATCH 24/51] allow fis role to encrypt logs

---
 .../modules/app/fault_injection_simulator.tf  | 28 +++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index a067d50fbe..102a82467e 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -8,8 +8,8 @@ data "aws_kms_alias" "cloudwatch_application_logs_encryption" {
 resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
   name              = "fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}"
   retention_in_days = 7
-  # kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
-  provider = aws.region
+  kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
+  provider          = aws.region
 }
 
 # Add resource policy to allow FIS or the FIS role to write logs - not working
@@ -43,6 +43,30 @@ resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
   policy_name     = "fis_app_ecs_tasks"
 }
 
+# Add log encryption permissions to the FIS role
+
+data "aws_iam_policy_document" "fis_role_log_encryption" {
+  provider  = aws.region
+  policy_id = "fis_role_log_encryption"
+  statement {
+    actions = [
+      "kms:Encrypt",
+      "kms:GenerateDataKey",
+    ]
+
+    resources = [
+      data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
+    ]
+  }
+}
+
+resource "aws_iam_role_policy" "fis_role_log_encryption" {
+  provider = aws.region
+  name     = "fis_role_log_encryption"
+  role     = var.fault_injection_simulator_role_arn
+  policy   = data.aws_iam_policy_document.fis_role_log_encryption.json
+}
+
 # Create experiment template for ECS tasks
 
 resource "aws_fis_experiment_template" "ecs_app" {

From 90f808e49281e27b1272da5c9114ae138ac04d34 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 19 Jan 2024 14:32:55 +0000
Subject: [PATCH 25/51] refactor role to pull in name and arn

---
 terraform/environment/region/ecs.tf           | 34 +++++++++----------
 .../modules/app/fault_injection_simulator.tf  |  4 +--
 .../region/modules/app/variables.tf           |  2 +-
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/terraform/environment/region/ecs.tf b/terraform/environment/region/ecs.tf
index 45384705ad..aafd58f90e 100644
--- a/terraform/environment/region/ecs.tf
+++ b/terraform/environment/region/ecs.tf
@@ -21,23 +21,23 @@ data "aws_ssm_parameter" "additional_allowed_ingress_cidrs" {
 }
 
 module "app" {
-  source                             = "./modules/app"
-  ecs_cluster                        = aws_ecs_cluster.main.id
-  ecs_execution_role                 = var.iam_roles.ecs_execution_role
-  ecs_task_role                      = var.iam_roles.app_ecs_task_role
-  ecs_service_desired_count          = 1
-  ecs_application_log_group_name     = module.application_logs.cloudwatch_log_group.name
-  ecs_capacity_provider              = var.ecs_capacity_provider
-  app_env_vars                       = var.app_env_vars
-  app_service_repository_url         = var.app_service_repository_url
-  app_service_container_version      = var.app_service_container_version
-  app_allowed_api_arns               = concat(var.uid_service.api_arns, var.lpa_store_service.api_arns)
-  ingress_allow_list_cidr            = concat(var.ingress_allow_list_cidr, split(",", data.aws_ssm_parameter.additional_allowed_ingress_cidrs.value))
-  alb_deletion_protection_enabled    = var.alb_deletion_protection_enabled
-  fault_injection_simulator_role_arn = var.iam_roles.fault_injection_simulator.arn
-  lpas_table                         = var.lpas_table
-  container_port                     = 8080
-  public_access_enabled              = var.public_access_enabled
+  source                          = "./modules/app"
+  ecs_cluster                     = aws_ecs_cluster.main.id
+  ecs_execution_role              = var.iam_roles.ecs_execution_role
+  ecs_task_role                   = var.iam_roles.app_ecs_task_role
+  ecs_service_desired_count       = 1
+  ecs_application_log_group_name  = module.application_logs.cloudwatch_log_group.name
+  ecs_capacity_provider           = var.ecs_capacity_provider
+  app_env_vars                    = var.app_env_vars
+  app_service_repository_url      = var.app_service_repository_url
+  app_service_container_version   = var.app_service_container_version
+  app_allowed_api_arns            = concat(var.uid_service.api_arns, var.lpa_store_service.api_arns)
+  ingress_allow_list_cidr         = concat(var.ingress_allow_list_cidr, split(",", data.aws_ssm_parameter.additional_allowed_ingress_cidrs.value))
+  alb_deletion_protection_enabled = var.alb_deletion_protection_enabled
+  fault_injection_simulator_role  = var.iam_roles.fault_injection_simulator
+  lpas_table                      = var.lpas_table
+  container_port                  = 8080
+  public_access_enabled           = var.public_access_enabled
   network = {
     vpc_id              = data.aws_vpc.main.id
     application_subnets = data.aws_subnet.application[*].id
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 102a82467e..084a831a97 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -63,7 +63,7 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
 resource "aws_iam_role_policy" "fis_role_log_encryption" {
   provider = aws.region
   name     = "fis_role_log_encryption"
-  role     = var.fault_injection_simulator_role_arn
+  role     = var.fault_injection_simulator_role.name
   policy   = data.aws_iam_policy_document.fis_role_log_encryption.json
 }
 
@@ -73,7 +73,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
   count       = data.aws_default_tags.current.tags.environment-name == "production" ? 0 : 1
   provider    = aws.region
   description = "Run ECS task experiments for the app service"
-  role_arn    = var.fault_injection_simulator_role_arn
+  role_arn    = var.fault_injection_simulator_role.arn
   tags = {
     Name = "${data.aws_default_tags.current.tags.environment-name} - APP ECS Task Experiments"
   }
diff --git a/terraform/environment/region/modules/app/variables.tf b/terraform/environment/region/modules/app/variables.tf
index be35c14146..e4174d3998 100644
--- a/terraform/environment/region/modules/app/variables.tf
+++ b/terraform/environment/region/modules/app/variables.tf
@@ -15,7 +15,7 @@ variable "ecs_task_role" {
   description = "ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services."
 }
 
-variable "fault_injection_simulator_role_arn" {
+variable "fault_injection_simulator_role" {
   type        = any
   description = "ARN of IAM role that allows AWS FIS to make calls to other AWS services."
 }

From 9bd10ebf917a0ca0499610b83e1be8f3a58f24f2 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Mon, 22 Jan 2024 16:02:21 +0000
Subject: [PATCH 26/51] resolve permissions message

---
 .../modules/app/fault_injection_simulator.tf  | 62 +++++++++++++++----
 1 file changed, 49 insertions(+), 13 deletions(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 084a831a97..3551b2661f 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -8,14 +8,14 @@ data "aws_kms_alias" "cloudwatch_application_logs_encryption" {
 resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
   name              = "fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}"
   retention_in_days = 7
-  kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
-  provider          = aws.region
+  # kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
+  provider = aws.region
 }
 
 # Add resource policy to allow FIS or the FIS role to write logs - not working
 data "aws_iam_policy_document" "fis_app_ecs_tasks" {
   provider  = aws.region
-  policy_id = "fis_app_ecs_tasks"
+  policy_id = "fis_app_ecs_tasks_service"
   statement {
     actions = [
       "logs:CreateLogDelivery",
@@ -26,15 +26,35 @@ data "aws_iam_policy_document" "fis_app_ecs_tasks" {
     ]
 
     resources = [
-      "arn:aws:logs:*:*:log-group:/fis/*",
-      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:fis/app-ecs-tasks-experiment-936mlpab157:*"
+      "arn:aws:logs:*:*:log-group:/aws/fis/*",
+      # "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:fis/*"
     ]
 
     principals {
-      identifiers = [data.aws_caller_identity.current.account_id]
-      type        = "AWS"
+      identifiers = ["fis.amazonaws.com"]
+      type        = "Service"
     }
   }
+  # statement {
+  #   actions = [
+  #     "logs:CreateLogDelivery",
+  #     "logs:DescribeLogGroups",
+  #     "logs:CreateLogStream",
+  #     "logs:PutLogEvents",
+  #     "logs:DescribeResourcePolicies",
+  #   ]
+
+  #   resources = [
+  #     "arn:aws:logs:*:*:log-group:/aws/fis/*",
+  #     # "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:fis/*",
+  #     # "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
+  #   ]
+
+  #   principals {
+  #     identifiers = [data.aws_caller_identity.current.account_id]
+  #     type        = "AWS"
+  #   }
+  # }
 }
 
 resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
@@ -58,6 +78,21 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
       data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
     ]
   }
+
+  statement {
+    actions = [
+      "logs:CreateLogDelivery",
+      "logs:DescribeLogGroups",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "logs:DescribeResourcePolicies",
+    ]
+
+    resources = [
+      aws_cloudwatch_log_group.fis_app_ecs_tasks.arn,
+      "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
+    ]
+  }
 }
 
 resource "aws_iam_role_policy" "fis_role_log_encryption" {
@@ -97,13 +132,14 @@ resource "aws_fis_experiment_template" "ecs_app" {
     value  = null
   }
 
-  # log_configuration {
-  #   log_schema_version = 1
+  log_configuration {
+    log_schema_version = 2
 
-  #   cloudwatch_logs_configuration {
-  #     log_group_arn = "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*" # tfsec:ignore:aws-cloudwatch-log-group-wildcard
-  #   }
-  # }
+    cloudwatch_logs_configuration {
+      log_group_arn = "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:aws/fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}:*" # tfsec:ignore:aws-cloudwatch-log-group-wildcard
+      # log_group_arn = "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*" # tfsec:ignore:aws-cloudwatch-log-group-wildcard
+    }
+  }
 
   target {
     name = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"

From 7131821c1e3370307c71236fa8091363b1e8b5f1 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 23 Jan 2024 12:57:26 +0000
Subject: [PATCH 27/51] cleanup for suport request

---
 .../global/iam_fault_injection_simulator.tf   |  13 ---
 .../modules/app/fault_injection_simulator.tf  | 108 ++++++++++++------
 2 files changed, 72 insertions(+), 49 deletions(-)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index 6462ceff40..604ee0f553 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -64,19 +64,6 @@ data "aws_iam_policy_document" "fault_injection_simulator_additional_permissions
       values   = ["arn:aws:fis:${data.aws_region.global.name}:${data.aws_caller_identity.global.account_id}:experiment/*"]
     }
   }
-
-  statement {
-    sid       = "AllowCloudWatchLogs"
-    effect    = "Allow"
-    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
-    actions = [
-      "logs:CreateLogDelivery",
-      "logs:DescribeLogGroups",
-      "logs:CreateLogStream",
-      "logs:PutLogEvents",
-      "logs:DescribeResourcePolicies",
-    ]
-  }
 }
 
 
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 3551b2661f..a71ae39a47 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -6,69 +6,65 @@ data "aws_kms_alias" "cloudwatch_application_logs_encryption" {
 }
 
 resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
-  name              = "fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}"
+  name              = "/aws/fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}"
   retention_in_days = 7
   # kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
   provider = aws.region
 }
 
 # Add resource policy to allow FIS or the FIS role to write logs - not working
-data "aws_iam_policy_document" "fis_app_ecs_tasks" {
+
+data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
   provider  = aws.region
   policy_id = "fis_app_ecs_tasks_service"
   statement {
+    sid    = "AWSLogDeliveryWrite20150319"
+    effect = "Allow"
+
+    principals {
+      identifiers = [
+        "fis.amazonaws.com"
+      ]
+      type = "Service"
+    }
+
     actions = [
-      "logs:CreateLogDelivery",
-      "logs:DescribeLogGroups",
       "logs:CreateLogStream",
-      "logs:PutLogEvents",
-      "logs:DescribeResourcePolicies",
+      "logs:PutLogEvents"
     ]
 
     resources = [
       "arn:aws:logs:*:*:log-group:/aws/fis/*",
-      # "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:fis/*"
+      "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
     ]
 
-    principals {
-      identifiers = ["fis.amazonaws.com"]
-      type        = "Service"
+    condition {
+      test     = "StringEquals"
+      variable = "aws:SourceAccount"
+      values   = [":${data.aws_caller_identity.current.account_id}"]
+    }
+
+    condition {
+      test     = "ArnLike"
+      variable = "aws:SourceArn"
+      values   = ["arn:aws:logs:${data.aws_region.current.name}::${data.aws_caller_identity.current.account_id}:*"]
     }
   }
-  # statement {
-  #   actions = [
-  #     "logs:CreateLogDelivery",
-  #     "logs:DescribeLogGroups",
-  #     "logs:CreateLogStream",
-  #     "logs:PutLogEvents",
-  #     "logs:DescribeResourcePolicies",
-  #   ]
-
-  #   resources = [
-  #     "arn:aws:logs:*:*:log-group:/aws/fis/*",
-  #     # "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:fis/*",
-  #     # "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
-  #   ]
-
-  #   principals {
-  #     identifiers = [data.aws_caller_identity.current.account_id]
-  #     type        = "AWS"
-  #   }
-  # }
 }
 
 resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
   provider        = aws.region
-  policy_document = data.aws_iam_policy_document.fis_app_ecs_tasks.json
+  policy_document = data.aws_iam_policy_document.cloudwatch_log_group_policy_fis_app_ecs_tasks.json
   policy_name     = "fis_app_ecs_tasks"
 }
 
-# Add log encryption permissions to the FIS role
+# Add log encryption and log write/delivery permissions to the FIS role
 
 data "aws_iam_policy_document" "fis_role_log_encryption" {
   provider  = aws.region
-  policy_id = "fis_role_log_encryption"
+  policy_id = "log_access"
   statement {
+    sid = "AllowCloudWatchLogsEncryption"
     actions = [
       "kms:Encrypt",
       "kms:GenerateDataKey",
@@ -80,6 +76,7 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
   }
 
   statement {
+    sid = "AllowCloudWatchLogs"
     actions = [
       "logs:CreateLogDelivery",
       "logs:DescribeLogGroups",
@@ -93,11 +90,51 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
       "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
     ]
   }
+
+  statement {
+    sid    = "AllowLogDeliveryActions"
+    effect = "Allow"
+    actions = [
+      "logs:PutDeliverySource",
+      "logs:GetDeliverySource",
+      "logs:DeleteDeliverySource",
+      "logs:DescribeDeliverySources",
+      "logs:PutDeliveryDestination",
+      "logs:GetDeliveryDestination",
+      "logs:DeleteDeliveryDestination",
+      "logs:DescribeDeliveryDestinations",
+      "logs:CreateDelivery",
+      "logs:GetDelivery",
+      "logs:DeleteDelivery",
+      "logs:DescribeDeliveries",
+      "logs:PutDeliveryDestinationPolicy",
+      "logs:GetDeliveryDestinationPolicy",
+      "logs:DeleteDeliveryDestinationPolicy"
+    ]
+    resources = [
+      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery-source:*",
+      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery:*",
+      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery-destination:*"
+    ]
+  }
+  statement {
+    sid    = "AllowUpdatesToResourcePolicyCWL"
+    effect = "Allow"
+    actions = [
+      "logs:PutResourcePolicy",
+      "logs:DescribeResourcePolicies",
+      "logs:DescribeLogGroups"
+    ]
+    resources = [
+      aws_cloudwatch_log_group.fis_app_ecs_tasks.arn,
+      "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
+    ]
+  }
 }
 
 resource "aws_iam_role_policy" "fis_role_log_encryption" {
   provider = aws.region
-  name     = "fis_role_log_encryption"
+  name     = "fis-role-log-permissions"
   role     = var.fault_injection_simulator_role.name
   policy   = data.aws_iam_policy_document.fis_role_log_encryption.json
 }
@@ -136,8 +173,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
     log_schema_version = 2
 
     cloudwatch_logs_configuration {
-      log_group_arn = "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:aws/fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}:*" # tfsec:ignore:aws-cloudwatch-log-group-wildcard
-      # log_group_arn = "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*" # tfsec:ignore:aws-cloudwatch-log-group-wildcard
+      log_group_arn = "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*" # tfsec:ignore:aws-cloudwatch-log-group-wildcard
     }
   }
 

From 818d3138441d8599025c32720a0ff57fff23dd5e Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 23 Jan 2024 16:33:32 +0000
Subject: [PATCH 28/51] add comment to test creds change - unrelated to ticket

---
 .../environment/region/modules/app/fault_injection_simulator.tf  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index a71ae39a47..3aed383eaf 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -52,6 +52,7 @@ data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
   }
 }
 
+#  Add resource policy to allow FIS or the FIS role to write logs - not working
 resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
   provider        = aws.region
   policy_document = data.aws_iam_policy_document.cloudwatch_log_group_policy_fis_app_ecs_tasks.json

From fe3ad361eb98af8ad61ad848ba29678b08a4ee78 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 24 Jan 2024 14:13:50 +0000
Subject: [PATCH 29/51] attache cloudwatch full access policy fix resource
 policy conditions

---
 .../global/iam_fault_injection_simulator.tf   |  6 ++++
 .../modules/app/fault_injection_simulator.tf  | 33 ++++++++++++-------
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index 604ee0f553..65ba542d5f 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -32,6 +32,12 @@ resource "aws_iam_role_policy_attachment" "fault_injection_simulator_ssm_access"
   provider   = aws.global
 }
 
+resource "aws_iam_role_policy_attachment" "cloudwatch_logs_full_access" {
+  role       = aws_iam_role.fault_injection_simulator.name
+  policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
+  provider   = aws.global
+}
+
 resource "aws_iam_role_policy" "fault_injection_simulator_additional_permissions" {
   name     = "additional-permissions"
   role     = aws_iam_role.fault_injection_simulator.name
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 3aed383eaf..3655b2bfcc 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -15,8 +15,7 @@ resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
 # Add resource policy to allow FIS or the FIS role to write logs - not working
 
 data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
-  provider  = aws.region
-  policy_id = "fis_app_ecs_tasks_service"
+  provider = aws.region
   statement {
     sid    = "AWSLogDeliveryWrite20150319"
     effect = "Allow"
@@ -35,19 +34,14 @@ data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
 
     resources = [
       "arn:aws:logs:*:*:log-group:/aws/fis/*",
-      "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
+      "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*",
+      "*"
     ]
 
     condition {
       test     = "StringEquals"
       variable = "aws:SourceAccount"
-      values   = [":${data.aws_caller_identity.current.account_id}"]
-    }
-
-    condition {
-      test     = "ArnLike"
-      variable = "aws:SourceArn"
-      values   = ["arn:aws:logs:${data.aws_region.current.name}::${data.aws_caller_identity.current.account_id}:*"]
+      values   = [data.aws_caller_identity.current.account_id]
     }
   }
 }
@@ -77,7 +71,8 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
   }
 
   statement {
-    sid = "AllowCloudWatchLogs"
+    sid    = "AllowCloudWatchLogs"
+    effect = "Allow"
     actions = [
       "logs:CreateLogDelivery",
       "logs:DescribeLogGroups",
@@ -92,6 +87,22 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
     ]
   }
 
+  statement {
+    sid    = "AllowFISExperimentRoleCloudwatch"
+    effect = "Allow"
+    actions = [
+      "logs:Describe*",
+      "logs:CreateLogDelivery",
+      "logs:PutLogEvents",
+      "logs:CreateLogStream",
+      "logs:PutResourcePolicy"
+    ]
+
+    resources = [
+      "*"
+    ]
+  }
+
   statement {
     sid    = "AllowLogDeliveryActions"
     effect = "Allow"

From 3a58c38b404100f3ca13d9a0b446dad23567a52e Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 24 Jan 2024 14:31:12 +0000
Subject: [PATCH 30/51] remove region from condition

---
 terraform/environment/global/iam_fault_injection_simulator.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index 65ba542d5f..6a9f2e9038 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -67,7 +67,7 @@ data "aws_iam_policy_document" "fault_injection_simulator_additional_permissions
     condition {
       test     = "ArnLike"
       variable = "aws:SourceArn"
-      values   = ["arn:aws:fis:${data.aws_region.global.name}:${data.aws_caller_identity.global.account_id}:experiment/*"]
+      values   = ["arn:aws:fis:*:${data.aws_caller_identity.global.account_id}:experiment/*"]
     }
   }
 }

From fdbb3ed99138a433038c50e19b0a9efb944d053b Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 24 Jan 2024 14:38:17 +0000
Subject: [PATCH 31/51] rename additional permission to create fis service
 linked role

---
 .../environment/global/iam_fault_injection_simulator.tf  | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index 6a9f2e9038..ee37e7c17f 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -38,14 +38,14 @@ resource "aws_iam_role_policy_attachment" "cloudwatch_logs_full_access" {
   provider   = aws.global
 }
 
-resource "aws_iam_role_policy" "fault_injection_simulator_additional_permissions" {
-  name     = "additional-permissions"
+resource "aws_iam_role_policy" "fault_injection_simulator_create_fis_service_linked_role" {
+  name     = "create-fis-service-linked-role-permissions"
   role     = aws_iam_role.fault_injection_simulator.name
-  policy   = data.aws_iam_policy_document.fault_injection_simulator_additional_permissions.json
+  policy   = data.aws_iam_policy_document.fault_injection_simulator_create_fis_service_linked_role.json
   provider = aws.global
 }
 
-data "aws_iam_policy_document" "fault_injection_simulator_additional_permissions" {
+data "aws_iam_policy_document" "fault_injection_simulator_create_fis_service_linked_role" {
   policy_id = "fix experiment permissions"
   statement {
     sid       = "AllowServiceLinkedRole"
@@ -72,7 +72,6 @@ data "aws_iam_policy_document" "fault_injection_simulator_additional_permissions
   }
 }
 
-
 # Create role for registering instance
 
 resource "aws_iam_role" "ssm_register_instance" {

From cbb47f1462f021e176d3466dd38395911a0f7c83 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 24 Jan 2024 14:42:29 +0000
Subject: [PATCH 32/51] cleanup

---
 .../region/modules/app/fault_injection_simulator.tf           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 3655b2bfcc..d73f948bf1 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -83,7 +83,7 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
 
     resources = [
       aws_cloudwatch_log_group.fis_app_ecs_tasks.arn,
-      "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
+      "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*",
     ]
   }
 
@@ -139,7 +139,7 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
     ]
     resources = [
       aws_cloudwatch_log_group.fis_app_ecs_tasks.arn,
-      "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*"
+      "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*",
     ]
   }
 }

From 5b3ab3d59dddf671a28c54400cb4618dd4f0e668 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Thu, 25 Jan 2024 18:30:26 +0000
Subject: [PATCH 33/51] fix lock

---
 terraform/environment/.terraform.lock.hcl | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/terraform/environment/.terraform.lock.hcl b/terraform/environment/.terraform.lock.hcl
index f37a04b7e0..2cb66e3111 100644
--- a/terraform/environment/.terraform.lock.hcl
+++ b/terraform/environment/.terraform.lock.hcl
@@ -45,11 +45,7 @@ provider "registry.terraform.io/hashicorp/local" {
 
 provider "registry.terraform.io/pagerduty/pagerduty" {
   version     = "3.4.1"
-<<<<<<< HEAD
   constraints = "3.4.1"
-=======
-  constraints = ">= 2.16.0, ~> 3.4.0"
->>>>>>> 1c4530c2 (update lock)
   hashes = [
     "h1:ntsWamEgQsmFukTV3vtgj6NFowleLE7V3l6U4dW2nOo=",
     "zh:0ac31f1a07ed501a1d14025b3d1196cfe06f9c96da010da50bc360c5186a514f",

From 4f76a6f777f8aa751b42eb650715413be610569b Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Thu, 25 Jan 2024 18:41:14 +0000
Subject: [PATCH 34/51] add caller identity

---
 terraform/environment/global/data_sources.tf             | 4 ++++
 terraform/environment/region/modules/app/data_sources.tf | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/terraform/environment/global/data_sources.tf b/terraform/environment/global/data_sources.tf
index 591c6a74d6..5e1cd451a7 100644
--- a/terraform/environment/global/data_sources.tf
+++ b/terraform/environment/global/data_sources.tf
@@ -1,3 +1,7 @@
 data "aws_default_tags" "current" {
   provider = aws.global
 }
+
+data "aws_caller_identity" "global" {
+  provider = aws.global
+}
diff --git a/terraform/environment/region/modules/app/data_sources.tf b/terraform/environment/region/modules/app/data_sources.tf
index 606b690309..2efbdedd48 100644
--- a/terraform/environment/region/modules/app/data_sources.tf
+++ b/terraform/environment/region/modules/app/data_sources.tf
@@ -5,3 +5,7 @@ data "aws_region" "current" {
 data "aws_default_tags" "current" {
   provider = aws.region
 }
+
+data "aws_caller_identity" "current" {
+  provider = aws.region
+}

From 4e167faefa9f51bde9ec79d68ec7fc7687b74f3d Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Mon, 29 Jan 2024 19:25:58 +0000
Subject: [PATCH 35/51] readonly root filesystem

---
 .../modules/app/fault_injection_simulator.tf      | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index d73f948bf1..90fb691bd1 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -205,13 +205,14 @@ resource "aws_fis_experiment_template" "ecs_app" {
 locals {
   amazon_ssm_agent = jsonencode(
     {
-      name         = "amazon-ssm-agent",
-      image        = "public.ecr.aws/amazon-ssm-agent/amazon-ssm-agent:latest",
-      cpu          = 0,
-      links        = [],
-      portMappings = [],
-      essential    = false,
-      entryPoint   = [],
+      name                   = "amazon-ssm-agent",
+      image                  = "public.ecr.aws/amazon-ssm-agent/amazon-ssm-agent:latest",
+      cpu                    = 0,
+      links                  = [],
+      portMappings           = [],
+      essential              = false,
+      entryPoint             = [],
+      readonlyRootFilesystem = true
       command = [
         "/bin/bash",
         "-c",

From 02c0447002fb384df493a4f60d81216d777573f0 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Mon, 29 Jan 2024 19:30:34 +0000
Subject: [PATCH 36/51] root access is needed for now

---
 .../environment/region/modules/app/fault_injection_simulator.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 90fb691bd1..630bd193c4 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -212,7 +212,7 @@ locals {
       portMappings           = [],
       essential              = false,
       entryPoint             = [],
-      readonlyRootFilesystem = true
+      readonlyRootFilesystem = false
       command = [
         "/bin/bash",
         "-c",

From fe60c21f5d1966e6514a68fcfb272489f5bb4c62 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Fri, 2 Feb 2024 18:19:48 +0000
Subject: [PATCH 37/51] clean up

---
 terraform/environment/global/README.md              | 13 +++++++++++++
 terraform/environment/region/README.md              |  2 +-
 terraform/environment/region/modules/app/README.md  | 12 ++++++++++++
 .../region/modules/app/fault_injection_simulator.tf |  6 ++----
 4 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/terraform/environment/global/README.md b/terraform/environment/global/README.md
index 0b0e0e1fe6..b88c29069c 100644
--- a/terraform/environment/global/README.md
+++ b/terraform/environment/global/README.md
@@ -65,6 +65,7 @@ No modules.
 
 | Name | Version |
 |------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.34.0 |
 | <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | ~> 5.34.0 |
 
 ## Modules
@@ -79,15 +80,27 @@ No modules.
 | [aws_iam_role.app_task_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.cross_account_put](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role.fault_injection_simulator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.s3_antivirus](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role.ssm_register_instance](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role_policy.execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_iam_role_policy.fault_injection_simulator_create_fis_service_linked_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_iam_role_policy.ssm_register_instance_permissions](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_iam_role_policy_attachment.cloudwatch_logs_full_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.fault_injection_simulator_ecs_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.fault_injection_simulator_ssm_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.s3_antivirus_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_resourcegroups_group.environment_global](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/resourcegroups_group) | resource |
+| [aws_caller_identity.global](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_default_tags.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source |
 | [aws_iam_policy_document.cross_account_put_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.execution_role_assume_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.fault_injection_simulator_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.fault_injection_simulator_create_fis_service_linked_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.lambda_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.ssm_register_instance_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.ssm_register_instance_permissions](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.task_role_assume_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 
 ## Inputs
diff --git a/terraform/environment/region/README.md b/terraform/environment/region/README.md
index e85a3c2559..acedbb81ed 100644
--- a/terraform/environment/region/README.md
+++ b/terraform/environment/region/README.md
@@ -109,7 +109,7 @@ This module creates the regional resources for an environment.
 | <a name="input_dns_weighting"></a> [dns\_weighting](#input\_dns\_weighting) | Weighting for DNS records | `number` | n/a | yes |
 | <a name="input_ecs_capacity_provider"></a> [ecs\_capacity\_provider](#input\_ecs\_capacity\_provider) | Name of the capacity provider to use. Valid values are FARGATE\_SPOT and FARGATE | `string` | n/a | yes |
 | <a name="input_ecs_task_autoscaling"></a> [ecs\_task\_autoscaling](#input\_ecs\_task\_autoscaling) | task minimum and maximum values for autoscaling | `any` | n/a | yes |
-| <a name="input_iam_roles"></a> [iam\_roles](#input\_iam\_roles) | ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services. | <pre>object({<br>    ecs_execution_role = any<br>    app_ecs_task_role  = any<br>    s3_antivirus       = any<br>    cross_account_put  = any<br>  })</pre> | n/a | yes |
+| <a name="input_iam_roles"></a> [iam\_roles](#input\_iam\_roles) | ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services. | <pre>object({<br>    ecs_execution_role        = any<br>    app_ecs_task_role         = any<br>    s3_antivirus              = any<br>    cross_account_put         = any<br>    fault_injection_simulator = any<br>  })</pre> | n/a | yes |
 | <a name="input_ingress_allow_list_cidr"></a> [ingress\_allow\_list\_cidr](#input\_ingress\_allow\_list\_cidr) | List of CIDR ranges permitted to access the service | `list(string)` | n/a | yes |
 | <a name="input_lpa_store_service"></a> [lpa\_store\_service](#input\_lpa\_store\_service) | n/a | <pre>object({<br>    base_url = string<br>    api_arns = list(string)<br>  })</pre> | n/a | yes |
 | <a name="input_lpas_table"></a> [lpas\_table](#input\_lpas\_table) | DynamoDB table for storing LPAs | `any` | n/a | yes |
diff --git a/terraform/environment/region/modules/app/README.md b/terraform/environment/region/modules/app/README.md
index 00066eb3df..1aa8a8164e 100644
--- a/terraform/environment/region/modules/app/README.md
+++ b/terraform/environment/region/modules/app/README.md
@@ -14,6 +14,7 @@ The module creates an ECS service for the Modernising LPA application, and assoc
 
 | Name | Version |
 |------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.34.0 |
 | <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
 
 ## Modules
@@ -24,9 +25,13 @@ No modules.
 
 | Name | Type |
 |------|------|
+| [aws_cloudwatch_log_group.fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
+| [aws_cloudwatch_log_resource_policy.fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_resource_policy) | resource |
 | [aws_ecs_service.app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
 | [aws_ecs_task_definition.app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
+| [aws_fis_experiment_template.ecs_app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/fis_experiment_template) | resource |
 | [aws_iam_role_policy.app_task_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_iam_role_policy.fis_role_log_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
 | [aws_lb.app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb) | resource |
 | [aws_lb_listener.app_loadbalancer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_listener) | resource |
 | [aws_lb_listener.app_loadbalancer_http_redirect](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_listener) | resource |
@@ -46,9 +51,15 @@ No modules.
 | [aws_security_group_rule.loadbalancer_ingress_route53_healthchecks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_wafv2_web_acl_association.app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/wafv2_web_acl_association) | resource |
 | [aws_acm_certificate.certificate_app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/acm_certificate) | data source |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_default_tags.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source |
+| [aws_iam_policy_document.cloudwatch_log_group_policy_fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.combined](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.ecs_task_role_fis_related_task_permissions](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.fis_role_log_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.task_role_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_ip_ranges.route53_healthchecks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ip_ranges) | data source |
+| [aws_kms_alias.cloudwatch_application_logs_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
 | [aws_kms_alias.dynamodb_encryption_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
 | [aws_kms_alias.reduced_fees_uploads_s3_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
 | [aws_kms_alias.secrets_manager_secret_encryption_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
@@ -82,6 +93,7 @@ No modules.
 | <a name="input_ecs_service_desired_count"></a> [ecs\_service\_desired\_count](#input\_ecs\_service\_desired\_count) | Number of instances of the task definition to place and keep running. Defaults to 0. Do not specify if using the DAEMON scheduling strategy. | `number` | `0` | no |
 | <a name="input_ecs_task_role"></a> [ecs\_task\_role](#input\_ecs\_task\_role) | ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services. | `any` | n/a | yes |
 | <a name="input_event_bus"></a> [event\_bus](#input\_event\_bus) | Name and ARN of the event bus to send events to | <pre>object({<br>    name = string<br>    arn  = string<br>  })</pre> | n/a | yes |
+| <a name="input_fault_injection_simulator_role"></a> [fault\_injection\_simulator\_role](#input\_fault\_injection\_simulator\_role) | ARN of IAM role that allows AWS FIS to make calls to other AWS services. | `any` | n/a | yes |
 | <a name="input_ingress_allow_list_cidr"></a> [ingress\_allow\_list\_cidr](#input\_ingress\_allow\_list\_cidr) | List of CIDR ranges permitted to access the service | `list(string)` | n/a | yes |
 | <a name="input_lpa_store_base_url"></a> [lpa\_store\_base\_url](#input\_lpa\_store\_base\_url) | n/a | `string` | n/a | yes |
 | <a name="input_lpas_table"></a> [lpas\_table](#input\_lpas\_table) | DynamoDB table for storing LPAs | `any` | n/a | yes |
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/app/fault_injection_simulator.tf
index 630bd193c4..a3b2abd6a2 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/app/fault_injection_simulator.tf
@@ -22,7 +22,7 @@ data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
 
     principals {
       identifiers = [
-        "fis.amazonaws.com"
+        "delivery.logs.amazonaws.com"
       ]
       type = "Service"
     }
@@ -33,9 +33,7 @@ data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
     ]
 
     resources = [
-      "arn:aws:logs:*:*:log-group:/aws/fis/*",
       "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*",
-      "*"
     ]
 
     condition {
@@ -50,7 +48,7 @@ data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
 resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
   provider        = aws.region
   policy_document = data.aws_iam_policy_document.cloudwatch_log_group_policy_fis_app_ecs_tasks.json
-  policy_name     = "fis_app_ecs_tasks"
+  policy_name     = "fis_app_ecs_tasks_logging"
 }
 
 # Add log encryption and log write/delivery permissions to the FIS role

From 052f30f09fc9cbcbd4362452433acf9aa71e19c0 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 6 Feb 2024 10:06:10 +0000
Subject: [PATCH 38/51] create var for enabling fault ionjection experiments

---
 terraform/account/README.md                   | 12 ++++-----
 terraform/account/region/README.md            | 10 +++----
 .../modules/antivirus_definitions/README.md   |  4 +--
 .../region/modules/dns_firewall/README.md     |  4 +--
 .../modules/s3_batch_manifests/README.md      |  4 +--
 .../s3_bucket_event_notifications/README.md   |  4 +--
 terraform/environment/README.md               | 18 ++++++-------
 terraform/environment/global/README.md        |  8 +++---
 terraform/environment/region/README.md        | 26 +++++++++----------
 .../environment/region/modules/app/README.md  |  6 ++---
 .../region/modules/application_logs/README.md |  4 +--
 .../region/modules/ecs_autoscaling/README.md  |  4 +--
 .../region/modules/event_bus/README.md        |  6 ++---
 .../region/modules/event_received/README.md   |  6 ++---
 .../region/modules/lambda/README.md           |  4 +--
 .../region/modules/mock_onelogin/README.md    |  4 +--
 .../region/modules/s3_antivirus/README.md     |  4 +--
 .../modules/uploads_s3_bucket/README.md       |  4 +--
 terraform/environment/terraform.tfvars.json   | 18 ++++++++-----
 terraform/environment/variables.tf            |  1 +
 20 files changed, 79 insertions(+), 72 deletions(-)

diff --git a/terraform/account/README.md b/terraform/account/README.md
index 56f5996592..7ef528eda8 100644
--- a/terraform/account/README.md
+++ b/terraform/account/README.md
@@ -90,17 +90,17 @@ For terraform_environment, this will be based on your PR and can be found in the
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | 1.7.1 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | 1.7.2 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.eu_west_1"></a> [aws.eu\_west\_1](#provider\_aws.eu\_west\_1) | 5.34.0 |
-| <a name="provider_aws.eu_west_2"></a> [aws.eu\_west\_2](#provider\_aws.eu\_west\_2) | 5.34.0 |
-| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | 5.34.0 |
-| <a name="provider_aws.management_global"></a> [aws.management\_global](#provider\_aws.management\_global) | 5.34.0 |
+| <a name="provider_aws.eu_west_1"></a> [aws.eu\_west\_1](#provider\_aws.eu\_west\_1) | 5.35.0 |
+| <a name="provider_aws.eu_west_2"></a> [aws.eu\_west\_2](#provider\_aws.eu\_west\_2) | 5.35.0 |
+| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | 5.35.0 |
+| <a name="provider_aws.management_global"></a> [aws.management\_global](#provider\_aws.management\_global) | 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/account/region/README.md b/terraform/account/region/README.md
index 03932b080b..95bb8d42d1 100644
--- a/terraform/account/region/README.md
+++ b/terraform/account/region/README.md
@@ -4,16 +4,16 @@
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.34.0 |
-| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | ~> 5.34.0 |
-| <a name="provider_aws.management"></a> [aws.management](#provider\_aws.management) | ~> 5.34.0 |
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.35.0 |
+| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | ~> 5.35.0 |
+| <a name="provider_aws.management"></a> [aws.management](#provider\_aws.management) | ~> 5.35.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/account/region/modules/antivirus_definitions/README.md b/terraform/account/region/modules/antivirus_definitions/README.md
index a93b903274..6b30d6bb4d 100644
--- a/terraform/account/region/modules/antivirus_definitions/README.md
+++ b/terraform/account/region/modules/antivirus_definitions/README.md
@@ -8,13 +8,13 @@ This module creates a S3 bucket for antivirus definitions, and a Lambda function
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/account/region/modules/dns_firewall/README.md b/terraform/account/region/modules/dns_firewall/README.md
index 972161d78c..2921b18ae6 100644
--- a/terraform/account/region/modules/dns_firewall/README.md
+++ b/terraform/account/region/modules/dns_firewall/README.md
@@ -8,13 +8,13 @@ This module creates a DNS Firewall rule group and  rule group associations.
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/account/region/modules/s3_batch_manifests/README.md b/terraform/account/region/modules/s3_batch_manifests/README.md
index 07d8d8f015..47aa07fe07 100644
--- a/terraform/account/region/modules/s3_batch_manifests/README.md
+++ b/terraform/account/region/modules/s3_batch_manifests/README.md
@@ -8,13 +8,13 @@ This module creates a S3 bucket for S3 Batch Job Manifests.
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/account/region/modules/s3_bucket_event_notifications/README.md b/terraform/account/region/modules/s3_bucket_event_notifications/README.md
index 4fa9e69bd9..042c227d04 100644
--- a/terraform/account/region/modules/s3_bucket_event_notifications/README.md
+++ b/terraform/account/region/modules/s3_bucket_event_notifications/README.md
@@ -8,13 +8,13 @@ This module creates a S3 bucket event notifications and event notification filte
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.34.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/README.md b/terraform/environment/README.md
index cee91d873c..5f6ae95de7 100644
--- a/terraform/environment/README.md
+++ b/terraform/environment/README.md
@@ -113,19 +113,19 @@ For terraform_environment, this will be based on your PR and can be found in the
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | 1.7.1 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
-| <a name="requirement_pagerduty"></a> [pagerduty](#requirement\_pagerduty) | 3.5.2 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | 1.7.2 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
+| <a name="requirement_pagerduty"></a> [pagerduty](#requirement\_pagerduty) | 3.7.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.eu_west_1"></a> [aws.eu\_west\_1](#provider\_aws.eu\_west\_1) | 5.34.0 |
-| <a name="provider_aws.eu_west_2"></a> [aws.eu\_west\_2](#provider\_aws.eu\_west\_2) | 5.34.0 |
-| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | 5.34.0 |
-| <a name="provider_aws.management_eu_west_1"></a> [aws.management\_eu\_west\_1](#provider\_aws.management\_eu\_west\_1) | 5.34.0 |
-| <a name="provider_aws.management_global"></a> [aws.management\_global](#provider\_aws.management\_global) | 5.34.0 |
+| <a name="provider_aws.eu_west_1"></a> [aws.eu\_west\_1](#provider\_aws.eu\_west\_1) | 5.35.0 |
+| <a name="provider_aws.eu_west_2"></a> [aws.eu\_west\_2](#provider\_aws.eu\_west\_2) | 5.35.0 |
+| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | 5.35.0 |
+| <a name="provider_aws.management_eu_west_1"></a> [aws.management\_eu\_west\_1](#provider\_aws.management\_eu\_west\_1) | 5.35.0 |
+| <a name="provider_aws.management_global"></a> [aws.management\_global](#provider\_aws.management\_global) | 5.35.0 |
 
 ## Modules
 
@@ -167,7 +167,7 @@ For terraform_environment, this will be based on your PR and can be found in the
 |------|-------------|------|---------|:--------:|
 | <a name="input_container_version"></a> [container\_version](#input\_container\_version) | n/a | `string` | `"latest"` | no |
 | <a name="input_default_role"></a> [default\_role](#input\_default\_role) | n/a | `string` | `"modernising-lpa-ci"` | no |
-| <a name="input_environments"></a> [environments](#input\_environments) | n/a | <pre>map(<br>    object({<br>      account_id    = string<br>      account_name  = string<br>      is_production = bool<br>      regions       = list(string)<br>      app = object({<br>        env = object({<br>          app_public_url         = string<br>          auth_redirect_base_url = string<br>          notify_is_production   = string<br>          onelogin_url           = string<br>        })<br>        autoscaling = object({<br>          minimum = number<br>          maximum = number<br>        })<br>        dependency_health_check_alarm_enabled   = bool<br>        service_health_check_alarm_enabled      = bool<br>        cloudwatch_application_insights_enabled = bool<br>      })<br>      mock_onelogin_enabled = bool<br>      uid_service = object({<br>        base_url = string<br>        api_arns = list(string)<br>      })<br>      lpa_store_service = object({<br>        base_url = string<br>        api_arns = list(string)<br>      })<br>      backups = object({<br>        backup_plan_enabled = bool<br>        copy_action_enabled = bool<br>      })<br>      dynamodb = object({<br>        region_replica_enabled = bool<br>        stream_enabled         = bool<br>      })<br>      ecs = object({<br>        fargate_spot_capacity_provider_enabled = bool<br><br>      })<br>      cloudwatch_log_groups = object({<br>        application_log_retention_days = number<br>      })<br>      application_load_balancer = object({<br>        deletion_protection_enabled = bool<br>      })<br>      cloudwatch_application_insights_enabled = bool<br>      pagerduty_service_name                  = string<br>      event_bus = object({<br>        target_event_bus_arn = string<br>        receive_account_ids  = list(string)<br>      })<br>      reduced_fees = object({<br>        enabled                                   = bool<br>        s3_object_replication_enabled             = bool<br>        target_environment                        = string<br>        destination_account_id                    = string<br>        enable_s3_batch_job_replication_scheduler = bool<br>      })<br>      s3_antivirus_provisioned_concurrency = number<br>    })<br>  )</pre> | n/a | yes |
+| <a name="input_environments"></a> [environments](#input\_environments) | n/a | <pre>map(<br>    object({<br>      account_id    = string<br>      account_name  = string<br>      is_production = bool<br>      regions       = list(string)<br>      app = object({<br>        env = object({<br>          app_public_url         = string<br>          auth_redirect_base_url = string<br>          notify_is_production   = string<br>          onelogin_url           = string<br>        })<br>        autoscaling = object({<br>          minimum = number<br>          maximum = number<br>        })<br>        dependency_health_check_alarm_enabled   = bool<br>        service_health_check_alarm_enabled      = bool<br>        cloudwatch_application_insights_enabled = bool<br>        fault_injection_enabled                 = bool<br>      })<br>      mock_onelogin_enabled = bool<br>      uid_service = object({<br>        base_url = string<br>        api_arns = list(string)<br>      })<br>      lpa_store_service = object({<br>        base_url = string<br>        api_arns = list(string)<br>      })<br>      backups = object({<br>        backup_plan_enabled = bool<br>        copy_action_enabled = bool<br>      })<br>      dynamodb = object({<br>        region_replica_enabled = bool<br>        stream_enabled         = bool<br>      })<br>      ecs = object({<br>        fargate_spot_capacity_provider_enabled = bool<br><br>      })<br>      cloudwatch_log_groups = object({<br>        application_log_retention_days = number<br>      })<br>      application_load_balancer = object({<br>        deletion_protection_enabled = bool<br>      })<br>      cloudwatch_application_insights_enabled = bool<br>      pagerduty_service_name                  = string<br>      event_bus = object({<br>        target_event_bus_arn = string<br>        receive_account_ids  = list(string)<br>      })<br>      reduced_fees = object({<br>        enabled                                   = bool<br>        s3_object_replication_enabled             = bool<br>        target_environment                        = string<br>        destination_account_id                    = string<br>        enable_s3_batch_job_replication_scheduler = bool<br>      })<br>      s3_antivirus_provisioned_concurrency = number<br>    })<br>  )</pre> | n/a | yes |
 | <a name="input_pagerduty_api_key"></a> [pagerduty\_api\_key](#input\_pagerduty\_api\_key) | n/a | `string` | n/a | yes |
 | <a name="input_public_access_enabled"></a> [public\_access\_enabled](#input\_public\_access\_enabled) | n/a | `bool` | `false` | no |
 
diff --git a/terraform/environment/global/README.md b/terraform/environment/global/README.md
index b88c29069c..fd07a93374 100644
--- a/terraform/environment/global/README.md
+++ b/terraform/environment/global/README.md
@@ -58,15 +58,15 @@ No modules.
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
-| <a name="requirement_pagerduty"></a> [pagerduty](#requirement\_pagerduty) | 3.5.2 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
+| <a name="requirement_pagerduty"></a> [pagerduty](#requirement\_pagerduty) | 3.7.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.34.0 |
-| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | ~> 5.34.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.35.0 |
+| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/region/README.md b/terraform/environment/region/README.md
index acedbb81ed..c4652221af 100644
--- a/terraform/environment/region/README.md
+++ b/terraform/environment/region/README.md
@@ -8,18 +8,18 @@ This module creates the regional resources for an environment.
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
-| <a name="requirement_pagerduty"></a> [pagerduty](#requirement\_pagerduty) | 3.5.2 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
+| <a name="requirement_pagerduty"></a> [pagerduty](#requirement\_pagerduty) | 3.7.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | ~> 5.34.0 |
-| <a name="provider_aws.management"></a> [aws.management](#provider\_aws.management) | ~> 5.34.0 |
-| <a name="provider_aws.management_global"></a> [aws.management\_global](#provider\_aws.management\_global) | ~> 5.34.0 |
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
-| <a name="provider_pagerduty"></a> [pagerduty](#provider\_pagerduty) | 3.5.2 |
+| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | ~> 5.35.0 |
+| <a name="provider_aws.management"></a> [aws.management](#provider\_aws.management) | ~> 5.35.0 |
+| <a name="provider_aws.management_global"></a> [aws.management\_global](#provider\_aws.management\_global) | ~> 5.35.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
+| <a name="provider_pagerduty"></a> [pagerduty](#provider\_pagerduty) | 3.7.0 |
 
 ## Modules
 
@@ -60,10 +60,10 @@ This module creates the regional resources for an environment.
 | [aws_sns_topic_subscription.dependency_health_check](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_subscription) | resource |
 | [aws_sns_topic_subscription.ecs_autoscaling_alarms](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_subscription) | resource |
 | [aws_sns_topic_subscription.service_health_check](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_subscription) | resource |
-| [pagerduty_service_integration.cloudwatch_application_insights](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.5.2/docs/resources/service_integration) | resource |
-| [pagerduty_service_integration.dependency_health_check](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.5.2/docs/resources/service_integration) | resource |
-| [pagerduty_service_integration.ecs_autoscaling_alarms](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.5.2/docs/resources/service_integration) | resource |
-| [pagerduty_service_integration.service_health_check](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.5.2/docs/resources/service_integration) | resource |
+| [pagerduty_service_integration.cloudwatch_application_insights](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.7.0/docs/resources/service_integration) | resource |
+| [pagerduty_service_integration.dependency_health_check](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.7.0/docs/resources/service_integration) | resource |
+| [pagerduty_service_integration.ecs_autoscaling_alarms](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.7.0/docs/resources/service_integration) | resource |
+| [pagerduty_service_integration.service_health_check](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.7.0/docs/resources/service_integration) | resource |
 | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_default_tags.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source |
@@ -92,8 +92,8 @@ This module creates the regional resources for an environment.
 | [aws_subnet.application](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source |
 | [aws_subnet.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source |
 | [aws_vpc.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source |
-| [pagerduty_service.main](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.5.2/docs/data-sources/service) | data source |
-| [pagerduty_vendor.cloudwatch](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.5.2/docs/data-sources/vendor) | data source |
+| [pagerduty_service.main](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.7.0/docs/data-sources/service) | data source |
+| [pagerduty_vendor.cloudwatch](https://registry.terraform.io/providers/PagerDuty/pagerduty/3.7.0/docs/data-sources/vendor) | data source |
 
 ## Inputs
 
diff --git a/terraform/environment/region/modules/app/README.md b/terraform/environment/region/modules/app/README.md
index 1aa8a8164e..84b760ec25 100644
--- a/terraform/environment/region/modules/app/README.md
+++ b/terraform/environment/region/modules/app/README.md
@@ -8,14 +8,14 @@ The module creates an ECS service for the Modernising LPA application, and assoc
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.34.0 |
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.35.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/region/modules/application_logs/README.md b/terraform/environment/region/modules/application_logs/README.md
index ce4c2106ac..b92fbcddf7 100644
--- a/terraform/environment/region/modules/application_logs/README.md
+++ b/terraform/environment/region/modules/application_logs/README.md
@@ -8,13 +8,13 @@ The module creates a cloudwatch log group and useful log queries for application
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/region/modules/ecs_autoscaling/README.md b/terraform/environment/region/modules/ecs_autoscaling/README.md
index 2b61f01409..2d2a473df5 100644
--- a/terraform/environment/region/modules/ecs_autoscaling/README.md
+++ b/terraform/environment/region/modules/ecs_autoscaling/README.md
@@ -8,13 +8,13 @@ This module creates the autoscaling resources for an ECS service.
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/region/modules/event_bus/README.md b/terraform/environment/region/modules/event_bus/README.md
index b982c7dd88..fa66554596 100644
--- a/terraform/environment/region/modules/event_bus/README.md
+++ b/terraform/environment/region/modules/event_bus/README.md
@@ -18,14 +18,14 @@ aws-vault exec mlpa-dev -- aws events put-events --entries file://reduced_fees_u
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.34.0 |
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.35.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/region/modules/event_received/README.md b/terraform/environment/region/modules/event_received/README.md
index 6a32717b21..403a471557 100644
--- a/terraform/environment/region/modules/event_received/README.md
+++ b/terraform/environment/region/modules/event_received/README.md
@@ -8,14 +8,14 @@ This module creates the resources required to receive and process events from th
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.34.0 |
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.35.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/region/modules/lambda/README.md b/terraform/environment/region/modules/lambda/README.md
index f0f6c093ed..5e64b2910f 100644
--- a/terraform/environment/region/modules/lambda/README.md
+++ b/terraform/environment/region/modules/lambda/README.md
@@ -8,13 +8,13 @@ This module creates the resources required to deploy an image based Lambda funct
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/region/modules/mock_onelogin/README.md b/terraform/environment/region/modules/mock_onelogin/README.md
index fd2f93344c..3f061e3334 100644
--- a/terraform/environment/region/modules/mock_onelogin/README.md
+++ b/terraform/environment/region/modules/mock_onelogin/README.md
@@ -8,13 +8,13 @@ This module creates the resources required to mock One Login.
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/region/modules/s3_antivirus/README.md b/terraform/environment/region/modules/s3_antivirus/README.md
index 0ec0f03765..f6a4e5f351 100644
--- a/terraform/environment/region/modules/s3_antivirus/README.md
+++ b/terraform/environment/region/modules/s3_antivirus/README.md
@@ -8,13 +8,13 @@ This module deploys a lambda function that scans S3 objects for viruses on put.
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/region/modules/uploads_s3_bucket/README.md b/terraform/environment/region/modules/uploads_s3_bucket/README.md
index 399cd2a0f5..6c2b0fd0ed 100644
--- a/terraform/environment/region/modules/uploads_s3_bucket/README.md
+++ b/terraform/environment/region/modules/uploads_s3_bucket/README.md
@@ -8,13 +8,13 @@ This module creates an S3 bucket for storing uploads, triggers for virus scannin
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.34.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.34.0 |
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
 
 ## Modules
 
diff --git a/terraform/environment/terraform.tfvars.json b/terraform/environment/terraform.tfvars.json
index 2d770078b9..f383cb6580 100644
--- a/terraform/environment/terraform.tfvars.json
+++ b/terraform/environment/terraform.tfvars.json
@@ -20,7 +20,8 @@
         },
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,
-        "cloudwatch_application_insights_enabled": false
+        "cloudwatch_application_insights_enabled": false,
+        "fault_injection_enabled": false
       },
       "mock_onelogin_enabled": false,
       "uid_service": {
@@ -95,7 +96,8 @@
         },
         "dependency_health_check_alarm_enabled": true,
         "service_health_check_alarm_enabled": true,
-        "cloudwatch_application_insights_enabled": true
+        "cloudwatch_application_insights_enabled": true,
+        "fault_injection_enabled": false
       },
       "mock_onelogin_enabled": true,
       "uid_service": {
@@ -170,7 +172,8 @@
         },
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,
-        "cloudwatch_application_insights_enabled": false
+        "cloudwatch_application_insights_enabled": false,
+        "fault_injection_enabled": false
       },
       "mock_onelogin_enabled": true,
       "uid_service": {
@@ -245,7 +248,8 @@
         },
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,
-        "cloudwatch_application_insights_enabled": true
+        "cloudwatch_application_insights_enabled": true,
+        "fault_injection_enabled": false
       },
       "mock_onelogin_enabled": true,
       "uid_service": {
@@ -320,7 +324,8 @@
         },
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,
-        "cloudwatch_application_insights_enabled": true
+        "cloudwatch_application_insights_enabled": true,
+        "fault_injection_enabled": false
       },
       "mock_onelogin_enabled": false,
       "uid_service": {
@@ -395,7 +400,8 @@
         },
         "dependency_health_check_alarm_enabled": true,
         "service_health_check_alarm_enabled": true,
-        "cloudwatch_application_insights_enabled": true
+        "cloudwatch_application_insights_enabled": true,
+        "fault_injection_enabled": false
       },
       "mock_onelogin_enabled": false,
       "uid_service": {
diff --git a/terraform/environment/variables.tf b/terraform/environment/variables.tf
index 916d429787..098a5d742a 100644
--- a/terraform/environment/variables.tf
+++ b/terraform/environment/variables.tf
@@ -46,6 +46,7 @@ variable "environments" {
         dependency_health_check_alarm_enabled   = bool
         service_health_check_alarm_enabled      = bool
         cloudwatch_application_insights_enabled = bool
+        fault_injection_enabled                 = bool
       })
       mock_onelogin_enabled = bool
       uid_service = object({

From d4cab80c39bd02b2e8409d23657e05e3e04e632f Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 6 Feb 2024 10:33:56 +0000
Subject: [PATCH 39/51] refactor experiments to it's own module

---
 terraform/environment/region/README.md        |  2 +
 terraform/environment/region/ecs.tf           |  1 +
 .../fault_injections_simulator_experiments.tf |  8 ++
 .../environment/region/modules/app/README.md  |  9 --
 .../region/modules/app/data_sources.tf        |  4 -
 .../environment/region/modules/app/ecs.tf     | 80 ++++++++++++++++-
 .../region/modules/app/variables.tf           | 14 +--
 .../README.md                                 | 43 ++++++++++
 .../data_sources.tf                           | 11 +++
 .../main.tf}                                  | 85 +------------------
 .../variables.tf                              |  5 ++
 .../versions.tf                               | 13 +++
 terraform/environment/region/variables.tf     |  5 ++
 terraform/environment/regions.tf              |  2 +
 14 files changed, 177 insertions(+), 105 deletions(-)
 create mode 100644 terraform/environment/region/fault_injections_simulator_experiments.tf
 create mode 100644 terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
 create mode 100644 terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
 rename terraform/environment/region/modules/{app/fault_injection_simulator.tf => fault_injection_simulator_experiments/main.tf} (51%)
 create mode 100644 terraform/environment/region/modules/fault_injection_simulator_experiments/variables.tf
 create mode 100644 terraform/environment/region/modules/fault_injection_simulator_experiments/versions.tf

diff --git a/terraform/environment/region/README.md b/terraform/environment/region/README.md
index c4652221af..ae4ac85a2e 100644
--- a/terraform/environment/region/README.md
+++ b/terraform/environment/region/README.md
@@ -30,6 +30,7 @@ This module creates the regional resources for an environment.
 | <a name="module_application_logs"></a> [application\_logs](#module\_application\_logs) | ./modules/application_logs | n/a |
 | <a name="module_event_bus"></a> [event\_bus](#module\_event\_bus) | ./modules/event_bus | n/a |
 | <a name="module_event_received"></a> [event\_received](#module\_event\_received) | ./modules/event_received | n/a |
+| <a name="module_fault_injection_simulator_experiments"></a> [fault\_injection\_simulator\_experiments](#module\_fault\_injection\_simulator\_experiments) | ./modules/fault_injection_simulator_experiments | n/a |
 | <a name="module_mock_onelogin"></a> [mock\_onelogin](#module\_mock\_onelogin) | ./modules/mock_onelogin | n/a |
 | <a name="module_s3_antivirus"></a> [s3\_antivirus](#module\_s3\_antivirus) | ./modules/s3_antivirus | n/a |
 | <a name="module_uploads_s3_bucket"></a> [uploads\_s3\_bucket](#module\_uploads\_s3\_bucket) | ./modules/uploads_s3_bucket | n/a |
@@ -109,6 +110,7 @@ This module creates the regional resources for an environment.
 | <a name="input_dns_weighting"></a> [dns\_weighting](#input\_dns\_weighting) | Weighting for DNS records | `number` | n/a | yes |
 | <a name="input_ecs_capacity_provider"></a> [ecs\_capacity\_provider](#input\_ecs\_capacity\_provider) | Name of the capacity provider to use. Valid values are FARGATE\_SPOT and FARGATE | `string` | n/a | yes |
 | <a name="input_ecs_task_autoscaling"></a> [ecs\_task\_autoscaling](#input\_ecs\_task\_autoscaling) | task minimum and maximum values for autoscaling | `any` | n/a | yes |
+| <a name="input_fault_injection_enabled"></a> [fault\_injection\_enabled](#input\_fault\_injection\_enabled) | Enable fault injection | `bool` | n/a | yes |
 | <a name="input_iam_roles"></a> [iam\_roles](#input\_iam\_roles) | ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services. | <pre>object({<br>    ecs_execution_role        = any<br>    app_ecs_task_role         = any<br>    s3_antivirus              = any<br>    cross_account_put         = any<br>    fault_injection_simulator = any<br>  })</pre> | n/a | yes |
 | <a name="input_ingress_allow_list_cidr"></a> [ingress\_allow\_list\_cidr](#input\_ingress\_allow\_list\_cidr) | List of CIDR ranges permitted to access the service | `list(string)` | n/a | yes |
 | <a name="input_lpa_store_service"></a> [lpa\_store\_service](#input\_lpa\_store\_service) | n/a | <pre>object({<br>    base_url = string<br>    api_arns = list(string)<br>  })</pre> | n/a | yes |
diff --git a/terraform/environment/region/ecs.tf b/terraform/environment/region/ecs.tf
index aafd58f90e..cff60f963f 100644
--- a/terraform/environment/region/ecs.tf
+++ b/terraform/environment/region/ecs.tf
@@ -56,6 +56,7 @@ module "app" {
   uid_base_url                                         = var.uid_service.base_url
   lpa_store_base_url                                   = var.lpa_store_service.base_url
   mock_onelogin_enabled                                = data.aws_default_tags.current.tags.environment-name != "production" && var.mock_onelogin_enabled
+  fault_injection_enabled                              = var.fault_injection_enabled
   providers = {
     aws.region = aws.region
   }
diff --git a/terraform/environment/region/fault_injections_simulator_experiments.tf b/terraform/environment/region/fault_injections_simulator_experiments.tf
new file mode 100644
index 0000000000..2b221aa817
--- /dev/null
+++ b/terraform/environment/region/fault_injections_simulator_experiments.tf
@@ -0,0 +1,8 @@
+module "fault_injection_simulator_experiments" {
+  source = "./modules/fault_injection_simulator_experiments"
+  providers = {
+    aws.region = aws.region
+  }
+  fault_injection_simulator_role = var.iam_roles.fault_injection_simulator
+
+}
diff --git a/terraform/environment/region/modules/app/README.md b/terraform/environment/region/modules/app/README.md
index 84b760ec25..e6142ba379 100644
--- a/terraform/environment/region/modules/app/README.md
+++ b/terraform/environment/region/modules/app/README.md
@@ -25,13 +25,9 @@ No modules.
 
 | Name | Type |
 |------|------|
-| [aws_cloudwatch_log_group.fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
-| [aws_cloudwatch_log_resource_policy.fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_resource_policy) | resource |
 | [aws_ecs_service.app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
 | [aws_ecs_task_definition.app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
-| [aws_fis_experiment_template.ecs_app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/fis_experiment_template) | resource |
 | [aws_iam_role_policy.app_task_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
-| [aws_iam_role_policy.fis_role_log_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
 | [aws_lb.app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb) | resource |
 | [aws_lb_listener.app_loadbalancer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_listener) | resource |
 | [aws_lb_listener.app_loadbalancer_http_redirect](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_listener) | resource |
@@ -51,15 +47,11 @@ No modules.
 | [aws_security_group_rule.loadbalancer_ingress_route53_healthchecks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_wafv2_web_acl_association.app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/wafv2_web_acl_association) | resource |
 | [aws_acm_certificate.certificate_app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/acm_certificate) | data source |
-| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_default_tags.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source |
-| [aws_iam_policy_document.cloudwatch_log_group_policy_fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.combined](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.ecs_task_role_fis_related_task_permissions](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
-| [aws_iam_policy_document.fis_role_log_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.task_role_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_ip_ranges.route53_healthchecks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ip_ranges) | data source |
-| [aws_kms_alias.cloudwatch_application_logs_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
 | [aws_kms_alias.dynamodb_encryption_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
 | [aws_kms_alias.reduced_fees_uploads_s3_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
 | [aws_kms_alias.secrets_manager_secret_encryption_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
@@ -93,7 +85,6 @@ No modules.
 | <a name="input_ecs_service_desired_count"></a> [ecs\_service\_desired\_count](#input\_ecs\_service\_desired\_count) | Number of instances of the task definition to place and keep running. Defaults to 0. Do not specify if using the DAEMON scheduling strategy. | `number` | `0` | no |
 | <a name="input_ecs_task_role"></a> [ecs\_task\_role](#input\_ecs\_task\_role) | ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services. | `any` | n/a | yes |
 | <a name="input_event_bus"></a> [event\_bus](#input\_event\_bus) | Name and ARN of the event bus to send events to | <pre>object({<br>    name = string<br>    arn  = string<br>  })</pre> | n/a | yes |
-| <a name="input_fault_injection_simulator_role"></a> [fault\_injection\_simulator\_role](#input\_fault\_injection\_simulator\_role) | ARN of IAM role that allows AWS FIS to make calls to other AWS services. | `any` | n/a | yes |
 | <a name="input_ingress_allow_list_cidr"></a> [ingress\_allow\_list\_cidr](#input\_ingress\_allow\_list\_cidr) | List of CIDR ranges permitted to access the service | `list(string)` | n/a | yes |
 | <a name="input_lpa_store_base_url"></a> [lpa\_store\_base\_url](#input\_lpa\_store\_base\_url) | n/a | `string` | n/a | yes |
 | <a name="input_lpas_table"></a> [lpas\_table](#input\_lpas\_table) | DynamoDB table for storing LPAs | `any` | n/a | yes |
diff --git a/terraform/environment/region/modules/app/data_sources.tf b/terraform/environment/region/modules/app/data_sources.tf
index 2efbdedd48..606b690309 100644
--- a/terraform/environment/region/modules/app/data_sources.tf
+++ b/terraform/environment/region/modules/app/data_sources.tf
@@ -5,7 +5,3 @@ data "aws_region" "current" {
 data "aws_default_tags" "current" {
   provider = aws.region
 }
-
-data "aws_caller_identity" "current" {
-  provider = aws.region
-}
diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index 18506a002a..274ff64eaa 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -162,10 +162,6 @@ data "aws_secretsmanager_secret" "rum_monitor_identity_pool_id" {
   provider = aws.region
 }
 
-locals {
-  policy_region_prefix = lower(replace(data.aws_region.current.name, "-", ""))
-}
-
 data "aws_iam_policy_document" "task_role_access_policy" {
   policy_id = "${local.policy_region_prefix}task_role_access_policy"
   statement {
@@ -455,4 +451,80 @@ locals {
       },
       environment = []
   })
+
+  amazon_ssm_agent = jsonencode(
+    {
+      name                   = "amazon-ssm-agent",
+      image                  = "public.ecr.aws/amazon-ssm-agent/amazon-ssm-agent:latest",
+      cpu                    = 0,
+      links                  = [],
+      portMappings           = [],
+      essential              = false,
+      entryPoint             = [],
+      readonlyRootFilesystem = false
+      command = [
+        "/bin/bash",
+        "-c",
+        "set -e; yum upgrade -y; yum install jq procps awscli -y; term_handler() { echo \"Deleting SSM activation $ACTIVATION_ID\"; if ! aws ssm delete-activation --activation-id $ACTIVATION_ID --region $ECS_TASK_REGION; then echo \"SSM activation $ACTIVATION_ID failed to be deleted\" 1>&2; fi; MANAGED_INSTANCE_ID=$(jq -e -r .ManagedInstanceID /var/lib/amazon/ssm/registration); echo \"Deregistering SSM Managed Instance $MANAGED_INSTANCE_ID\"; if ! aws ssm deregister-managed-instance --instance-id $MANAGED_INSTANCE_ID --region $ECS_TASK_REGION; then echo \"SSM Managed Instance $MANAGED_INSTANCE_ID failed to be deregistered\" 1>&2; fi; kill -SIGTERM $SSM_AGENT_PID; }; trap term_handler SIGTERM SIGINT; if [[ -z $MANAGED_INSTANCE_ROLE_NAME ]]; then echo \"Environment variable MANAGED_INSTANCE_ROLE_NAME not set, exiting\" 1>&2; exit 1; fi; if ! ps ax | grep amazon-ssm-agent | grep -v grep > /dev/null; then if [[ -n $ECS_CONTAINER_METADATA_URI_V4 ]] ; then echo \"Found ECS Container Metadata, running activation with metadata\"; TASK_METADATA=$(curl \"$${ECS_CONTAINER_METADATA_URI_V4}/task\"); ECS_TASK_AVAILABILITY_ZONE=$(echo $TASK_METADATA | jq -e -r '.AvailabilityZone'); ECS_TASK_ARN=$(echo $TASK_METADATA | jq -e -r '.TaskARN'); ECS_TASK_REGION=$(echo $ECS_TASK_AVAILABILITY_ZONE | sed 's/.$//'); ECS_TASK_AVAILABILITY_ZONE_REGEX='^(af|ap|ca|cn|eu|me|sa|us|us-gov)-(central|north|(north(east|west))|south|south(east|west)|east|west)-[0-9]{1}[a-z]{1}$'; if ! [[ $ECS_TASK_AVAILABILITY_ZONE =~ $ECS_TASK_AVAILABILITY_ZONE_REGEX ]]; then echo \"Error extracting Availability Zone from ECS Container Metadata, exiting\" 1>&2; exit 1; fi; ECS_TASK_ARN_REGEX='^arn:(aws|aws-cn|aws-us-gov):ecs:[a-z0-9-]+:[0-9]{12}:task/[a-zA-Z0-9_-]+/[a-zA-Z0-9]+$'; if ! [[ $ECS_TASK_ARN =~ $ECS_TASK_ARN_REGEX ]]; then echo \"Error extracting Task ARN from ECS Container Metadata, exiting\" 1>&2; exit 1; fi; CREATE_ACTIVATION_OUTPUT=$(aws ssm create-activation --iam-role $MANAGED_INSTANCE_ROLE_NAME --tags Key=ECS_TASK_AVAILABILITY_ZONE,Value=$ECS_TASK_AVAILABILITY_ZONE Key=ECS_TASK_ARN,Value=$ECS_TASK_ARN Key=FAULT_INJECTION_SIDECAR,Value=true --region $ECS_TASK_REGION); ACTIVATION_CODE=$(echo $CREATE_ACTIVATION_OUTPUT | jq -e -r .ActivationCode); ACTIVATION_ID=$(echo $CREATE_ACTIVATION_OUTPUT | jq -e -r .ActivationId); if ! amazon-ssm-agent -register -code $ACTIVATION_CODE -id $ACTIVATION_ID -region $ECS_TASK_REGION; then echo \"Failed to register with AWS Systems Manager (SSM), exiting\" 1>&2; exit 1; fi; amazon-ssm-agent & SSM_AGENT_PID=$!; wait $SSM_AGENT_PID; else echo \"ECS Container Metadata not found, exiting\" 1>&2; exit 1; fi; else echo \"SSM agent is already running, exiting\" 1>&2; exit 1; fi"
+      ],
+      environment = [
+        {
+          name  = "MANAGED_INSTANCE_ROLE_NAME",
+          value = "ssm-register-instance-${data.aws_default_tags.current.tags.environment-name}"
+        }
+      ],
+      logConfiguration = {
+        logDriver = "awslogs",
+        options = {
+          awslogs-group         = var.ecs_application_log_group_name,
+          awslogs-region        = data.aws_region.current.name,
+          awslogs-stream-prefix = "${data.aws_default_tags.current.tags.environment-name}.otel.app"
+        }
+      },
+      environmentFiles      = [],
+      mountPoints           = [],
+      volumesFrom           = [],
+      secrets               = [],
+      dnsServers            = [],
+      dnsSearchDomains      = [],
+      extraHosts            = [],
+      dockerSecurityOptions = [],
+      dockerLabels          = {},
+      ulimits               = [],
+      systemControls        = []
+  })
+}
+
+# Additional permissions for the ECS task role to run experiments
+
+data "aws_iam_policy_document" "ecs_task_role_fis_related_task_permissions" {
+  policy_id = "${local.policy_region_prefix}_fis_ecs_task_actions"
+  statement {
+    sid       = "AllowSSMCommands"
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "ssm:CreateActivation",
+      "ssm:AddTagsToResource",
+    ]
+  }
+
+  statement {
+    sid       = "ManagedInstancePermissions"
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "ssm:DeleteActivation",
+      "ssm:DeregisterManagedInstance",
+    ]
+  }
+
+  statement {
+    sid       = "AllowPassRole"
+    effect    = "Allow"
+    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
+    actions = [
+      "iam:PassRole",
+    ]
+  }
 }
diff --git a/terraform/environment/region/modules/app/variables.tf b/terraform/environment/region/modules/app/variables.tf
index e4174d3998..c42094297c 100644
--- a/terraform/environment/region/modules/app/variables.tf
+++ b/terraform/environment/region/modules/app/variables.tf
@@ -2,6 +2,10 @@ locals {
   name_prefix = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}"
 }
 
+locals {
+  policy_region_prefix = lower(replace(data.aws_region.current.name, "-", ""))
+}
+
 variable "ecs_execution_role" {
   type = object({
     id  = string
@@ -15,11 +19,6 @@ variable "ecs_task_role" {
   description = "ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services."
 }
 
-variable "fault_injection_simulator_role" {
-  type        = any
-  description = "ARN of IAM role that allows AWS FIS to make calls to other AWS services."
-}
-
 variable "ecs_cluster" {
   type        = string
   description = "ARN of an ECS cluster."
@@ -133,3 +132,8 @@ variable "lpa_store_base_url" {
 variable "mock_onelogin_enabled" {
   type = bool
 }
+
+# variable "fault_injection_enabled" {
+#   type        = bool
+#   description = "Enable fault injection"
+# }
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md b/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
new file mode 100644
index 0000000000..1f78ed3660
--- /dev/null
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
@@ -0,0 +1,43 @@
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.5.2 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.35.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws.region"></a> [aws.region](#provider\_aws.region) | ~> 5.35.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_cloudwatch_log_group.fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
+| [aws_cloudwatch_log_resource_policy.fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_resource_policy) | resource |
+| [aws_fis_experiment_template.ecs_app](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/fis_experiment_template) | resource |
+| [aws_iam_role_policy.fis_role_log_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_default_tags.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source |
+| [aws_iam_policy_document.cloudwatch_log_group_policy_fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.fis_role_log_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_kms_alias.cloudwatch_application_logs_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
+| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_fault_injection_simulator_role"></a> [fault\_injection\_simulator\_role](#input\_fault\_injection\_simulator\_role) | ARN of IAM role that allows AWS FIS to make calls to other AWS services. | `any` | n/a | yes |
+
+## Outputs
+
+No outputs.
+<!-- END_TF_DOCS -->
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
new file mode 100644
index 0000000000..2efbdedd48
--- /dev/null
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
@@ -0,0 +1,11 @@
+data "aws_region" "current" {
+  provider = aws.region
+}
+
+data "aws_default_tags" "current" {
+  provider = aws.region
+}
+
+data "aws_caller_identity" "current" {
+  provider = aws.region
+}
diff --git a/terraform/environment/region/modules/app/fault_injection_simulator.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
similarity index 51%
rename from terraform/environment/region/modules/app/fault_injection_simulator.tf
rename to terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
index a3b2abd6a2..4088b37091 100644
--- a/terraform/environment/region/modules/app/fault_injection_simulator.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
@@ -8,8 +8,8 @@ data "aws_kms_alias" "cloudwatch_application_logs_encryption" {
 resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
   name              = "/aws/fis/app-ecs-tasks-experiment-${data.aws_default_tags.current.tags.environment-name}"
   retention_in_days = 7
-  # kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
-  provider = aws.region
+  kms_key_id        = data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
+  provider          = aws.region
 }
 
 # Add resource policy to allow FIS or the FIS role to write logs - not working
@@ -152,7 +152,6 @@ resource "aws_iam_role_policy" "fis_role_log_encryption" {
 # Create experiment template for ECS tasks
 
 resource "aws_fis_experiment_template" "ecs_app" {
-  count       = data.aws_default_tags.current.tags.environment-name == "production" ? 0 : 1
   provider    = aws.region
   description = "Run ECS task experiments for the app service"
   role_arn    = var.fault_injection_simulator_role.arn
@@ -197,83 +196,3 @@ resource "aws_fis_experiment_template" "ecs_app" {
     selection_mode = "ALL"
   }
 }
-
-# Create ECS task definition for ssm agent, used to run experiments
-
-locals {
-  amazon_ssm_agent = jsonencode(
-    {
-      name                   = "amazon-ssm-agent",
-      image                  = "public.ecr.aws/amazon-ssm-agent/amazon-ssm-agent:latest",
-      cpu                    = 0,
-      links                  = [],
-      portMappings           = [],
-      essential              = false,
-      entryPoint             = [],
-      readonlyRootFilesystem = false
-      command = [
-        "/bin/bash",
-        "-c",
-        "set -e; yum upgrade -y; yum install jq procps awscli -y; term_handler() { echo \"Deleting SSM activation $ACTIVATION_ID\"; if ! aws ssm delete-activation --activation-id $ACTIVATION_ID --region $ECS_TASK_REGION; then echo \"SSM activation $ACTIVATION_ID failed to be deleted\" 1>&2; fi; MANAGED_INSTANCE_ID=$(jq -e -r .ManagedInstanceID /var/lib/amazon/ssm/registration); echo \"Deregistering SSM Managed Instance $MANAGED_INSTANCE_ID\"; if ! aws ssm deregister-managed-instance --instance-id $MANAGED_INSTANCE_ID --region $ECS_TASK_REGION; then echo \"SSM Managed Instance $MANAGED_INSTANCE_ID failed to be deregistered\" 1>&2; fi; kill -SIGTERM $SSM_AGENT_PID; }; trap term_handler SIGTERM SIGINT; if [[ -z $MANAGED_INSTANCE_ROLE_NAME ]]; then echo \"Environment variable MANAGED_INSTANCE_ROLE_NAME not set, exiting\" 1>&2; exit 1; fi; if ! ps ax | grep amazon-ssm-agent | grep -v grep > /dev/null; then if [[ -n $ECS_CONTAINER_METADATA_URI_V4 ]] ; then echo \"Found ECS Container Metadata, running activation with metadata\"; TASK_METADATA=$(curl \"$${ECS_CONTAINER_METADATA_URI_V4}/task\"); ECS_TASK_AVAILABILITY_ZONE=$(echo $TASK_METADATA | jq -e -r '.AvailabilityZone'); ECS_TASK_ARN=$(echo $TASK_METADATA | jq -e -r '.TaskARN'); ECS_TASK_REGION=$(echo $ECS_TASK_AVAILABILITY_ZONE | sed 's/.$//'); ECS_TASK_AVAILABILITY_ZONE_REGEX='^(af|ap|ca|cn|eu|me|sa|us|us-gov)-(central|north|(north(east|west))|south|south(east|west)|east|west)-[0-9]{1}[a-z]{1}$'; if ! [[ $ECS_TASK_AVAILABILITY_ZONE =~ $ECS_TASK_AVAILABILITY_ZONE_REGEX ]]; then echo \"Error extracting Availability Zone from ECS Container Metadata, exiting\" 1>&2; exit 1; fi; ECS_TASK_ARN_REGEX='^arn:(aws|aws-cn|aws-us-gov):ecs:[a-z0-9-]+:[0-9]{12}:task/[a-zA-Z0-9_-]+/[a-zA-Z0-9]+$'; if ! [[ $ECS_TASK_ARN =~ $ECS_TASK_ARN_REGEX ]]; then echo \"Error extracting Task ARN from ECS Container Metadata, exiting\" 1>&2; exit 1; fi; CREATE_ACTIVATION_OUTPUT=$(aws ssm create-activation --iam-role $MANAGED_INSTANCE_ROLE_NAME --tags Key=ECS_TASK_AVAILABILITY_ZONE,Value=$ECS_TASK_AVAILABILITY_ZONE Key=ECS_TASK_ARN,Value=$ECS_TASK_ARN Key=FAULT_INJECTION_SIDECAR,Value=true --region $ECS_TASK_REGION); ACTIVATION_CODE=$(echo $CREATE_ACTIVATION_OUTPUT | jq -e -r .ActivationCode); ACTIVATION_ID=$(echo $CREATE_ACTIVATION_OUTPUT | jq -e -r .ActivationId); if ! amazon-ssm-agent -register -code $ACTIVATION_CODE -id $ACTIVATION_ID -region $ECS_TASK_REGION; then echo \"Failed to register with AWS Systems Manager (SSM), exiting\" 1>&2; exit 1; fi; amazon-ssm-agent & SSM_AGENT_PID=$!; wait $SSM_AGENT_PID; else echo \"ECS Container Metadata not found, exiting\" 1>&2; exit 1; fi; else echo \"SSM agent is already running, exiting\" 1>&2; exit 1; fi"
-      ],
-      environment = [
-        {
-          name  = "MANAGED_INSTANCE_ROLE_NAME",
-          value = "ssm-register-instance-${data.aws_default_tags.current.tags.environment-name}"
-        }
-      ],
-      logConfiguration = {
-        logDriver = "awslogs",
-        options = {
-          awslogs-group         = var.ecs_application_log_group_name,
-          awslogs-region        = data.aws_region.current.name,
-          awslogs-stream-prefix = "${data.aws_default_tags.current.tags.environment-name}.otel.app"
-        }
-      },
-      environmentFiles      = [],
-      mountPoints           = [],
-      volumesFrom           = [],
-      secrets               = [],
-      dnsServers            = [],
-      dnsSearchDomains      = [],
-      extraHosts            = [],
-      dockerSecurityOptions = [],
-      dockerLabels          = {},
-      ulimits               = [],
-      systemControls        = []
-  })
-}
-
-# Additional permissions for the ECS task role to run experiments
-
-data "aws_iam_policy_document" "ecs_task_role_fis_related_task_permissions" {
-  policy_id = "${local.policy_region_prefix}_fis_ecs_task_actions"
-  statement {
-    sid       = "AllowSSMCommands"
-    effect    = "Allow"
-    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
-    actions = [
-      "ssm:CreateActivation",
-      "ssm:AddTagsToResource",
-    ]
-  }
-
-  statement {
-    sid       = "ManagedInstancePermissions"
-    effect    = "Allow"
-    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
-    actions = [
-      "ssm:DeleteActivation",
-      "ssm:DeregisterManagedInstance",
-    ]
-  }
-
-  statement {
-    sid       = "AllowPassRole"
-    effect    = "Allow"
-    resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards
-    actions = [
-      "iam:PassRole",
-    ]
-  }
-}
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/variables.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/variables.tf
new file mode 100644
index 0000000000..cc791839d2
--- /dev/null
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/variables.tf
@@ -0,0 +1,5 @@
+variable "fault_injection_simulator_role" {
+  type        = any
+  description = "ARN of IAM role that allows AWS FIS to make calls to other AWS services."
+
+}
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/versions.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/versions.tf
new file mode 100644
index 0000000000..785d777364
--- /dev/null
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/versions.tf
@@ -0,0 +1,13 @@
+terraform {
+  required_version = ">= 1.5.2"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.35.0"
+      configuration_aliases = [
+        aws.region,
+      ]
+    }
+  }
+}
diff --git a/terraform/environment/region/variables.tf b/terraform/environment/region/variables.tf
index e719745134..6b7518e818 100644
--- a/terraform/environment/region/variables.tf
+++ b/terraform/environment/region/variables.tf
@@ -147,3 +147,8 @@ variable "cloudwatch_application_insights_enabled" {
   type        = bool
   description = "Enable CloudWatch Application Insights"
 }
+
+variable "fault_injection_enabled" {
+  type        = bool
+  description = "Enable fault injection"
+}
diff --git a/terraform/environment/regions.tf b/terraform/environment/regions.tf
index c2845bf33e..2da0375155 100644
--- a/terraform/environment/regions.tf
+++ b/terraform/environment/regions.tf
@@ -61,6 +61,7 @@ module "eu_west_1" {
   dependency_health_check_alarm_enabled   = local.environment.app.dependency_health_check_alarm_enabled
   service_health_check_alarm_enabled      = local.environment.app.service_health_check_alarm_enabled
   cloudwatch_application_insights_enabled = local.environment.app.cloudwatch_application_insights_enabled
+  fault_injection_enabled                 = local.environment.app.fault_injection_enabled
   providers = {
     aws.region            = aws.eu_west_1
     aws.global            = aws.global
@@ -118,6 +119,7 @@ module "eu_west_2" {
   dependency_health_check_alarm_enabled   = local.environment.app.dependency_health_check_alarm_enabled
   service_health_check_alarm_enabled      = local.environment.app.service_health_check_alarm_enabled
   cloudwatch_application_insights_enabled = local.environment.app.cloudwatch_application_insights_enabled
+  fault_injection_enabled                 = local.environment.app.fault_injection_enabled
   providers = {
     aws.region            = aws.eu_west_2
     aws.global            = aws.global

From 0fa04243e5364e2cc1828c9622633ab98db2401a Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 6 Feb 2024 10:40:24 +0000
Subject: [PATCH 40/51] make experiements conditional

---
 terraform/environment/region/ecs.tf                             | 2 --
 .../region/fault_injections_simulator_experiments.tf            | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/terraform/environment/region/ecs.tf b/terraform/environment/region/ecs.tf
index cff60f963f..90e957a9ae 100644
--- a/terraform/environment/region/ecs.tf
+++ b/terraform/environment/region/ecs.tf
@@ -34,7 +34,6 @@ module "app" {
   app_allowed_api_arns            = concat(var.uid_service.api_arns, var.lpa_store_service.api_arns)
   ingress_allow_list_cidr         = concat(var.ingress_allow_list_cidr, split(",", data.aws_ssm_parameter.additional_allowed_ingress_cidrs.value))
   alb_deletion_protection_enabled = var.alb_deletion_protection_enabled
-  fault_injection_simulator_role  = var.iam_roles.fault_injection_simulator
   lpas_table                      = var.lpas_table
   container_port                  = 8080
   public_access_enabled           = var.public_access_enabled
@@ -56,7 +55,6 @@ module "app" {
   uid_base_url                                         = var.uid_service.base_url
   lpa_store_base_url                                   = var.lpa_store_service.base_url
   mock_onelogin_enabled                                = data.aws_default_tags.current.tags.environment-name != "production" && var.mock_onelogin_enabled
-  fault_injection_enabled                              = var.fault_injection_enabled
   providers = {
     aws.region = aws.region
   }
diff --git a/terraform/environment/region/fault_injections_simulator_experiments.tf b/terraform/environment/region/fault_injections_simulator_experiments.tf
index 2b221aa817..499d4133c7 100644
--- a/terraform/environment/region/fault_injections_simulator_experiments.tf
+++ b/terraform/environment/region/fault_injections_simulator_experiments.tf
@@ -1,4 +1,5 @@
 module "fault_injection_simulator_experiments" {
+  count  = var.fault_injection_enabled ? 1 : 0
   source = "./modules/fault_injection_simulator_experiments"
   providers = {
     aws.region = aws.region

From b0bd15e87b35d6bee1fc98185d9e5e350bca0205 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 6 Feb 2024 10:51:20 +0000
Subject: [PATCH 41/51] reduce permissions

---
 terraform/environment/.envrc                  |   3 +-
 terraform/environment/global/README.md        |   1 -
 .../global/iam_fault_injection_simulator.tf   |   8 +-
 .../README.md                                 |   1 -
 .../data_sources.tf                           |   6 +-
 .../main.tf                                   | 111 +++++++++---------
 6 files changed, 62 insertions(+), 68 deletions(-)

diff --git a/terraform/environment/.envrc b/terraform/environment/.envrc
index b5f110a13c..96a4c30808 100644
--- a/terraform/environment/.envrc
+++ b/terraform/environment/.envrc
@@ -2,4 +2,5 @@ source ../../scripts/switch-terraform-version.sh
 export TF_CLI_ARGS_init="-backend-config=role_arn=arn:aws:iam::311462405659:role/operator -upgrade -reconfigure"
 export TF_VAR_default_role=operator
 export TF_VAR_pagerduty_api_key=$(aws-vault exec mlpa-dev -- aws secretsmanager get-secret-value --secret-id "pagerduty_api_key" | jq -r .'SecretString')
-export TF_VAR_container_version=$(aws-vault exec management-global -- aws ssm get-parameter --name "/modernising-lpa/container-version/production" --query 'Parameter.Value' --output text)
+# export TF_VAR_container_version=$(aws-vault exec management-global -- aws ssm get-parameter --name "/modernising-lpa/container-version/production" --query 'Parameter.Value' --output text)
+export TF_VAR_container_version="v0.992.0-MLPAB1570organiseafaultinjectionsimulationevent.0"
diff --git a/terraform/environment/global/README.md b/terraform/environment/global/README.md
index fd07a93374..58df19d56b 100644
--- a/terraform/environment/global/README.md
+++ b/terraform/environment/global/README.md
@@ -86,7 +86,6 @@ No modules.
 | [aws_iam_role_policy.execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
 | [aws_iam_role_policy.fault_injection_simulator_create_fis_service_linked_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
 | [aws_iam_role_policy.ssm_register_instance_permissions](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
-| [aws_iam_role_policy_attachment.cloudwatch_logs_full_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.fault_injection_simulator_ecs_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.fault_injection_simulator_ssm_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.s3_antivirus_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index ee37e7c17f..8513b4dd8e 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -18,7 +18,7 @@ data "aws_iam_policy_document" "fault_injection_simulator_assume" {
   provider = aws.global
 }
 
-# Add permissions for FIS to run experiments (ECS, Logging, SSM)
+# Add permissions for FIS to run experiments (ECS and SSM)
 
 resource "aws_iam_role_policy_attachment" "fault_injection_simulator_ecs_access" {
   role       = aws_iam_role.fault_injection_simulator.name
@@ -32,12 +32,6 @@ resource "aws_iam_role_policy_attachment" "fault_injection_simulator_ssm_access"
   provider   = aws.global
 }
 
-resource "aws_iam_role_policy_attachment" "cloudwatch_logs_full_access" {
-  role       = aws_iam_role.fault_injection_simulator.name
-  policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
-  provider   = aws.global
-}
-
 resource "aws_iam_role_policy" "fault_injection_simulator_create_fis_service_linked_role" {
   name     = "create-fis-service-linked-role-permissions"
   role     = aws_iam_role.fault_injection_simulator.name
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md b/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
index 1f78ed3660..f09e236beb 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
@@ -29,7 +29,6 @@ No modules.
 | [aws_iam_policy_document.cloudwatch_log_group_policy_fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.fis_role_log_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_kms_alias.cloudwatch_application_logs_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
-| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 
 ## Inputs
 
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
index 2efbdedd48..e522f10bfe 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
@@ -1,6 +1,6 @@
-data "aws_region" "current" {
-  provider = aws.region
-}
+# data "aws_region" "current" {
+#   provider = aws.region
+# }
 
 data "aws_default_tags" "current" {
   provider = aws.region
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
index 4088b37091..9a6378ef30 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
@@ -85,61 +85,62 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
     ]
   }
 
-  statement {
-    sid    = "AllowFISExperimentRoleCloudwatch"
-    effect = "Allow"
-    actions = [
-      "logs:Describe*",
-      "logs:CreateLogDelivery",
-      "logs:PutLogEvents",
-      "logs:CreateLogStream",
-      "logs:PutResourcePolicy"
-    ]
-
-    resources = [
-      "*"
-    ]
-  }
-
-  statement {
-    sid    = "AllowLogDeliveryActions"
-    effect = "Allow"
-    actions = [
-      "logs:PutDeliverySource",
-      "logs:GetDeliverySource",
-      "logs:DeleteDeliverySource",
-      "logs:DescribeDeliverySources",
-      "logs:PutDeliveryDestination",
-      "logs:GetDeliveryDestination",
-      "logs:DeleteDeliveryDestination",
-      "logs:DescribeDeliveryDestinations",
-      "logs:CreateDelivery",
-      "logs:GetDelivery",
-      "logs:DeleteDelivery",
-      "logs:DescribeDeliveries",
-      "logs:PutDeliveryDestinationPolicy",
-      "logs:GetDeliveryDestinationPolicy",
-      "logs:DeleteDeliveryDestinationPolicy"
-    ]
-    resources = [
-      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery-source:*",
-      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery:*",
-      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery-destination:*"
-    ]
-  }
-  statement {
-    sid    = "AllowUpdatesToResourcePolicyCWL"
-    effect = "Allow"
-    actions = [
-      "logs:PutResourcePolicy",
-      "logs:DescribeResourcePolicies",
-      "logs:DescribeLogGroups"
-    ]
-    resources = [
-      aws_cloudwatch_log_group.fis_app_ecs_tasks.arn,
-      "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*",
-    ]
-  }
+  # statement {
+  #   sid    = "AllowFISExperimentRoleCloudwatch"
+  #   effect = "Allow"
+  #   actions = [
+  #     "logs:Describe*",
+  #     "logs:CreateLogDelivery",
+  #     "logs:PutLogEvents",
+  #     "logs:CreateLogStream",
+  #     "logs:PutResourcePolicy"
+  #   ]
+
+  #   resources = [
+  #     "*"
+  #   ]
+  # }
+
+  # statement {
+  #   sid    = "AllowLogDeliveryActions"
+  #   effect = "Allow"
+  #   actions = [
+  #     "logs:PutDeliverySource",
+  #     "logs:GetDeliverySource",
+  #     "logs:DeleteDeliverySource",
+  #     "logs:DescribeDeliverySources",
+  #     "logs:PutDeliveryDestination",
+  #     "logs:GetDeliveryDestination",
+  #     "logs:DeleteDeliveryDestination",
+  #     "logs:DescribeDeliveryDestinations",
+  #     "logs:CreateDelivery",
+  #     "logs:GetDelivery",
+  #     "logs:DeleteDelivery",
+  #     "logs:DescribeDeliveries",
+  #     "logs:PutDeliveryDestinationPolicy",
+  #     "logs:GetDeliveryDestinationPolicy",
+  #     "logs:DeleteDeliveryDestinationPolicy"
+  #   ]
+  #   resources = [
+  #     "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery-source:*",
+  #     "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery:*",
+  #     "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery-destination:*"
+  #   ]
+  # }
+
+  # statement {
+  #   sid    = "AllowUpdatesToResourcePolicyCWL"
+  #   effect = "Allow"
+  #   actions = [
+  #     "logs:PutResourcePolicy",
+  #     "logs:DescribeResourcePolicies",
+  #     "logs:DescribeLogGroups"
+  #   ]
+  #   resources = [
+  #     aws_cloudwatch_log_group.fis_app_ecs_tasks.arn,
+  #     "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*",
+  #   ]
+  # }
 }
 
 resource "aws_iam_role_policy" "fis_role_log_encryption" {

From a5dddcfccd90e63d6baabb115c20ded53861de63 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 6 Feb 2024 10:59:12 +0000
Subject: [PATCH 42/51] enable experiments

---
 terraform/environment/terraform.tfvars.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/environment/terraform.tfvars.json b/terraform/environment/terraform.tfvars.json
index f383cb6580..7f3405077a 100644
--- a/terraform/environment/terraform.tfvars.json
+++ b/terraform/environment/terraform.tfvars.json
@@ -21,7 +21,7 @@
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,
         "cloudwatch_application_insights_enabled": false,
-        "fault_injection_enabled": false
+        "fault_injection_enabled": true
       },
       "mock_onelogin_enabled": false,
       "uid_service": {

From d62e4fec2a84ce54bf51f75e576920a22d40efcc Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Tue, 6 Feb 2024 11:00:53 +0000
Subject: [PATCH 43/51] rename variable

---
 terraform/environment/README.md                      |  2 +-
 terraform/environment/region/README.md               |  2 +-
 .../region/fault_injections_simulator_experiments.tf |  2 +-
 .../environment/region/modules/app/variables.tf      |  2 +-
 terraform/environment/region/variables.tf            |  2 +-
 terraform/environment/regions.tf                     |  4 ++--
 terraform/environment/terraform.tfvars.json          | 12 ++++++------
 terraform/environment/variables.tf                   |  2 +-
 8 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/terraform/environment/README.md b/terraform/environment/README.md
index 5f6ae95de7..df2fbb9ff7 100644
--- a/terraform/environment/README.md
+++ b/terraform/environment/README.md
@@ -167,7 +167,7 @@ For terraform_environment, this will be based on your PR and can be found in the
 |------|-------------|------|---------|:--------:|
 | <a name="input_container_version"></a> [container\_version](#input\_container\_version) | n/a | `string` | `"latest"` | no |
 | <a name="input_default_role"></a> [default\_role](#input\_default\_role) | n/a | `string` | `"modernising-lpa-ci"` | no |
-| <a name="input_environments"></a> [environments](#input\_environments) | n/a | <pre>map(<br>    object({<br>      account_id    = string<br>      account_name  = string<br>      is_production = bool<br>      regions       = list(string)<br>      app = object({<br>        env = object({<br>          app_public_url         = string<br>          auth_redirect_base_url = string<br>          notify_is_production   = string<br>          onelogin_url           = string<br>        })<br>        autoscaling = object({<br>          minimum = number<br>          maximum = number<br>        })<br>        dependency_health_check_alarm_enabled   = bool<br>        service_health_check_alarm_enabled      = bool<br>        cloudwatch_application_insights_enabled = bool<br>        fault_injection_enabled                 = bool<br>      })<br>      mock_onelogin_enabled = bool<br>      uid_service = object({<br>        base_url = string<br>        api_arns = list(string)<br>      })<br>      lpa_store_service = object({<br>        base_url = string<br>        api_arns = list(string)<br>      })<br>      backups = object({<br>        backup_plan_enabled = bool<br>        copy_action_enabled = bool<br>      })<br>      dynamodb = object({<br>        region_replica_enabled = bool<br>        stream_enabled         = bool<br>      })<br>      ecs = object({<br>        fargate_spot_capacity_provider_enabled = bool<br><br>      })<br>      cloudwatch_log_groups = object({<br>        application_log_retention_days = number<br>      })<br>      application_load_balancer = object({<br>        deletion_protection_enabled = bool<br>      })<br>      cloudwatch_application_insights_enabled = bool<br>      pagerduty_service_name                  = string<br>      event_bus = object({<br>        target_event_bus_arn = string<br>        receive_account_ids  = list(string)<br>      })<br>      reduced_fees = object({<br>        enabled                                   = bool<br>        s3_object_replication_enabled             = bool<br>        target_environment                        = string<br>        destination_account_id                    = string<br>        enable_s3_batch_job_replication_scheduler = bool<br>      })<br>      s3_antivirus_provisioned_concurrency = number<br>    })<br>  )</pre> | n/a | yes |
+| <a name="input_environments"></a> [environments](#input\_environments) | n/a | <pre>map(<br>    object({<br>      account_id    = string<br>      account_name  = string<br>      is_production = bool<br>      regions       = list(string)<br>      app = object({<br>        env = object({<br>          app_public_url         = string<br>          auth_redirect_base_url = string<br>          notify_is_production   = string<br>          onelogin_url           = string<br>        })<br>        autoscaling = object({<br>          minimum = number<br>          maximum = number<br>        })<br>        dependency_health_check_alarm_enabled   = bool<br>        service_health_check_alarm_enabled      = bool<br>        cloudwatch_application_insights_enabled = bool<br>        fault_injection_experiments_enabled     = bool<br>      })<br>      mock_onelogin_enabled = bool<br>      uid_service = object({<br>        base_url = string<br>        api_arns = list(string)<br>      })<br>      lpa_store_service = object({<br>        base_url = string<br>        api_arns = list(string)<br>      })<br>      backups = object({<br>        backup_plan_enabled = bool<br>        copy_action_enabled = bool<br>      })<br>      dynamodb = object({<br>        region_replica_enabled = bool<br>        stream_enabled         = bool<br>      })<br>      ecs = object({<br>        fargate_spot_capacity_provider_enabled = bool<br><br>      })<br>      cloudwatch_log_groups = object({<br>        application_log_retention_days = number<br>      })<br>      application_load_balancer = object({<br>        deletion_protection_enabled = bool<br>      })<br>      cloudwatch_application_insights_enabled = bool<br>      pagerduty_service_name                  = string<br>      event_bus = object({<br>        target_event_bus_arn = string<br>        receive_account_ids  = list(string)<br>      })<br>      reduced_fees = object({<br>        enabled                                   = bool<br>        s3_object_replication_enabled             = bool<br>        target_environment                        = string<br>        destination_account_id                    = string<br>        enable_s3_batch_job_replication_scheduler = bool<br>      })<br>      s3_antivirus_provisioned_concurrency = number<br>    })<br>  )</pre> | n/a | yes |
 | <a name="input_pagerduty_api_key"></a> [pagerduty\_api\_key](#input\_pagerduty\_api\_key) | n/a | `string` | n/a | yes |
 | <a name="input_public_access_enabled"></a> [public\_access\_enabled](#input\_public\_access\_enabled) | n/a | `bool` | `false` | no |
 
diff --git a/terraform/environment/region/README.md b/terraform/environment/region/README.md
index ae4ac85a2e..f09c9be840 100644
--- a/terraform/environment/region/README.md
+++ b/terraform/environment/region/README.md
@@ -110,7 +110,7 @@ This module creates the regional resources for an environment.
 | <a name="input_dns_weighting"></a> [dns\_weighting](#input\_dns\_weighting) | Weighting for DNS records | `number` | n/a | yes |
 | <a name="input_ecs_capacity_provider"></a> [ecs\_capacity\_provider](#input\_ecs\_capacity\_provider) | Name of the capacity provider to use. Valid values are FARGATE\_SPOT and FARGATE | `string` | n/a | yes |
 | <a name="input_ecs_task_autoscaling"></a> [ecs\_task\_autoscaling](#input\_ecs\_task\_autoscaling) | task minimum and maximum values for autoscaling | `any` | n/a | yes |
-| <a name="input_fault_injection_enabled"></a> [fault\_injection\_enabled](#input\_fault\_injection\_enabled) | Enable fault injection | `bool` | n/a | yes |
+| <a name="input_fault_injection_experiments_enabled"></a> [fault\_injection\_experiments\_enabled](#input\_fault\_injection\_experiments\_enabled) | Enable fault injection | `bool` | n/a | yes |
 | <a name="input_iam_roles"></a> [iam\_roles](#input\_iam\_roles) | ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services. | <pre>object({<br>    ecs_execution_role        = any<br>    app_ecs_task_role         = any<br>    s3_antivirus              = any<br>    cross_account_put         = any<br>    fault_injection_simulator = any<br>  })</pre> | n/a | yes |
 | <a name="input_ingress_allow_list_cidr"></a> [ingress\_allow\_list\_cidr](#input\_ingress\_allow\_list\_cidr) | List of CIDR ranges permitted to access the service | `list(string)` | n/a | yes |
 | <a name="input_lpa_store_service"></a> [lpa\_store\_service](#input\_lpa\_store\_service) | n/a | <pre>object({<br>    base_url = string<br>    api_arns = list(string)<br>  })</pre> | n/a | yes |
diff --git a/terraform/environment/region/fault_injections_simulator_experiments.tf b/terraform/environment/region/fault_injections_simulator_experiments.tf
index 499d4133c7..701b4354d2 100644
--- a/terraform/environment/region/fault_injections_simulator_experiments.tf
+++ b/terraform/environment/region/fault_injections_simulator_experiments.tf
@@ -1,5 +1,5 @@
 module "fault_injection_simulator_experiments" {
-  count  = var.fault_injection_enabled ? 1 : 0
+  count  = var.fault_injection_experiments_enabled ? 1 : 0
   source = "./modules/fault_injection_simulator_experiments"
   providers = {
     aws.region = aws.region
diff --git a/terraform/environment/region/modules/app/variables.tf b/terraform/environment/region/modules/app/variables.tf
index c42094297c..96a2380e03 100644
--- a/terraform/environment/region/modules/app/variables.tf
+++ b/terraform/environment/region/modules/app/variables.tf
@@ -133,7 +133,7 @@ variable "mock_onelogin_enabled" {
   type = bool
 }
 
-# variable "fault_injection_enabled" {
+# variable "fault_injection_experiments_enabled" {
 #   type        = bool
 #   description = "Enable fault injection"
 # }
diff --git a/terraform/environment/region/variables.tf b/terraform/environment/region/variables.tf
index 6b7518e818..435c6f15f8 100644
--- a/terraform/environment/region/variables.tf
+++ b/terraform/environment/region/variables.tf
@@ -148,7 +148,7 @@ variable "cloudwatch_application_insights_enabled" {
   description = "Enable CloudWatch Application Insights"
 }
 
-variable "fault_injection_enabled" {
+variable "fault_injection_experiments_enabled" {
   type        = bool
   description = "Enable fault injection"
 }
diff --git a/terraform/environment/regions.tf b/terraform/environment/regions.tf
index 2da0375155..2cd2508bed 100644
--- a/terraform/environment/regions.tf
+++ b/terraform/environment/regions.tf
@@ -61,7 +61,7 @@ module "eu_west_1" {
   dependency_health_check_alarm_enabled   = local.environment.app.dependency_health_check_alarm_enabled
   service_health_check_alarm_enabled      = local.environment.app.service_health_check_alarm_enabled
   cloudwatch_application_insights_enabled = local.environment.app.cloudwatch_application_insights_enabled
-  fault_injection_enabled                 = local.environment.app.fault_injection_enabled
+  fault_injection_experiments_enabled     = local.environment.app.fault_injection_experiments_enabled
   providers = {
     aws.region            = aws.eu_west_1
     aws.global            = aws.global
@@ -119,7 +119,7 @@ module "eu_west_2" {
   dependency_health_check_alarm_enabled   = local.environment.app.dependency_health_check_alarm_enabled
   service_health_check_alarm_enabled      = local.environment.app.service_health_check_alarm_enabled
   cloudwatch_application_insights_enabled = local.environment.app.cloudwatch_application_insights_enabled
-  fault_injection_enabled                 = local.environment.app.fault_injection_enabled
+  fault_injection_experiments_enabled     = local.environment.app.fault_injection_experiments_enabled
   providers = {
     aws.region            = aws.eu_west_2
     aws.global            = aws.global
diff --git a/terraform/environment/terraform.tfvars.json b/terraform/environment/terraform.tfvars.json
index 7f3405077a..5dddaa2b26 100644
--- a/terraform/environment/terraform.tfvars.json
+++ b/terraform/environment/terraform.tfvars.json
@@ -21,7 +21,7 @@
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,
         "cloudwatch_application_insights_enabled": false,
-        "fault_injection_enabled": true
+        "fault_injection_experiments_enabled": true
       },
       "mock_onelogin_enabled": false,
       "uid_service": {
@@ -97,7 +97,7 @@
         "dependency_health_check_alarm_enabled": true,
         "service_health_check_alarm_enabled": true,
         "cloudwatch_application_insights_enabled": true,
-        "fault_injection_enabled": false
+        "fault_injection_experiments_enabled": false
       },
       "mock_onelogin_enabled": true,
       "uid_service": {
@@ -173,7 +173,7 @@
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,
         "cloudwatch_application_insights_enabled": false,
-        "fault_injection_enabled": false
+        "fault_injection_experiments_enabled": false
       },
       "mock_onelogin_enabled": true,
       "uid_service": {
@@ -249,7 +249,7 @@
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,
         "cloudwatch_application_insights_enabled": true,
-        "fault_injection_enabled": false
+        "fault_injection_experiments_enabled": false
       },
       "mock_onelogin_enabled": true,
       "uid_service": {
@@ -325,7 +325,7 @@
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,
         "cloudwatch_application_insights_enabled": true,
-        "fault_injection_enabled": false
+        "fault_injection_experiments_enabled": false
       },
       "mock_onelogin_enabled": false,
       "uid_service": {
@@ -401,7 +401,7 @@
         "dependency_health_check_alarm_enabled": true,
         "service_health_check_alarm_enabled": true,
         "cloudwatch_application_insights_enabled": true,
-        "fault_injection_enabled": false
+        "fault_injection_experiments_enabled": false
       },
       "mock_onelogin_enabled": false,
       "uid_service": {
diff --git a/terraform/environment/variables.tf b/terraform/environment/variables.tf
index 098a5d742a..9e8207b844 100644
--- a/terraform/environment/variables.tf
+++ b/terraform/environment/variables.tf
@@ -46,7 +46,7 @@ variable "environments" {
         dependency_health_check_alarm_enabled   = bool
         service_health_check_alarm_enabled      = bool
         cloudwatch_application_insights_enabled = bool
-        fault_injection_enabled                 = bool
+        fault_injection_experiments_enabled     = bool
       })
       mock_onelogin_enabled = bool
       uid_service = object({

From 9b3167ffe92e9ec8d044312c75a0219f65501235 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 7 Feb 2024 12:16:09 +0000
Subject: [PATCH 44/51] add example actions

---
 terraform/environment/global/README.md        |   1 +
 .../global/iam_fault_injection_simulator.tf   |   8 +-
 .../main.tf                                   | 138 ++++++++++--------
 3 files changed, 86 insertions(+), 61 deletions(-)

diff --git a/terraform/environment/global/README.md b/terraform/environment/global/README.md
index 58df19d56b..fd07a93374 100644
--- a/terraform/environment/global/README.md
+++ b/terraform/environment/global/README.md
@@ -86,6 +86,7 @@ No modules.
 | [aws_iam_role_policy.execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
 | [aws_iam_role_policy.fault_injection_simulator_create_fis_service_linked_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
 | [aws_iam_role_policy.ssm_register_instance_permissions](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_iam_role_policy_attachment.cloudwatch_logs_full_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.fault_injection_simulator_ecs_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.fault_injection_simulator_ssm_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.s3_antivirus_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index 8513b4dd8e..ee37e7c17f 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -18,7 +18,7 @@ data "aws_iam_policy_document" "fault_injection_simulator_assume" {
   provider = aws.global
 }
 
-# Add permissions for FIS to run experiments (ECS and SSM)
+# Add permissions for FIS to run experiments (ECS, Logging, SSM)
 
 resource "aws_iam_role_policy_attachment" "fault_injection_simulator_ecs_access" {
   role       = aws_iam_role.fault_injection_simulator.name
@@ -32,6 +32,12 @@ resource "aws_iam_role_policy_attachment" "fault_injection_simulator_ssm_access"
   provider   = aws.global
 }
 
+resource "aws_iam_role_policy_attachment" "cloudwatch_logs_full_access" {
+  role       = aws_iam_role.fault_injection_simulator.name
+  policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
+  provider   = aws.global
+}
+
 resource "aws_iam_role_policy" "fault_injection_simulator_create_fis_service_linked_role" {
   name     = "create-fis-service-linked-role-permissions"
   role     = aws_iam_role.fault_injection_simulator.name
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
index 9a6378ef30..620c84a023 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
@@ -17,7 +17,7 @@ resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
 data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
   provider = aws.region
   statement {
-    sid    = "AWSLogDeliveryWrite20150319"
+    sid    = "AWSLogDeliveryWrite"
     effect = "Allow"
 
     principals {
@@ -36,6 +36,32 @@ data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
       "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*",
     ]
 
+    condition {
+      test     = "StringEquals"
+      variable = "aws:SourceAccount"
+      values   = [data.aws_caller_identity.current.account_id]
+    }
+  }
+  statement {
+    sid    = "AWSLogEncrypt"
+    effect = "Allow"
+
+    principals {
+      identifiers = [
+        "delivery.logs.amazonaws.com"
+      ]
+      type = "Service"
+    }
+
+    actions = [
+      "kms:Encrypt",
+      "kms:GenerateDataKey",
+    ]
+
+    resources = [
+      data.aws_kms_alias.cloudwatch_application_logs_encryption.target_key_arn
+    ]
+
     condition {
       test     = "StringEquals"
       variable = "aws:SourceAccount"
@@ -77,6 +103,7 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
       "logs:CreateLogStream",
       "logs:PutLogEvents",
       "logs:DescribeResourcePolicies",
+      "logs:PutResourcePolicy",
     ]
 
     resources = [
@@ -84,63 +111,6 @@ data "aws_iam_policy_document" "fis_role_log_encryption" {
       "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*",
     ]
   }
-
-  # statement {
-  #   sid    = "AllowFISExperimentRoleCloudwatch"
-  #   effect = "Allow"
-  #   actions = [
-  #     "logs:Describe*",
-  #     "logs:CreateLogDelivery",
-  #     "logs:PutLogEvents",
-  #     "logs:CreateLogStream",
-  #     "logs:PutResourcePolicy"
-  #   ]
-
-  #   resources = [
-  #     "*"
-  #   ]
-  # }
-
-  # statement {
-  #   sid    = "AllowLogDeliveryActions"
-  #   effect = "Allow"
-  #   actions = [
-  #     "logs:PutDeliverySource",
-  #     "logs:GetDeliverySource",
-  #     "logs:DeleteDeliverySource",
-  #     "logs:DescribeDeliverySources",
-  #     "logs:PutDeliveryDestination",
-  #     "logs:GetDeliveryDestination",
-  #     "logs:DeleteDeliveryDestination",
-  #     "logs:DescribeDeliveryDestinations",
-  #     "logs:CreateDelivery",
-  #     "logs:GetDelivery",
-  #     "logs:DeleteDelivery",
-  #     "logs:DescribeDeliveries",
-  #     "logs:PutDeliveryDestinationPolicy",
-  #     "logs:GetDeliveryDestinationPolicy",
-  #     "logs:DeleteDeliveryDestinationPolicy"
-  #   ]
-  #   resources = [
-  #     "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery-source:*",
-  #     "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery:*",
-  #     "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:delivery-destination:*"
-  #   ]
-  # }
-
-  # statement {
-  #   sid    = "AllowUpdatesToResourcePolicyCWL"
-  #   effect = "Allow"
-  #   actions = [
-  #     "logs:PutResourcePolicy",
-  #     "logs:DescribeResourcePolicies",
-  #     "logs:DescribeLogGroups"
-  #   ]
-  #   resources = [
-  #     aws_cloudwatch_log_group.fis_app_ecs_tasks.arn,
-  #     "${aws_cloudwatch_log_group.fis_app_ecs_tasks.arn}:*",
-  #   ]
-  # }
 }
 
 resource "aws_iam_role_policy" "fis_role_log_encryption" {
@@ -160,13 +130,13 @@ resource "aws_fis_experiment_template" "ecs_app" {
     Name = "${data.aws_default_tags.current.tags.environment-name} - APP ECS Task Experiments"
   }
 
-  action {
+  action { # defaults to 100% CPU
     action_id   = "aws:ecs:task-cpu-stress"
     description = null
     name        = "cpu_stress_100_percent"
     parameter {
       key   = "duration"
-      value = "PT5M"
+      value = "PT10M"
     }
     target {
       key   = "Tasks"
@@ -174,6 +144,50 @@ resource "aws_fis_experiment_template" "ecs_app" {
     }
   }
 
+  action {
+    action_id   = "aws:ecs:task-io-stress"
+    description = null
+    name        = "io_stress"
+    start_after = [
+      "cpu_stress_100_percent"
+    ]
+    parameter {
+      key   = "duration"
+      value = "PT10M"
+    }
+    target {
+      key   = "Tasks"
+      value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
+    }
+  }
+
+  # action {
+  #   action_id   = "aws:ecs:stop-task"
+  #   name        = "stop_task"
+  #   start_after = []
+
+  #   target {
+  #     key   = "Tasks"
+  #     value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
+  #   }
+  # }
+
+  # action { # not supported for FARGATE tasks
+  #   action_id   = "aws:ecs:task-network-latency"
+  #   name        = "network_latency"
+  #   start_after = []
+
+  #   parameter {
+  #     key   = "duration"
+  #     value = "PT5M"
+  #   }
+
+  #   target {
+  #     key   = "Tasks"
+  #     value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
+  #   }
+  # }
+
   stop_condition {
     source = "none"
     value  = null
@@ -193,6 +207,10 @@ resource "aws_fis_experiment_template" "ecs_app" {
       key   = "environment-name"
       value = data.aws_default_tags.current.tags.environment-name
     }
+    # parameters = {
+    #   "cluster" = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}"
+    #   "service" = "app"
+    # }
     resource_type  = "aws:ecs:task"
     selection_mode = "ALL"
   }

From ab8a995cfa6030dfce82a471e5162b30758d0a6e Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 7 Feb 2024 14:32:33 +0000
Subject: [PATCH 45/51] conditionally deploy ssm agent container

---
 terraform/environment/.envrc                             | 3 +--
 .../environment/global/iam_fault_injection_simulator.tf  | 1 +
 terraform/environment/region/ecs.tf                      | 1 +
 terraform/environment/region/modules/app/README.md       | 1 +
 terraform/environment/region/modules/app/ecs.tf          | 9 +++------
 terraform/environment/region/modules/app/variables.tf    | 8 ++++----
 .../fault_injection_simulator_experiments/README.md      | 1 +
 .../data_sources.tf                                      | 6 +++---
 .../fault_injection_simulator_experiments/main.tf        | 8 ++++----
 9 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/terraform/environment/.envrc b/terraform/environment/.envrc
index 96a4c30808..b5f110a13c 100644
--- a/terraform/environment/.envrc
+++ b/terraform/environment/.envrc
@@ -2,5 +2,4 @@ source ../../scripts/switch-terraform-version.sh
 export TF_CLI_ARGS_init="-backend-config=role_arn=arn:aws:iam::311462405659:role/operator -upgrade -reconfigure"
 export TF_VAR_default_role=operator
 export TF_VAR_pagerduty_api_key=$(aws-vault exec mlpa-dev -- aws secretsmanager get-secret-value --secret-id "pagerduty_api_key" | jq -r .'SecretString')
-# export TF_VAR_container_version=$(aws-vault exec management-global -- aws ssm get-parameter --name "/modernising-lpa/container-version/production" --query 'Parameter.Value' --output text)
-export TF_VAR_container_version="v0.992.0-MLPAB1570organiseafaultinjectionsimulationevent.0"
+export TF_VAR_container_version=$(aws-vault exec management-global -- aws ssm get-parameter --name "/modernising-lpa/container-version/production" --query 'Parameter.Value' --output text)
diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index ee37e7c17f..d2797cd5b6 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -108,6 +108,7 @@ data "aws_iam_policy_document" "ssm_register_instance_permissions" {
     actions = [
       "ssm:CreateActivation",
       "ssm:AddTagsToResource",
+      "iam:PassRole",
     ]
   }
 
diff --git a/terraform/environment/region/ecs.tf b/terraform/environment/region/ecs.tf
index 90e957a9ae..7a4211c0de 100644
--- a/terraform/environment/region/ecs.tf
+++ b/terraform/environment/region/ecs.tf
@@ -55,6 +55,7 @@ module "app" {
   uid_base_url                                         = var.uid_service.base_url
   lpa_store_base_url                                   = var.lpa_store_service.base_url
   mock_onelogin_enabled                                = data.aws_default_tags.current.tags.environment-name != "production" && var.mock_onelogin_enabled
+  fault_injection_experiments_enabled                  = var.fault_injection_experiments_enabled
   providers = {
     aws.region = aws.region
   }
diff --git a/terraform/environment/region/modules/app/README.md b/terraform/environment/region/modules/app/README.md
index e6142ba379..c10abb35c6 100644
--- a/terraform/environment/region/modules/app/README.md
+++ b/terraform/environment/region/modules/app/README.md
@@ -85,6 +85,7 @@ No modules.
 | <a name="input_ecs_service_desired_count"></a> [ecs\_service\_desired\_count](#input\_ecs\_service\_desired\_count) | Number of instances of the task definition to place and keep running. Defaults to 0. Do not specify if using the DAEMON scheduling strategy. | `number` | `0` | no |
 | <a name="input_ecs_task_role"></a> [ecs\_task\_role](#input\_ecs\_task\_role) | ARN of IAM role that allows your Amazon ECS container task to make calls to other AWS services. | `any` | n/a | yes |
 | <a name="input_event_bus"></a> [event\_bus](#input\_event\_bus) | Name and ARN of the event bus to send events to | <pre>object({<br>    name = string<br>    arn  = string<br>  })</pre> | n/a | yes |
+| <a name="input_fault_injection_experiments_enabled"></a> [fault\_injection\_experiments\_enabled](#input\_fault\_injection\_experiments\_enabled) | Enable fault injection | `bool` | n/a | yes |
 | <a name="input_ingress_allow_list_cidr"></a> [ingress\_allow\_list\_cidr](#input\_ingress\_allow\_list\_cidr) | List of CIDR ranges permitted to access the service | `list(string)` | n/a | yes |
 | <a name="input_lpa_store_base_url"></a> [lpa\_store\_base\_url](#input\_lpa\_store\_base\_url) | n/a | `string` | n/a | yes |
 | <a name="input_lpas_table"></a> [lpas\_table](#input\_lpas\_table) | DynamoDB table for storing LPAs | `any` | n/a | yes |
diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index 274ff64eaa..c12ff914b8 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -84,18 +84,15 @@ resource "aws_ecs_task_definition" "app" {
     operating_system_family = "LINUX"
     cpu_architecture        = "X86_64"
   }
-  #TODO: conditionally add ssm agent container
-  container_definitions = "[${local.app}, ${local.aws_otel_collector}, ${local.amazon_ssm_agent}]"
+  container_definitions = "[${local.app}, ${local.aws_otel_collector}, ${var.fault_injection_experiments_enabled ? local.amazon_ssm_agent : null}]"
   task_role_arn         = var.ecs_task_role.arn
   execution_role_arn    = var.ecs_execution_role.arn
   provider              = aws.region
 }
 
 resource "aws_iam_role_policy" "app_task_role" {
-  name = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}-app-task-role"
-  #TODO: make fis policy conditional using the combined policy document
-  # policy   = data.aws_iam_policy_document.task_role_access_policy.json
-  policy   = data.aws_iam_policy_document.combined.json
+  name     = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}-app-task-role"
+  policy   = var.fault_injection_experiments_enabled ? data.aws_iam_policy_document.combined.json : data.aws_iam_policy_document.task_role_access_policy.json
   role     = var.ecs_task_role.name
   provider = aws.region
 }
diff --git a/terraform/environment/region/modules/app/variables.tf b/terraform/environment/region/modules/app/variables.tf
index 96a2380e03..0c7562549d 100644
--- a/terraform/environment/region/modules/app/variables.tf
+++ b/terraform/environment/region/modules/app/variables.tf
@@ -133,7 +133,7 @@ variable "mock_onelogin_enabled" {
   type = bool
 }
 
-# variable "fault_injection_experiments_enabled" {
-#   type        = bool
-#   description = "Enable fault injection"
-# }
+variable "fault_injection_experiments_enabled" {
+  type        = bool
+  description = "Enable fault injection"
+}
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md b/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
index f09e236beb..1f78ed3660 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
@@ -29,6 +29,7 @@ No modules.
 | [aws_iam_policy_document.cloudwatch_log_group_policy_fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.fis_role_log_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_kms_alias.cloudwatch_application_logs_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
+| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 
 ## Inputs
 
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
index e522f10bfe..2efbdedd48 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
@@ -1,6 +1,6 @@
-# data "aws_region" "current" {
-#   provider = aws.region
-# }
+data "aws_region" "current" {
+  provider = aws.region
+}
 
 data "aws_default_tags" "current" {
   provider = aws.region
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
index 620c84a023..506babeb4b 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
@@ -207,10 +207,10 @@ resource "aws_fis_experiment_template" "ecs_app" {
       key   = "environment-name"
       value = data.aws_default_tags.current.tags.environment-name
     }
-    # parameters = {
-    #   "cluster" = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}"
-    #   "service" = "app"
-    # }
+    parameters = {
+      "cluster" = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}"
+      "service" = "app"
+    }
     resource_type  = "aws:ecs:task"
     selection_mode = "ALL"
   }

From cd03f574bd92fce73235ae4b9cb495f879a0af46 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 7 Feb 2024 14:58:15 +0000
Subject: [PATCH 46/51] ssm role doesn't need passrole

---
 terraform/environment/global/iam_fault_injection_simulator.tf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/terraform/environment/global/iam_fault_injection_simulator.tf b/terraform/environment/global/iam_fault_injection_simulator.tf
index d2797cd5b6..ee37e7c17f 100644
--- a/terraform/environment/global/iam_fault_injection_simulator.tf
+++ b/terraform/environment/global/iam_fault_injection_simulator.tf
@@ -108,7 +108,6 @@ data "aws_iam_policy_document" "ssm_register_instance_permissions" {
     actions = [
       "ssm:CreateActivation",
       "ssm:AddTagsToResource",
-      "iam:PassRole",
     ]
   }
 

From 3811a9e1e011273a44ac6c31a0ff37a8f291f816 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 7 Feb 2024 15:39:07 +0000
Subject: [PATCH 47/51] fix conditional container definitions

---
 terraform/environment/region/modules/app/ecs.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index c12ff914b8..8a45022fc7 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -84,7 +84,7 @@ resource "aws_ecs_task_definition" "app" {
     operating_system_family = "LINUX"
     cpu_architecture        = "X86_64"
   }
-  container_definitions = "[${local.app}, ${local.aws_otel_collector}, ${var.fault_injection_experiments_enabled ? local.amazon_ssm_agent : null}]"
+  container_definitions = var.fault_injection_experiments_enabled ? "[${local.app}, ${local.aws_otel_collector}, ${local.amazon_ssm_agent}]" : "[${local.app}, ${local.aws_otel_collector}]"
   task_role_arn         = var.ecs_task_role.arn
   execution_role_arn    = var.ecs_execution_role.arn
   provider              = aws.region

From f3778f0005111382d817986ffa89f4d4bb132043 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 7 Feb 2024 16:41:47 +0000
Subject: [PATCH 48/51] set desired count to autoscaling minimum target cluster
 and app service

---
 terraform/environment/.envrc                  |  3 +-
 terraform/environment/region/ecs.tf           |  2 +-
 .../fault_injections_simulator_experiments.tf |  1 +
 .../README.md                                 |  2 +-
 .../data_sources.tf                           |  4 --
 .../main.tf                                   | 65 +++++++++----------
 .../variables.tf                              |  5 ++
 terraform/environment/terraform.tfvars.json   |  4 +-
 8 files changed, 44 insertions(+), 42 deletions(-)

diff --git a/terraform/environment/.envrc b/terraform/environment/.envrc
index b5f110a13c..ac375475aa 100644
--- a/terraform/environment/.envrc
+++ b/terraform/environment/.envrc
@@ -2,4 +2,5 @@ source ../../scripts/switch-terraform-version.sh
 export TF_CLI_ARGS_init="-backend-config=role_arn=arn:aws:iam::311462405659:role/operator -upgrade -reconfigure"
 export TF_VAR_default_role=operator
 export TF_VAR_pagerduty_api_key=$(aws-vault exec mlpa-dev -- aws secretsmanager get-secret-value --secret-id "pagerduty_api_key" | jq -r .'SecretString')
-export TF_VAR_container_version=$(aws-vault exec management-global -- aws ssm get-parameter --name "/modernising-lpa/container-version/production" --query 'Parameter.Value' --output text)
+# export TF_VAR_container_version=$(aws-vault exec management-global -- aws ssm get-parameter --name "/modernising-lpa/container-version/production" --query 'Parameter.Value' --output text)
+export TF_VAR_container_version="v0.996.0-MLPAB1570organiseafaultinjectionsimulationevent.0"
diff --git a/terraform/environment/region/ecs.tf b/terraform/environment/region/ecs.tf
index 7a4211c0de..c4dd2cbe07 100644
--- a/terraform/environment/region/ecs.tf
+++ b/terraform/environment/region/ecs.tf
@@ -25,7 +25,7 @@ module "app" {
   ecs_cluster                     = aws_ecs_cluster.main.id
   ecs_execution_role              = var.iam_roles.ecs_execution_role
   ecs_task_role                   = var.iam_roles.app_ecs_task_role
-  ecs_service_desired_count       = 1
+  ecs_service_desired_count       = var.ecs_task_autoscaling.minimum
   ecs_application_log_group_name  = module.application_logs.cloudwatch_log_group.name
   ecs_capacity_provider           = var.ecs_capacity_provider
   app_env_vars                    = var.app_env_vars
diff --git a/terraform/environment/region/fault_injections_simulator_experiments.tf b/terraform/environment/region/fault_injections_simulator_experiments.tf
index 701b4354d2..0e5febb364 100644
--- a/terraform/environment/region/fault_injections_simulator_experiments.tf
+++ b/terraform/environment/region/fault_injections_simulator_experiments.tf
@@ -5,5 +5,6 @@ module "fault_injection_simulator_experiments" {
     aws.region = aws.region
   }
   fault_injection_simulator_role = var.iam_roles.fault_injection_simulator
+  ecs_cluster                    = aws_ecs_cluster.main.id
 
 }
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md b/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
index 1f78ed3660..3bfb0adeaf 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/README.md
@@ -29,12 +29,12 @@ No modules.
 | [aws_iam_policy_document.cloudwatch_log_group_policy_fis_app_ecs_tasks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.fis_role_log_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_kms_alias.cloudwatch_application_logs_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_alias) | data source |
-| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_ecs_cluster"></a> [ecs\_cluster](#input\_ecs\_cluster) | Name of the ECS cluster to run the experiments on. | `string` | n/a | yes |
 | <a name="input_fault_injection_simulator_role"></a> [fault\_injection\_simulator\_role](#input\_fault\_injection\_simulator\_role) | ARN of IAM role that allows AWS FIS to make calls to other AWS services. | `any` | n/a | yes |
 
 ## Outputs
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
index 2efbdedd48..ad61091f3f 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/data_sources.tf
@@ -1,7 +1,3 @@
-data "aws_region" "current" {
-  provider = aws.region
-}
-
 data "aws_default_tags" "current" {
   provider = aws.region
 }
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
index 506babeb4b..68afd5723e 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
@@ -130,24 +130,36 @@ resource "aws_fis_experiment_template" "ecs_app" {
     Name = "${data.aws_default_tags.current.tags.environment-name} - APP ECS Task Experiments"
   }
 
+  action {
+    action_id   = "aws:ecs:stop-task"
+    name        = "stop_tasks"
+    start_after = []
+
+    target {
+      key   = "Tasks"
+      value = "two-tasks"
+    }
+  }
+
   action { # defaults to 100% CPU
     action_id   = "aws:ecs:task-cpu-stress"
     description = null
-    name        = "cpu_stress_100_percent"
+    name        = "cpu_stress_100_percent_10_mins"
+    start_after = ["stop_tasks"]
     parameter {
       key   = "duration"
       value = "PT10M"
     }
     target {
       key   = "Tasks"
-      value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
+      value = "all-tasks"
     }
   }
 
   action {
     action_id   = "aws:ecs:task-io-stress"
     description = null
-    name        = "io_stress"
+    name        = "io_stress_10_mins"
     start_after = [
       "cpu_stress_100_percent"
     ]
@@ -157,37 +169,10 @@ resource "aws_fis_experiment_template" "ecs_app" {
     }
     target {
       key   = "Tasks"
-      value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
+      value = "all-tasks"
     }
   }
 
-  # action {
-  #   action_id   = "aws:ecs:stop-task"
-  #   name        = "stop_task"
-  #   start_after = []
-
-  #   target {
-  #     key   = "Tasks"
-  #     value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
-  #   }
-  # }
-
-  # action { # not supported for FARGATE tasks
-  #   action_id   = "aws:ecs:task-network-latency"
-  #   name        = "network_latency"
-  #   start_after = []
-
-  #   parameter {
-  #     key   = "duration"
-  #     value = "PT5M"
-  #   }
-
-  #   target {
-  #     key   = "Tasks"
-  #     value = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
-  #   }
-  # }
-
   stop_condition {
     source = "none"
     value  = null
@@ -202,16 +187,30 @@ resource "aws_fis_experiment_template" "ecs_app" {
   }
 
   target {
-    name = "app-ecs-tasks-${data.aws_default_tags.current.tags.environment-name}"
+    name = "all-tasks"
     resource_tag {
       key   = "environment-name"
       value = data.aws_default_tags.current.tags.environment-name
     }
     parameters = {
-      "cluster" = "${data.aws_default_tags.current.tags.environment-name}-${data.aws_region.current.name}"
+      "cluster" = var.ecs_cluster
       "service" = "app"
     }
     resource_type  = "aws:ecs:task"
     selection_mode = "ALL"
   }
+
+  target {
+    name = "two-tasks"
+    resource_tag {
+      key   = "environment-name"
+      value = data.aws_default_tags.current.tags.environment-name
+    }
+    parameters = {
+      "cluster" = var.ecs_cluster
+      "service" = "app"
+    }
+    resource_type  = "aws:ecs:task"
+    selection_mode = "COUNT(2)"
+  }
 }
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/variables.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/variables.tf
index cc791839d2..d997fb70aa 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/variables.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/variables.tf
@@ -1,5 +1,10 @@
 variable "fault_injection_simulator_role" {
   type        = any
   description = "ARN of IAM role that allows AWS FIS to make calls to other AWS services."
+}
+
+variable "ecs_cluster" {
+  type        = string
+  description = "Name of the ECS cluster to run the experiments on."
 
 }
diff --git a/terraform/environment/terraform.tfvars.json b/terraform/environment/terraform.tfvars.json
index 5dddaa2b26..b1a5009429 100644
--- a/terraform/environment/terraform.tfvars.json
+++ b/terraform/environment/terraform.tfvars.json
@@ -15,8 +15,8 @@
           "onelogin_url": "https://home.integration.account.gov.uk"
         },
         "autoscaling": {
-          "minimum": 1,
-          "maximum": 3
+          "minimum": 4,
+          "maximum": 6
         },
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,

From a4b57b7951be72c4cea38dd0f367d44910b0b745 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 7 Feb 2024 17:32:57 +0000
Subject: [PATCH 49/51] fix name of action

---
 .../modules/fault_injection_simulator_experiments/main.tf       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
index 68afd5723e..5839d958f7 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
@@ -161,7 +161,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
     description = null
     name        = "io_stress_10_mins"
     start_after = [
-      "cpu_stress_100_percent"
+      "cpu_stress_100_percent_10_mins"
     ]
     parameter {
       key   = "duration"

From 0f5b84dfb09d5d1111a68b1fc8a90bcb7c42ff8f Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 7 Feb 2024 17:43:39 +0000
Subject: [PATCH 50/51] turn off experiments and reset autoscaling

---
 terraform/environment/.envrc                | 3 +--
 terraform/environment/terraform.tfvars.json | 6 +++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/terraform/environment/.envrc b/terraform/environment/.envrc
index ac375475aa..b5f110a13c 100644
--- a/terraform/environment/.envrc
+++ b/terraform/environment/.envrc
@@ -2,5 +2,4 @@ source ../../scripts/switch-terraform-version.sh
 export TF_CLI_ARGS_init="-backend-config=role_arn=arn:aws:iam::311462405659:role/operator -upgrade -reconfigure"
 export TF_VAR_default_role=operator
 export TF_VAR_pagerduty_api_key=$(aws-vault exec mlpa-dev -- aws secretsmanager get-secret-value --secret-id "pagerduty_api_key" | jq -r .'SecretString')
-# export TF_VAR_container_version=$(aws-vault exec management-global -- aws ssm get-parameter --name "/modernising-lpa/container-version/production" --query 'Parameter.Value' --output text)
-export TF_VAR_container_version="v0.996.0-MLPAB1570organiseafaultinjectionsimulationevent.0"
+export TF_VAR_container_version=$(aws-vault exec management-global -- aws ssm get-parameter --name "/modernising-lpa/container-version/production" --query 'Parameter.Value' --output text)
diff --git a/terraform/environment/terraform.tfvars.json b/terraform/environment/terraform.tfvars.json
index b1a5009429..886e8b5f9c 100644
--- a/terraform/environment/terraform.tfvars.json
+++ b/terraform/environment/terraform.tfvars.json
@@ -15,13 +15,13 @@
           "onelogin_url": "https://home.integration.account.gov.uk"
         },
         "autoscaling": {
-          "minimum": 4,
-          "maximum": 6
+          "minimum": 1,
+          "maximum": 3
         },
         "dependency_health_check_alarm_enabled": false,
         "service_health_check_alarm_enabled": false,
         "cloudwatch_application_insights_enabled": false,
-        "fault_injection_experiments_enabled": true
+        "fault_injection_experiments_enabled": false
       },
       "mock_onelogin_enabled": false,
       "uid_service": {

From 05e0ad52c9b7881edb581e0abc69bf7f4e2864b1 Mon Sep 17 00:00:00 2001
From: Andrew Pearce <andrew.pearce@digital.justice.gov.uk>
Date: Wed, 7 Feb 2024 19:43:33 +0000
Subject: [PATCH 51/51] cleanup comments

---
 terraform/environment/region/modules/app/ecs.tf            | 1 -
 .../modules/fault_injection_simulator_experiments/main.tf  | 7 +++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/terraform/environment/region/modules/app/ecs.tf b/terraform/environment/region/modules/app/ecs.tf
index 8a45022fc7..510fc6fa71 100644
--- a/terraform/environment/region/modules/app/ecs.tf
+++ b/terraform/environment/region/modules/app/ecs.tf
@@ -32,7 +32,6 @@ resource "aws_ecs_service" "app" {
     create = "7m"
     update = "4m"
   }
-
   provider = aws.region
 }
 
diff --git a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
index 5839d958f7..8a96f37dad 100644
--- a/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
+++ b/terraform/environment/region/modules/fault_injection_simulator_experiments/main.tf
@@ -12,7 +12,7 @@ resource "aws_cloudwatch_log_group" "fis_app_ecs_tasks" {
   provider          = aws.region
 }
 
-# Add resource policy to allow FIS or the FIS role to write logs - not working
+# Add resource policy to allow FIS or the FIS role to write logs
 
 data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
   provider = aws.region
@@ -70,7 +70,6 @@ data "aws_iam_policy_document" "cloudwatch_log_group_policy_fis_app_ecs_tasks" {
   }
 }
 
-#  Add resource policy to allow FIS or the FIS role to write logs - not working
 resource "aws_cloudwatch_log_resource_policy" "fis_app_ecs_tasks" {
   provider        = aws.region
   policy_document = data.aws_iam_policy_document.cloudwatch_log_group_policy_fis_app_ecs_tasks.json
@@ -132,7 +131,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
 
   action {
     action_id   = "aws:ecs:stop-task"
-    name        = "stop_tasks"
+    name        = "stop_two_tasks"
     start_after = []
 
     target {
@@ -141,7 +140,7 @@ resource "aws_fis_experiment_template" "ecs_app" {
     }
   }
 
-  action { # defaults to 100% CPU
+  action {
     action_id   = "aws:ecs:task-cpu-stress"
     description = null
     name        = "cpu_stress_100_percent_10_mins"