Merge pull request #4158 from ministryofjustice/monitoring-oasys

monitoring oasys
ministryofjustice · Nov 27, 2023 · 689162f · 689162f
2 parents fbd765b + 534f4f1
commit 689162f
Show file tree

Hide file tree

Showing 6 changed files with 149 additions and 5 deletions.
diff --git a/terraform/environments/oasys/locals.tf b/terraform/environments/oasys/locals.tf
@@ -27,9 +27,14 @@ locals {
   region            = "eu-west-2"
   availability_zone = "eu-west-2a"
 
-  ###
+  ######
   ### env independent webserver vars
+  ######
+
   ###
+  #  web
+  ###
+
   webserver_a = {
     config = merge(module.baseline_presets.ec2_instance.config.default, {
       ami_name                  = "oasys_webserver_release_2023-07-02*"
@@ -41,7 +46,11 @@ locals {
       monitoring             = true
       vpc_security_group_ids = ["private_web"]
     })
-    cloudwatch_metric_alarms = {}
+    cloudwatch_metric_alarms = merge(
+      module.baseline_presets.cloudwatch_metric_alarms.ec2,
+      module.baseline_presets.cloudwatch_metric_alarms.ec2_cwagent_linux,
+      module.baseline_presets.cloudwatch_metric_alarms.ec2_instance_cwagent_collectd_service_status,
+    )
     user_data_cloud_init     = module.baseline_presets.ec2_instance.user_data_cloud_init.ssm_agent_ansible_no_tags
     autoscaling_schedules = {
       "scale_up" = {
@@ -98,6 +107,10 @@ locals {
     }
   }
 
+  ###
+  #  db
+  ###
+
   database_a = {
     config = merge(module.baseline_presets.ec2_instance.config.db, {
       ami_name          = "oasys_oracle_db_release_2023-06-26T10-16-03.670Z"
@@ -113,6 +126,24 @@ locals {
         backup-plan = "daily-and-weekly"
       }
     })
+    cloudwatch_metric_alarms = merge(
+      module.baseline_presets.cloudwatch_metric_alarms.ec2,
+      module.baseline_presets.cloudwatch_metric_alarms.ec2_cwagent_linux,
+      module.baseline_presets.cloudwatch_metric_alarms.ec2_instance_cwagent_collectd_service_status,
+      {
+        cpu-utilization-high = {
+          comparison_operator = "GreaterThanOrEqualToThreshold"
+          evaluation_periods  = "120"
+          datapoints_to_alarm = "120"
+          metric_name         = "CPUUtilization"
+          namespace           = "AWS/EC2"
+          period              = "60"
+          statistic           = "Maximum"
+          threshold           = "95"
+          alarm_description   = "Triggers if the average cpu remains at 95% utilization or above for 2 hours on an oasys-db instance"
+          alarm_actions       = ["dso_pagerduty"]
+        }
+    })
     autoscaling_schedules = {}
     autoscaling_group     = module.baseline_presets.ec2_autoscaling_group.default
     user_data_cloud_init  = module.baseline_presets.ec2_instance.user_data_cloud_init.ssm_agent_ansible_no_tags
@@ -208,6 +239,9 @@ locals {
     })
   })
 
+  ###
+  #  bip
+  ###
 
   bip_a = {
     config = merge(module.baseline_presets.ec2_instance.config.default, {
@@ -223,7 +257,11 @@ locals {
         backup-plan = "daily-and-weekly"
       }
     })
-    cloudwatch_metric_alarms = {}
+    cloudwatch_metric_alarms = merge(
+      module.baseline_presets.cloudwatch_metric_alarms.ec2,
+      module.baseline_presets.cloudwatch_metric_alarms.ec2_cwagent_linux,
+      module.baseline_presets.cloudwatch_metric_alarms.ec2_instance_cwagent_collectd_service_status,
+    )
     user_data_cloud_init     = module.baseline_presets.ec2_instance.user_data_cloud_init.ssm_agent_ansible_no_tags
     autoscaling_schedules    = module.baseline_presets.ec2_autoscaling_schedules.working_hours
     autoscaling_group = merge(module.baseline_presets.ec2_autoscaling_group.default, {
@@ -252,7 +290,15 @@ locals {
     })
   })
 
+  ###
+  #  other
+  ###
+
   baseline_secretsmanager_secrets = {}
 
+  baseline_cloudwatch_log_groups = {}
+  baseline_cloudwatch_metric_alarms = {}
+  baseline_cloudwatch_log_metric_filters = {}
+
   public_key_data = jsondecode(file("./files/bastion_linux.json"))
-}
+}
diff --git a/terraform/environments/oasys/locals_development.tf b/terraform/environments/oasys/locals_development.tf
@@ -163,5 +163,26 @@ locals {
       #   ]
       # }
     }
+
+    baseline_cloudwatch_log_groups = {
+      session-manager-logs = {
+        retention_in_days = 1
+      }
+      cwagent-var-log-messages = {
+        retention_in_days = 1
+      }
+      cwagent-var-log-secure = {
+        retention_in_days = 1
+      }
+      cwagent-windows-system = {
+        retention_in_days = 1
+      }
+      cwagent-oasys-autologoff = {
+        retention_in_days = 1
+      }
+      cwagent-web-logs = {
+        retention_in_days = 1
+      }
+    }
   }
 }
diff --git a/terraform/environments/oasys/locals_preproduction.tf b/terraform/environments/oasys/locals_preproduction.tf
@@ -125,6 +125,27 @@ locals {
       # }
     }
 
+    baseline_cloudwatch_log_groups = {
+      session-manager-logs = {
+        retention_in_days = 14
+      }
+      cwagent-var-log-messages = {
+        retention_in_days = 14
+      }
+      cwagent-var-log-secure = {
+        retention_in_days = 14
+      }
+      cwagent-windows-system = {
+        retention_in_days = 14
+      }
+      cwagent-oasys-autologoff = {
+        retention_in_days = 14
+      }
+      cwagent-web-logs = {
+        retention_in_days = 14
+      }
+    }
+
   }
 }
 
diff --git a/terraform/environments/oasys/locals_production.tf b/terraform/environments/oasys/locals_production.tf
@@ -74,5 +74,26 @@ locals {
       #   ]
       # }
     }
+
+    baseline_cloudwatch_log_groups = {
+      session-manager-logs = {
+        retention_in_days = 400
+      }
+      cwagent-var-log-messages = {
+        retention_in_days = 90
+      }
+      cwagent-var-log-secure = {
+        retention_in_days = 400
+      }
+      cwagent-windows-system = {
+        retention_in_days = 90
+      }
+      cwagent-oasys-autologoff = {
+        retention_in_days = 400
+      }
+      cwagent-web-logs = {
+        retention_in_days = 90
+      }
+    }
   }
 }
diff --git a/terraform/environments/oasys/locals_test.tf b/terraform/environments/oasys/locals_test.tf
@@ -554,5 +554,26 @@ locals {
         ]
       }
     }
+
+    baseline_cloudwatch_log_groups = {
+      session-manager-logs = {
+        retention_in_days = 7
+      }
+      cwagent-var-log-messages = {
+        retention_in_days = 7
+      }
+      cwagent-var-log-secure = {
+        retention_in_days = 7
+      }
+      cwagent-windows-system = {
+        retention_in_days = 7
+      }
+      cwagent-oasys-autologoff = {
+        retention_in_days = 7
+      }
+      cwagent-web-logs = {
+        retention_in_days = 7
+      }
+    }
   }
 }
diff --git a/terraform/environments/oasys/main.tf b/terraform/environments/oasys/main.tf
@@ -68,7 +68,21 @@ module "baseline" {
     lookup(local.environment_config, "baseline_acm_certificates", {})
   )
 
-  cloudwatch_log_groups  = module.baseline_presets.cloudwatch_log_groups
+  cloudwatch_metric_alarms = merge(
+    local.baseline_cloudwatch_metric_alarms,
+    lookup(local.environment_config, "baseline_cloudwatch_metric_alarms", {})
+  )
+
+  cloudwatch_log_metric_filters = merge(
+    local.baseline_cloudwatch_log_metric_filters,
+    lookup(local.environment_config, "baseline_cloudwatch_log_metric_filters", {})
+  )
+
+  cloudwatch_log_groups = merge(
+    module.baseline_presets.cloudwatch_log_groups,
+    local.baseline_cloudwatch_log_groups,
+    lookup(local.environment_config, "baseline_cloudwatch_log_groups", {})
+  )
   ec2_autoscaling_groups = lookup(local.environment_config, "baseline_ec2_autoscaling_groups", {})
   ec2_instances          = lookup(local.environment_config, "baseline_ec2_instances", {})
   environment            = module.environment