Adding back alerts for DB memory consumption (#768)

We now have a better understanding of expected values, so we are creating a low severity alert at 90% and high severity at 95%.

Signed-off-by: Hayden B <[email protected]>

terraform/gcp/modules/monitoring/infra/alerts.tf

@@ -93,7 +93,7 @@ resource "google_monitoring_alert_policy" "cloud_sql_cpu_utilization_warning" {
   }
   display_name = "Cloud SQL Database CPU Utilization > 80%"
   documentation {
-    content   = "Cloud SQL Database CPU Utilization is >80%. Please increase CPU capacity."
+    content   = "Cloud SQL Database CPU Utilization is >80%. Please increase CPU capacity via the database tier (https://cloud.google.com/sql/docs/mysql/instance-settings)."
     mime_type = "text/markdown"
   }
   enabled               = "true"
@@ -132,14 +132,102 @@ resource "google_monitoring_alert_policy" "cloud_sql_cpu_utilization" {
   }
   display_name = "Cloud SQL Database CPU Utilization > 90%"
   documentation {
-    content   = "Cloud SQL Database CPU Utilization is >90%. Please increase CPU capacity."
+    content   = "Cloud SQL Database CPU Utilization is >90%. Please increase CPU capacity via the database tier (https://cloud.google.com/sql/docs/mysql/instance-settings)."
     mime_type = "text/markdown"
   }
   enabled               = "true"
   notification_channels = local.notification_channels
   project               = var.project_id
 }
 
+# Cloud SQL Database Memory Utilization > 90%
+resource "google_monitoring_alert_policy" "cloud_sql_memory_utilization_warning" {
+  # In the absence of data, incident will auto-close in 7 days
+  alert_strategy {
+    auto_close = "604800s"
+  }
+
+  combiner = "OR"
+
+  conditions {
+    condition_threshold {
+      aggregations {
+        alignment_period   = "300s"
+        per_series_aligner = "ALIGN_MEAN"
+      }
+
+      comparison      = "COMPARISON_GT"
+      duration        = "0s"
+      filter          = "metric.type=\"cloudsql.googleapis.com/database/memory/utilization\" resource.type=\"cloudsql_database\""
+      threshold_value = "0.9"
+
+      trigger {
+        count   = "1"
+        percent = "0"
+      }
+    }
+
+    display_name = "Cloud SQL Database - Memory utilization [MEAN]"
+  }
+
+  display_name = "Cloud SQL Database Memory Utilization > 90%"
+
+  documentation {
+    content   = "Cloud SQL Database Memory Utilization is >90%. Please increase memory capacity via the database tier (https://cloud.google.com/sql/docs/mysql/instance-settings)."
+    mime_type = "text/markdown"
+  }
+
+  enabled               = "true"
+  notification_channels = local.notification_channels
+  project               = var.project_id
+
+  user_labels = {
+    severity = "warning"
+  }
+}
+
+# Cloud SQL Database Memory Utilization > 95%
+resource "google_monitoring_alert_policy" "cloud_sql_memory_utilization" {
+  # In the absence of data, incident will auto-close in 7 days
+  alert_strategy {
+    auto_close = "604800s"
+  }
+
+  combiner = "OR"
+
+  conditions {
+    condition_threshold {
+      aggregations {
+        alignment_period   = "300s"
+        per_series_aligner = "ALIGN_MEAN"
+      }
+
+      comparison      = "COMPARISON_GT"
+      duration        = "0s"
+      filter          = "metric.type=\"cloudsql.googleapis.com/database/memory/utilization\" resource.type=\"cloudsql_database\""
+      threshold_value = "0.95"
+
+      trigger {
+        count   = "1"
+        percent = "0"
+      }
+    }
+
+    display_name = "Cloud SQL Database - Memory utilization [MEAN]"
+  }
+
+  display_name = "Cloud SQL Database Memory Utilization > 95%"
+
+  documentation {
+    content   = "Cloud SQL Database Memory Utilization is >95%. Please increase memory capacity via the database tier (https://cloud.google.com/sql/docs/mysql/instance-settings)."
+    mime_type = "text/markdown"
+  }
+
+  enabled               = "true"
+  notification_channels = local.notification_channels
+  project               = var.project_id
+}
+
 # Cloud SQL Database Disk has < 20GiB Free
 resource "google_monitoring_alert_policy" "cloud_sql_disk_utilization" {
   # In the absence of data, incident will auto-close in 7 days