Merge pull request #5473 from ministryofjustice/feature/op-alerting

📟 Observability Platform Slack alerting

terraform/environments/observability-platform/.gitignore

@@ -1 +1,2 @@
 builds
+contrib/*.json

terraform/environments/observability-platform/contrib/delete-contact-point.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+# Contact Points and the Notification Policy Tree are tightly coupled, and you cannot delete a Contact Point without first removing it from the Notification Policy Tree.
+# This script will delete a Contact Point from the Notification Policy Tree and then delete the Contact Point itself.
+# To run this script, you will need to provide the following arguments:
+# 1. The stage you are working in (e.g. development, production)
+# 2. The name of the Contact Point you want to delete (e.g. xxx-slack, yyy-pagerduty)
+# Example usage: bash contrib/delete-contact-point.sh development xxx-slack
+
+ENVIRONMENT=$(basename ${PWD})
+STAGE="${1}"
+ROLE="modernisation-platform-developer"
+CONTACT_POINT="${2}"
+
+GRAFANA_API_KEY="$(aws-sso exec --profile ${ENVIRONMENT}-${STAGE}:${ROLE} -- aws secretsmanager get-secret-value --secret-id grafana/api-key --query SecretString --output text)"
+GRAFANA_WORKSPACE_ID="$(aws-sso exec --profile ${ENVIRONMENT}-${STAGE}:${ROLE} -- aws grafana list-workspaces | jq -r '.workspaces[] | select(.name=="observability-platform") | .id')"
+GRAFANA_ENDPOINT="https://${GRAFANA_WORKSPACE_ID}.grafana-workspace.eu-west-2.amazonaws.com"
+
+# Get Notification Policy Tree
+curl \
+  --silent \
+  --request GET \
+  --header "Accept: application/json" \
+  --header "Content-Type: application/json" \
+  --header "Authorization: Bearer ${GRAFANA_API_KEY}" \
+  --url "${GRAFANA_ENDPOINT}/api/v1/provisioning/policies" | jq > contrib/notification-policies-original.json
+
+# Delete Contact Point from Notification Policy Tree
+jq --arg CONTACT_POINT "${CONTACT_POINT}" 'del(.routes[] | select(.receiver==$CONTACT_POINT))' contrib/notification-policies-original.json > contrib/notification-policies-updated.json
+
+# Put modified Notification Policy Tree
+curl \
+  --silent \
+  --request PUT \
+  --header "Accept: application/json" \
+  --header "Content-Type: application/json" \
+  --header "Authorization: Bearer ${GRAFANA_API_KEY}" \
+  --data @contrib/notification-policies-updated.json \
+  --url "${GRAFANA_ENDPOINT}/api/v1/provisioning/policies"
+
+# Get Contact Point UID
+getContactPointUid=$(curl \
+  --silent \
+  --request GET \
+  --header "Accept: application/json" \
+  --header "Content-Type: application/json" \
+  --header "Authorization: Bearer ${GRAFANA_API_KEY}" \
+  --url "${GRAFANA_ENDPOINT}/api/v1/provisioning/contact-points" | jq -r --arg CONTACT_POINT "${CONTACT_POINT}" '.[] | select(.name==$CONTACT_POINT) | .uid')
+
+# Delete Contact Point
+curl \
+  --silent \
+  --request DELETE \
+  --header "Accept: application/json" \
+  --header "Content-Type: application/json" \
+  --header "Authorization: Bearer ${GRAFANA_API_KEY}" \
+  --url "${GRAFANA_ENDPOINT}/api/v1/provisioning/contact-points/${getContactPointUid}"

terraform/environments/observability-platform/environment-configurations.tf

@@ -5,6 +5,7 @@ locals {
       tenant_configuration = {
         "observability-platform" = {
           identity_centre_team = "observability-platform"
+          slack_channels       = ["observability-platform-development-alerts"]
           aws_accounts = {
             "observability-platform-development" = {
               cloudwatch_enabled      = true
@@ -69,6 +70,7 @@ locals {
       tenant_configuration = {
         "observability-platform" = {
           identity_centre_team = "observability-platform"
+          slack_channels       = ["observability-platform-production-alerts"]
           aws_accounts = {
             "observability-platform-production" = {
               cloudwatch_enabled      = true

terraform/environments/observability-platform/locals.tf

@@ -5,6 +5,11 @@ locals {
     lookup(tenant_config, "identity_centre_team", []) if tenant_name != "observability-platform"
   ]))
 
+  all_slack_channels = distinct(flatten([
+    for tenant in local.environment_configuration.tenant_configuration :
+    [for channel in lookup(tenant, "slack_channels", []) : channel]
+  ]))
+
   all_aws_accounts = flatten([
     for tenant_name, tenant_config in local.environment_configuration.tenant_configuration : [
       for account_name, _ in lookup(tenant_config, "aws_accounts", {}) : account_name

terraform/environments/observability-platform/managed-grafana.tf

@@ -37,6 +37,38 @@ module "managed_grafana" {
   tags = local.tags
 }
 
+/* Slack Contact Points */
+module "contact_point_slack" {
+  for_each = toset(local.all_slack_channels)
+
+  source = "./modules/grafana/contact-point/slack"
+
+  channel = each.value
+}
+
+/* Notification Policy */
+resource "grafana_notification_policy" "root" {
+  contact_point   = "grafana-default-sns"
+  group_by        = ["..."]
+  group_wait      = "30s"
+  group_interval  = "5m"
+  repeat_interval = "4h"
+
+  dynamic "policy" {
+    for_each = toset(local.all_slack_channels)
+    content {
+      matcher {
+        label = "slack-channel"
+        match = "="
+        value = policy.value
+      }
+      contact_point = "${policy.value}-slack"
+    }
+  }
+
+  depends_on = [module.contact_point_slack]
+}
+
 /* Prometheus Source */
 resource "grafana_data_source" "observability_platform_prometheus" {
   type       = "prometheus"

terraform/environments/observability-platform/modules/grafana/contact-point/slack/main.tf

@@ -0,0 +1,12 @@
+data "aws_secretsmanager_secret_version" "github_token" {
+  secret_id = "grafana/notifications/slack-token"
+}
+
+resource "grafana_contact_point" "this" {
+  name = "${var.channel}-slack"
+
+  slack {
+    recipient = var.channel
+    token     = data.aws_secretsmanager_secret_version.github_token.secret_string
+  }
+}

terraform/environments/observability-platform/modules/grafana/contact-point/slack/providers.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    grafana = {
+      source  = "grafana/grafana"
+      version = "~> 2.0"
+    }
+  }
+}

terraform/environments/observability-platform/modules/grafana/contact-point/slack/variables.tf

@@ -0,0 +1,3 @@
+variable "channel" {
+  type = string
+}