From 835eb3481e644c064a237949ee88bbb637d8518e Mon Sep 17 00:00:00 2001 From: DFurt Date: Fri, 13 Dec 2024 16:46:13 +0000 Subject: [PATCH 1/4] add alert condition for synthetics checks --- main.tf | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++ variables.tf | 5 ++ 2 files changed, 146 insertions(+) diff --git a/main.tf b/main.tf index 151017c..88b6f19 100644 --- a/main.tf +++ b/main.tf @@ -952,3 +952,144 @@ resource "pagerduty_service_integration" "non_critical_events_API_v2" { service = pagerduty_service.non_critical[each.key].id type = "events_api_v2_inbound_integration" } + +########################## + +# NewRelic Paid Synthetics Checks Alert + +########################## + +resource "newrelic_alert_policy" "synthetics_checks_alert" { + count = var.synthetics_checks_alert ? 1 : 0 + + name = "Synthetics-Checks-Alert" + incident_preference = "PER_CONDITION_AND_TARGET" +} + +resource "newrelic_nrql_alert_condition" "synthetics_checks_alert" { + count = var.synthetics_checks_alert ? 1 : 0 + + policy_id = newrelic_alert_policy.synthetics_checks_alert[0].id + name = "Synthetics Checks Alert" + enabled = true + + nrql { + query = "SELECT (sum(syntheticsFailedCheckCount) + sum(syntheticsSuccessCheckCount)) AS 'Total Checks' FROM NrDailyUsage WHERE syntheticsTypeLabel != 'Ping' SINCE this month" + } + + critical { + operator = "above_or_equals" + threshold = 9500 + threshold_duration = 300 + threshold_occurrences = "at_least_once" + } + + warning { + operator = "above_or_equals" + threshold = 8000 + threshold_duration = 300 + threshold_occurrences = "at_least_once" + } +} + +resource "newrelic_workflow" "critical_synthetics_checks_alert" { + count = var.synthetics_checks_alert ? 1 : 0 + + name = "Critical-Synthetics-Checks-Alert" + muting_rules_handling = "NOTIFY_ALL_ISSUES" + + issues_filter { + name = "workflow-filter" + type = "FILTER" + + predicate { + attribute = "labels.policyIds" + operator = "EXACTLY_MATCHES" + values = [newrelic_alert_policy.synthetics_checks_alert[0].id] + } + } + destination { + channel_id = newrelic_notification_channel.critical_synthetics_checks_alert[0].id + notification_triggers = ["ACTIVATED", "CLOSED"] + } +} + +resource "newrelic_notification_channel" "critical_synthetics_checks_alert" { + count = var.synthetics_checks_alert ? 1 : 0 + + name = "Synthetics Checks Alert" + type = "PAGERDUTY_SERVICE_INTEGRATION" + destination_id = newrelic_notification_destination.synthetics_checks_alert[0].id + product = "IINT" + property { + key = "summary" + value = "Synthetics checks > ${newrelic_nrql_alert_condition.synthetics_checks_alert[0].critical[0].threshold}" + } + property { + key = "policy_id" + value = newrelic_alert_policy.synthetics_checks_alert[0].id + } + property { + key = "service_key" + value = pagerduty_service_integration.non_critical["NewRelic"].integration_key + } +} + +resource "newrelic_notification_destination" "synthetics_checks_alert" { + count = var.synthetics_checks_alert ? 1 : 0 + + name = "${pagerduty_service.non_critical["NewRelic"].name}-sythetics-checks" + type = "PAGERDUTY_SERVICE_INTEGRATION" + + property { + key = "" + value = "" + } + auth_token { + prefix = "service-integration-id" + token = pagerduty_service_integration.non_critical["NewRelic"].integration_key + } +} + +resource "newrelic_workflow" "non_critical_synthetics_checks_alert" { + count = var.synthetics_checks_alert ? 1 : 0 + + name = "Non-Critical-Synthetics-Checks-Alert" + muting_rules_handling = "NOTIFY_ALL_ISSUES" + + issues_filter { + name = "workflow-filter" + type = "FILTER" + + predicate { + attribute = "labels.policyIds" + operator = "EXACTLY_MATCHES" + values = [newrelic_alert_policy.synthetics_checks_alert[0].id] + } + } + destination { + channel_id = newrelic_notification_channel.non_critical_synthetics_checks_alert[0].id + notification_triggers = ["ACTIVATED", "CLOSED"] + } +} + +resource "newrelic_notification_channel" "non_critical_synthetics_checks_alert" { + count = var.synthetics_checks_alert ? 1 : 0 + + name = "Synthetics Checks Alert" + type = "PAGERDUTY_SERVICE_INTEGRATION" + destination_id = newrelic_notification_destination.synthetics_checks_alert[0].id + product = "IINT" + property { + key = "summary" + value = "Synthetics checks > ${newrelic_nrql_alert_condition.synthetics_checks_alert[0].warning[0].threshold}" + } + property { + key = "policy_id" + value = newrelic_alert_policy.synthetics_checks_alert[0].id + } + property { + key = "service_key" + value = pagerduty_service_integration.non_critical["NewRelic"].integration_key + } +} diff --git a/variables.tf b/variables.tf index 44258de..2db254d 100644 --- a/variables.tf +++ b/variables.tf @@ -13,6 +13,11 @@ variable "newrelic_resource_name_suffix" { default = "" } +variable "synthetics_checks_alert" { + type = bool + default = false +} + variable "simple_monitors" { type = map(object({ name = string From 8965371846d8e21e0b82bc1662c46fe9c98986d3 Mon Sep 17 00:00:00 2001 From: DFurt Date: Mon, 16 Dec 2024 12:10:15 +0000 Subject: [PATCH 2/4] fix for_each in resources --- main.tf | 56 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/main.tf b/main.tf index 88b6f19..c8da1ce 100644 --- a/main.tf +++ b/main.tf @@ -9,10 +9,28 @@ locals { # constructs the name string for each monitor in a single location to avoid repetition. prefix_suffix_map = { for key, _ in local.all_monitors : key => "${local.nr_entity_prefix}${key}${local.nr_entity_suffix}" } + + non_critical_apm_resources = merge( + { for key, value in var.simple_monitors : "${key}" => value if coalesce(value.create_non_critical_apm_resources, false) }, + { for key, value in var.browser_monitors : "${key}" => value if coalesce(value.create_non_critical_apm_resources, false) }, + { for key, value in var.script_monitors : "${key}" => value if coalesce(value.create_non_critical_apm_resources, false) }, + { for key, value in var.step_monitors : "${key}" => value if coalesce(value.create_non_critical_apm_resources, false) }, + { for key, value in var.broken_links_monitors : "${key}" => value if coalesce(value.create_non_critical_apm_resources, false) }, + { for key, value in var.cert_check_monitors : "${key}" => value if coalesce(value.create_non_critical_apm_resources, false) } + ) + + critical_apm_resources = merge( + { for key, value in var.simple_monitors : "${key}" => value if coalesce(value.create_critical_apm_resources, false) }, + { for key, value in var.browser_monitors : "${key}" => value if coalesce(value.create_critical_apm_resources, false) }, + { for key, value in var.script_monitors : "${key}" => value if coalesce(value.create_critical_apm_resources, false) }, + { for key, value in var.step_monitors : "${key}" => value if coalesce(value.create_critical_apm_resources, false) }, + { for key, value in var.broken_links_monitors : "${key}" => value if coalesce(value.create_critical_apm_resources, false) }, + { for key, value in var.cert_check_monitors : "${key}" => value if coalesce(value.create_critical_apm_resources, false) } + ) } data "newrelic_entity" "this" { - for_each = { for key, value in local.all_monitors : key => value if value.create_non_critical_apm_resources || value.create_critical_apm_resources } + for_each = merge(local.non_critical_apm_resources, local.critical_apm_resources) name = local.prefix_suffix_map[each.key] domain = "APM" @@ -371,14 +389,14 @@ resource "pagerduty_service_integration" "synthetics_newrelic" { ########################## resource "newrelic_alert_policy" "critical_apm_response_time" { - for_each = { for key, value in local.all_monitors : key => value if value.create_critical_apm_resources } + for_each = local.critical_apm_resources name = "APM-${local.prefix_suffix_map[each.key]}-response-time-Critical" incident_preference = "PER_CONDITION_AND_TARGET" } resource "newrelic_alert_policy" "critical_apm_error_rate" { - for_each = { for key, value in local.all_monitors : key => value if value.create_critical_apm_resources } + for_each = local.critical_apm_resources name = "APM-${local.prefix_suffix_map[each.key]}-error-rate-Critical" incident_preference = "PER_CONDITION_AND_TARGET" @@ -386,7 +404,7 @@ resource "newrelic_alert_policy" "critical_apm_error_rate" { resource "newrelic_notification_destination" "critical_apm" { # we create only one - count = length({ for key, value in local.all_monitors : key => value if value.create_critical_apm_resources }) > 0 ? 1 : 0 + count = length(local.critical_apm_resources) > 0 ? 1 : 0 name = "${pagerduty_service.critical["NewRelic"].name}-APM" type = "PAGERDUTY_SERVICE_INTEGRATION" @@ -402,7 +420,7 @@ resource "newrelic_notification_destination" "critical_apm" { } resource "newrelic_notification_channel" "critical_apm_response_time" { - for_each = { for key, value in local.all_monitors : key => value if value.create_critical_apm_resources } + for_each = local.critical_apm_resources name = "APM-${local.prefix_suffix_map[each.key]}-response-time-Critical" type = "PAGERDUTY_SERVICE_INTEGRATION" @@ -424,7 +442,7 @@ resource "newrelic_notification_channel" "critical_apm_response_time" { } resource "newrelic_notification_channel" "critical_apm_error_rate" { - for_each = { for key, value in local.all_monitors : key => value if value.create_critical_apm_resources } + for_each = local.critical_apm_resources name = "APM-${local.prefix_suffix_map[each.key]}-error-rate-Critical" type = "PAGERDUTY_SERVICE_INTEGRATION" @@ -446,7 +464,7 @@ resource "newrelic_notification_channel" "critical_apm_error_rate" { } resource "newrelic_nrql_alert_condition" "critical_response_time" { - for_each = { for key, value in local.all_monitors : key => value if value.create_critical_apm_resources } + for_each = local.critical_apm_resources policy_id = newrelic_alert_policy.critical_apm_response_time[each.key].id name = "${data.newrelic_entity.this[each.key].name}-Critical-response-time" @@ -465,7 +483,7 @@ resource "newrelic_nrql_alert_condition" "critical_response_time" { } resource "newrelic_nrql_alert_condition" "critical_error_rate" { - for_each = { for key, value in local.all_monitors : key => value if value.create_critical_apm_resources } + for_each = local.critical_apm_resources policy_id = newrelic_alert_policy.critical_apm_error_rate[each.key].id name = "${data.newrelic_entity.this[each.key].name}-Critical-error-rate" @@ -484,7 +502,7 @@ resource "newrelic_nrql_alert_condition" "critical_error_rate" { } resource "newrelic_workflow" "critical_apm_response_time" { - for_each = { for key, value in local.all_monitors : key => value if value.create_critical_apm_resources } + for_each = local.critical_apm_resources name = "APM-${data.newrelic_entity.this[each.key].name}-Critical-response-time" muting_rules_handling = "NOTIFY_ALL_ISSUES" @@ -506,7 +524,7 @@ resource "newrelic_workflow" "critical_apm_response_time" { } resource "newrelic_workflow" "critical_apm_error_rate" { - for_each = { for key, value in local.all_monitors : key => value if value.create_critical_apm_resources } + for_each = local.critical_apm_resources name = "APM-${data.newrelic_entity.this[each.key].name}-Critical-error-rate" muting_rules_handling = "NOTIFY_ALL_ISSUES" @@ -535,7 +553,7 @@ resource "newrelic_workflow" "critical_apm_error_rate" { resource "newrelic_notification_destination" "non_critical_apm" { - count = length({ for key, value in local.all_monitors : key => value if value.create_non_critical_apm_resources }) > 0 ? 1 : 0 + count = length(local.non_critical_apm_resources) > 0 ? 1 : 0 name = "${pagerduty_service.non_critical["NewRelic"].name}-APM" type = "PAGERDUTY_SERVICE_INTEGRATION" @@ -551,21 +569,21 @@ resource "newrelic_notification_destination" "non_critical_apm" { } resource "newrelic_alert_policy" "non_critical_apm_response_time" { - for_each = { for key, value in local.all_monitors : key => value if value.create_non_critical_apm_resources } + for_each = local.non_critical_apm_resources name = "APM-${local.prefix_suffix_map[each.key]}-response-time-Non_Critical" incident_preference = "PER_CONDITION_AND_TARGET" } resource "newrelic_alert_policy" "non_critical_apm_error_rate" { - for_each = { for key, value in local.all_monitors : key => value if value.create_non_critical_apm_resources } + for_each = local.non_critical_apm_resources name = "APM-${local.prefix_suffix_map[each.key]}-error-rate-Non_Critical" incident_preference = "PER_CONDITION_AND_TARGET" } resource "newrelic_notification_channel" "non_critical_apm_response_time" { - for_each = { for key, value in local.all_monitors : key => value if value.create_non_critical_apm_resources } + for_each = local.non_critical_apm_resources name = "APM-${local.prefix_suffix_map[each.key]}-response-time-Non_Critical" type = "PAGERDUTY_SERVICE_INTEGRATION" @@ -588,7 +606,7 @@ resource "newrelic_notification_channel" "non_critical_apm_response_time" { } resource "newrelic_notification_channel" "non_critical_apm_error_rate" { - for_each = { for key, value in local.all_monitors : key => value if value.create_non_critical_apm_resources } + for_each = local.non_critical_apm_resources name = "APM-${local.prefix_suffix_map[each.key]}-error-rate-Non_Critical" type = "PAGERDUTY_SERVICE_INTEGRATION" @@ -609,7 +627,7 @@ resource "newrelic_notification_channel" "non_critical_apm_error_rate" { } resource "newrelic_nrql_alert_condition" "non_critical_response_time" { - for_each = { for key, value in local.all_monitors : key => value if value.create_non_critical_apm_resources } + for_each = local.non_critical_apm_resources policy_id = newrelic_alert_policy.non_critical_apm_response_time[each.key].id name = "${data.newrelic_entity.this[each.key].name}-Non_Critical-response-time" @@ -628,7 +646,7 @@ resource "newrelic_nrql_alert_condition" "non_critical_response_time" { } resource "newrelic_nrql_alert_condition" "non_critical_error_rate" { - for_each = { for key, value in local.all_monitors : key => value if value.create_non_critical_apm_resources } + for_each = local.non_critical_apm_resources policy_id = newrelic_alert_policy.non_critical_apm_error_rate[each.key].id name = "${data.newrelic_entity.this[each.key].name}-Non_Critical-error-rate" @@ -647,7 +665,7 @@ resource "newrelic_nrql_alert_condition" "non_critical_error_rate" { } resource "newrelic_workflow" "non_critical_apm_response_time" { - for_each = { for key, value in local.all_monitors : key => value if value.create_non_critical_apm_resources } + for_each = local.non_critical_apm_resources name = "APM-${data.newrelic_entity.this[each.key].name}-Non_Critical-response-time" muting_rules_handling = "NOTIFY_ALL_ISSUES" @@ -669,7 +687,7 @@ resource "newrelic_workflow" "non_critical_apm_response_time" { } resource "newrelic_workflow" "non_critical_apm_error_rate" { - for_each = { for key, value in local.all_monitors : key => value if value.create_non_critical_apm_resources } + for_each = local.non_critical_apm_resources name = "APM-${data.newrelic_entity.this[each.key].name}-Non_Critical-error-rate" muting_rules_handling = "NOTIFY_ALL_ISSUES" From f80f7a477392db9a555e9c886b38783ea928b836 Mon Sep 17 00:00:00 2001 From: DFurt Date: Thu, 19 Dec 2024 09:33:44 +0000 Subject: [PATCH 3/4] remove synthetics check alert --- main.tf | 141 -------------------------------------------------------- 1 file changed, 141 deletions(-) diff --git a/main.tf b/main.tf index c8da1ce..a26dc31 100644 --- a/main.tf +++ b/main.tf @@ -970,144 +970,3 @@ resource "pagerduty_service_integration" "non_critical_events_API_v2" { service = pagerduty_service.non_critical[each.key].id type = "events_api_v2_inbound_integration" } - -########################## - -# NewRelic Paid Synthetics Checks Alert - -########################## - -resource "newrelic_alert_policy" "synthetics_checks_alert" { - count = var.synthetics_checks_alert ? 1 : 0 - - name = "Synthetics-Checks-Alert" - incident_preference = "PER_CONDITION_AND_TARGET" -} - -resource "newrelic_nrql_alert_condition" "synthetics_checks_alert" { - count = var.synthetics_checks_alert ? 1 : 0 - - policy_id = newrelic_alert_policy.synthetics_checks_alert[0].id - name = "Synthetics Checks Alert" - enabled = true - - nrql { - query = "SELECT (sum(syntheticsFailedCheckCount) + sum(syntheticsSuccessCheckCount)) AS 'Total Checks' FROM NrDailyUsage WHERE syntheticsTypeLabel != 'Ping' SINCE this month" - } - - critical { - operator = "above_or_equals" - threshold = 9500 - threshold_duration = 300 - threshold_occurrences = "at_least_once" - } - - warning { - operator = "above_or_equals" - threshold = 8000 - threshold_duration = 300 - threshold_occurrences = "at_least_once" - } -} - -resource "newrelic_workflow" "critical_synthetics_checks_alert" { - count = var.synthetics_checks_alert ? 1 : 0 - - name = "Critical-Synthetics-Checks-Alert" - muting_rules_handling = "NOTIFY_ALL_ISSUES" - - issues_filter { - name = "workflow-filter" - type = "FILTER" - - predicate { - attribute = "labels.policyIds" - operator = "EXACTLY_MATCHES" - values = [newrelic_alert_policy.synthetics_checks_alert[0].id] - } - } - destination { - channel_id = newrelic_notification_channel.critical_synthetics_checks_alert[0].id - notification_triggers = ["ACTIVATED", "CLOSED"] - } -} - -resource "newrelic_notification_channel" "critical_synthetics_checks_alert" { - count = var.synthetics_checks_alert ? 1 : 0 - - name = "Synthetics Checks Alert" - type = "PAGERDUTY_SERVICE_INTEGRATION" - destination_id = newrelic_notification_destination.synthetics_checks_alert[0].id - product = "IINT" - property { - key = "summary" - value = "Synthetics checks > ${newrelic_nrql_alert_condition.synthetics_checks_alert[0].critical[0].threshold}" - } - property { - key = "policy_id" - value = newrelic_alert_policy.synthetics_checks_alert[0].id - } - property { - key = "service_key" - value = pagerduty_service_integration.non_critical["NewRelic"].integration_key - } -} - -resource "newrelic_notification_destination" "synthetics_checks_alert" { - count = var.synthetics_checks_alert ? 1 : 0 - - name = "${pagerduty_service.non_critical["NewRelic"].name}-sythetics-checks" - type = "PAGERDUTY_SERVICE_INTEGRATION" - - property { - key = "" - value = "" - } - auth_token { - prefix = "service-integration-id" - token = pagerduty_service_integration.non_critical["NewRelic"].integration_key - } -} - -resource "newrelic_workflow" "non_critical_synthetics_checks_alert" { - count = var.synthetics_checks_alert ? 1 : 0 - - name = "Non-Critical-Synthetics-Checks-Alert" - muting_rules_handling = "NOTIFY_ALL_ISSUES" - - issues_filter { - name = "workflow-filter" - type = "FILTER" - - predicate { - attribute = "labels.policyIds" - operator = "EXACTLY_MATCHES" - values = [newrelic_alert_policy.synthetics_checks_alert[0].id] - } - } - destination { - channel_id = newrelic_notification_channel.non_critical_synthetics_checks_alert[0].id - notification_triggers = ["ACTIVATED", "CLOSED"] - } -} - -resource "newrelic_notification_channel" "non_critical_synthetics_checks_alert" { - count = var.synthetics_checks_alert ? 1 : 0 - - name = "Synthetics Checks Alert" - type = "PAGERDUTY_SERVICE_INTEGRATION" - destination_id = newrelic_notification_destination.synthetics_checks_alert[0].id - product = "IINT" - property { - key = "summary" - value = "Synthetics checks > ${newrelic_nrql_alert_condition.synthetics_checks_alert[0].warning[0].threshold}" - } - property { - key = "policy_id" - value = newrelic_alert_policy.synthetics_checks_alert[0].id - } - property { - key = "service_key" - value = pagerduty_service_integration.non_critical["NewRelic"].integration_key - } -} From f3fdc1e48b5f444a050eaf8e656a9e96dbb1d6af Mon Sep 17 00:00:00 2001 From: DFurt Date: Thu, 19 Dec 2024 09:46:15 +0000 Subject: [PATCH 4/4] remove variable --- variables.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/variables.tf b/variables.tf index 2db254d..44258de 100644 --- a/variables.tf +++ b/variables.tf @@ -13,11 +13,6 @@ variable "newrelic_resource_name_suffix" { default = "" } -variable "synthetics_checks_alert" { - type = bool - default = false -} - variable "simple_monitors" { type = map(object({ name = string