Heartbeat fixes and improvments for running vm (#557)

* feat: remove heartbeat default maxDelay and increase timeframe feat: make filtering for running vm optional * remove useless spaces in generated terraform * improve variable description for timeframe
claranet · Jun 11, 2024 · 4f1ea67 · 4f1ea67
1 parent 93dd417
commit 4f1ea67
Show file tree

Hide file tree

Showing 113 changed files with 477 additions and 270 deletions.
diff --git a/modules/cwagent-ec2/detectors-gen.tf b/modules/cwagent-ec2/detectors-gen.tf
@@ -7,7 +7,7 @@ resource "signalfx_detector" "heartbeat" {
 
   program_text = <<-EOF
     from signalfx.detectors.not_reporting import not_reporting
-    signal = data('mem_used_percent', filter=${local.not_running_vm_filters} and ${module.filtering.signalflow})${var.heartbeat_aggregation_function}.publish('signal')
+    signal = data('mem_used_percent', filter=%{if var.heartbeat_exclude_not_running_vm}${local.not_running_vm_filters} and %{endif}${module.filtering.signalflow})${var.heartbeat_aggregation_function}.publish('signal')
     not_reporting.detector(stream=signal, resource_identifier=None, duration='${var.heartbeat_timeframe}', auto_resolve_after='${local.heartbeat_auto_resolve_after}').publish('CRIT')
 EOF
 

diff --git a/modules/cwagent-ec2/variables-gen.tf b/modules/cwagent-ec2/variables-gen.tf
@@ -15,7 +15,7 @@ variable "heartbeat_aggregation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -36,10 +36,16 @@ variable "heartbeat_disabled" {
   default     = null
 }
 
+variable "heartbeat_exclude_not_running_vm" {
+  description = "Don’t send alerts if associated VM is stopped or stopping (metadata provided by cloud provider integration). Can be useful for ephemeral infrastructure (such as auto scaling groups) as VM will be stopped and started regularly. Note that timeframe must be at least 25 minutes for the metadata to be available to the detector."
+  type        = bool
+  default     = true
+}
+
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\"). Must be at least \"25m\" if \"heartbeat_exclude_not_running_vm\" is true"
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # mem detector

diff --git a/modules/fame_azure-vpn/variables-gen.tf b/modules/fame_azure-vpn/variables-gen.tf
@@ -21,7 +21,7 @@ variable "heartbeat_transformation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -43,9 +43,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # totalflowcount detector

diff --git a/modules/integration_aws-alb/variables-gen.tf b/modules/integration_aws-alb/variables-gen.tf
@@ -15,7 +15,7 @@ variable "heartbeat_aggregation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -37,9 +37,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # latency detector

diff --git a/modules/integration_aws-beanstalk/variables-gen.tf b/modules/integration_aws-beanstalk/variables-gen.tf
@@ -21,7 +21,7 @@ variable "heartbeat_transformation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -43,9 +43,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # health detector

diff --git a/modules/integration_aws-ecs-cluster/variables-gen.tf b/modules/integration_aws-ecs-cluster/variables-gen.tf
@@ -15,7 +15,7 @@ variable "heartbeat_aggregation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -37,9 +37,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # cpu_utilization detector

diff --git a/modules/integration_aws-ecs-service/variables-gen.tf b/modules/integration_aws-ecs-service/variables-gen.tf
@@ -15,7 +15,7 @@ variable "heartbeat_aggregation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -37,9 +37,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # cpu_utilization detector

diff --git a/modules/integration_aws-elasticache-common/variables-gen.tf b/modules/integration_aws-elasticache-common/variables-gen.tf
@@ -15,7 +15,7 @@ variable "heartbeat_aggregation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -37,9 +37,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # evictions detector

diff --git a/modules/integration_aws-elasticsearch/variables-gen.tf b/modules/integration_aws-elasticsearch/variables-gen.tf
@@ -15,7 +15,7 @@ variable "heartbeat_aggregation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -37,9 +37,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # jvm_memory_pressure detector

diff --git a/modules/integration_aws-elb/variables-gen.tf b/modules/integration_aws-elb/variables-gen.tf
@@ -15,7 +15,7 @@ variable "heartbeat_aggregation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -37,9 +37,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # backend_latency detector

diff --git a/modules/integration_aws-kinesis-firehose/variables-gen.tf b/modules/integration_aws-kinesis-firehose/variables-gen.tf
@@ -21,7 +21,7 @@ variable "heartbeat_transformation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -43,9 +43,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # incoming_records detector

diff --git a/modules/integration_aws-nlb/variables-gen.tf b/modules/integration_aws-nlb/variables-gen.tf
@@ -21,7 +21,7 @@ variable "heartbeat_transformation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -43,9 +43,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # no_healthy_instances detector

diff --git a/modules/integration_aws-rds-common/variables-gen.tf b/modules/integration_aws-rds-common/variables-gen.tf
@@ -15,7 +15,7 @@ variable "heartbeat_aggregation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -37,9 +37,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # cpu_usage detector

diff --git a/modules/integration_aws-redshift/variables-gen.tf b/modules/integration_aws-redshift/variables-gen.tf
@@ -15,7 +15,7 @@ variable "heartbeat_aggregation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -37,9 +37,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # cpu_usage detector

diff --git a/modules/integration_aws-sqs/variables-gen.tf b/modules/integration_aws-sqs/variables-gen.tf
@@ -21,7 +21,7 @@ variable "heartbeat_transformation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -43,9 +43,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # visible_messages detector

diff --git a/modules/integration_aws-vpn/variables-gen.tf b/modules/integration_aws-vpn/variables-gen.tf
@@ -21,7 +21,7 @@ variable "heartbeat_transformation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -43,9 +43,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # vpn_status detector

diff --git a/modules/integration_azure-api-management-service/variables-gen.tf b/modules/integration_azure-api-management-service/variables-gen.tf
@@ -21,7 +21,7 @@ variable "heartbeat_transformation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -43,9 +43,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # capacity detector

diff --git a/modules/integration_azure-app-service-plan/variables-gen.tf b/modules/integration_azure-app-service-plan/variables-gen.tf
@@ -21,7 +21,7 @@ variable "heartbeat_transformation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -43,9 +43,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # cpu detector

diff --git a/modules/integration_azure-app-service/variables-gen.tf b/modules/integration_azure-app-service/variables-gen.tf
@@ -21,7 +21,7 @@ variable "heartbeat_transformation_function" {
 variable "heartbeat_max_delay" {
   description = "Enforce max delay for heartbeat detector (use \"0\" or \"null\" for \"Auto\")"
   type        = number
-  default     = 900
+  default     = null
 }
 
 variable "heartbeat_tip" {
@@ -43,9 +43,9 @@ variable "heartbeat_disabled" {
 }
 
 variable "heartbeat_timeframe" {
-  description = "Timeframe for heartbeat detector (i.e. \"10m\")"
+  description = "Timeframe for heartbeat detector (i.e. \"25m\")."
   type        = string
-  default     = "10m"
+  default     = "25m"
 }
 
 # response_time detector