handover VPA alert to turtle/phoenix (#919)

* handover VPA alert to turtle/phoenix * Add WorkloadClusterWebhookDurationExceedsTimeoutPhoenix for vpa webhook and remove it from atlas
giantswarm · Oct 5, 2023 · df2869a · df2869a
1 parent 85bd281
commit df2869a
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 7 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+
+- Change ownership from Atlas to Turtles/Phoenix for all vertical pod autoscaler alerts.
+
 ## [2.137.0] - 2023-10-04
 
 ### Removed

diff --git a/helm/prometheus-rules/templates/alerting-rules/apiserver.workload-cluster.rules.yml b/helm/prometheus-rules/templates/alerting-rules/apiserver.workload-cluster.rules.yml
@@ -84,12 +84,26 @@ spec:
         team: cabbage
         topic: kubernetes
 
+      # Webhooks owned by Phoenix
+    - alert: WorkloadClusterWebhookDurationExceedsTimeoutPhoenix
+      annotations:
+        description: '{{`Kubernetes API Server admission webhook {{ $labels.name }} is timing out.`}}'
+        opsrecipe: apiserver-admission-webhook-errors/
+      expr: histogram_quantile(0.95, sum(rate(apiserver_admission_webhook_admission_duration_seconds_bucket{cluster_type="workload_cluster", name=~".*(vpa.k8s.io).*"}[5m])) by (cluster_id, name, app, le)) > 5
+      for: 15m
+      labels:
+        area: kaas
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
+        severity: page
+        team: phoenix
+        topic: kubernetes
+
       # Webhooks owned by Atlas
     - alert: WorkloadClusterWebhookDurationExceedsTimeoutAtlas
       annotations:
         description: '{{`Kubernetes API Server admission webhook {{ $labels.name }} is timing out.`}}'
         opsrecipe: apiserver-admission-webhook-errors/
-      expr: histogram_quantile(0.95, sum(rate(apiserver_admission_webhook_admission_duration_seconds_bucket{cluster_type="workload_cluster", name=~".*(prometheus|vpa.k8s.io).*"}[5m])) by (cluster_id, name, app, le)) > 5
+      expr: histogram_quantile(0.95, sum(rate(apiserver_admission_webhook_admission_duration_seconds_bucket{cluster_type="workload_cluster", name=~".*(prometheus).*"}[5m])) by (cluster_id, name, app, le)) > 5
       for: 15m
       labels:
         area: kaas

diff --git a/helm/prometheus-rules/templates/alerting-rules/vpa.all.rules.yml b/helm/prometheus-rules/templates/alerting-rules/vpa.all.rules.yml
@@ -19,13 +19,13 @@ spec:
         1 - sum(increase(kube_pod_container_status_restarts_total{container="vertical-pod-autoscaler-app"}[10m])) by (container, cluster_id, cluster_type, customer, installation, pipeline, provider, region)/100 < 0.98
       for: 10m
       labels:
-        area: managedservices
+        area: kaas
         cancel_if_apiserver_down: "true"
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_status_updating: "true"
         cancel_if_scrape_timeout: "true"
         cancel_if_outside_working_hours: "true"
         severity: page
-        team: atlas
+        team: phoenix
         topic: observability
diff --git a/helm/prometheus-rules/templates/recording-rules/service-level.rules.yml b/helm/prometheus-rules/templates/recording-rules/service-level.rules.yml
@@ -331,10 +331,10 @@ spec:
 
     # -- VPA
     # Amount of requests for VPA
-    - expr: label_replace(count(up{app=~'vertical-pod-autoscaler.*'}) by (cluster_type,cluster_id), "label_application_giantswarm_io_team", "atlas", "", "")
+    - expr: label_replace(count(up{app=~'vertical-pod-autoscaler.*'}) by (cluster_type,cluster_id), "label_application_giantswarm_io_team", "phoenix", "", "")
       labels:
         class: MEDIUM
-        area: platform
+        area: kaas
         service: vertical-pod-autoscaler
       record: raw_slo_requests
 
@@ -344,10 +344,10 @@ spec:
     # and summed with 1 so the final result is 0 : no error recorded.
     # If up was unsuccessful, there is an error. Up returns 0, multiplied by -1 and summed
     # with 1 so the final result is 1 : 1 error is recorded .
-    - expr: label_replace(sum((up{app=~'vertical-pod-autoscaler.*'} * -1) + 1) by (cluster_id, cluster_type), "label_application_giantswarm_io_team", "atlas", "", "")
+    - expr: label_replace(sum((up{app=~'vertical-pod-autoscaler.*'} * -1) + 1) by (cluster_id, cluster_type), "label_application_giantswarm_io_team", "phoenix", "", "")
       labels:
         class: MEDIUM
-        area: platform
+        area: kaas
         service: vertical-pod-autoscaler
       record: raw_slo_errors