Add alerts for excessive GC of ws-daemon

gitpod-io · Feb 14, 2022 · a48e177 · a48e177
1 parent c0bbf91
commit a48e177
Showing 1 changed file with 29 additions and 1 deletion.
diff --git a/operations/observability/mixins/workspace/rules/components/ws-daemon/alerts.libsonnet b/operations/observability/mixins/workspace/rules/components/ws-daemon/alerts.libsonnet
@@ -23,8 +23,36 @@
               increase(kube_pod_container_status_restarts_total{container="ws-daemon"}[10m]) > 0
             |||,
           },
+          {
+            alert: 'GitpodWsDaemonExcessiveGC',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              runbook_url: '',
+              summary: 'Ws-daemon is doing excessive garbage collection.',
+              description: 'Ws-daemon has excessive garbage collection time. Collecting garbage for more than 1 second.',
+            },
+            expr: |||
+              go_gc_duration_seconds{job="ws-daemon", quantile="1"} > 1
+            |||,
+          },
+          {
+            alert: 'GitpodWsDaemonExcessiveGC',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              runbook_url: '',
+              summary: 'Ws-daemon is doing excessive garbage collection.',
+              description: 'Ws-daemon has excessive garbage collection time. Collecting garbage for more than 1 minute.',
+            },
+            expr: |||
+              go_gc_duration_seconds{job="ws-daemon", quantile="1"} > 60
+            |||,
+          },
         ],
       },
     ],
   },
-}
+}