diff --git a/kubernetes/cmsweb/monitoring/prometheus/rules/crabserver.rules b/kubernetes/cmsweb/monitoring/prometheus/rules/crabserver.rules index 733313c05..498c240ca 100644 --- a/kubernetes/cmsweb/monitoring/prometheus/rules/crabserver.rules +++ b/kubernetes/cmsweb/monitoring/prometheus/rules/crabserver.rules @@ -1,10 +1,8 @@ groups: - name: crabserver rules: - - record: avg_open_fds_10m - expr: avg_over_time(crabserver_process_open_fds[10m]) - - record: avg_open_fds_30m - expr: avg_over_time(crabserver_process_open_fds[30m]) + - record: avg_open_fds_8m + expr: avg_over_time(crabserver_process_open_fds[8m]) - alert: CRAB server is down expr: crabserver_num_cpus == 0 for: 5m @@ -17,20 +15,8 @@ groups: annotations: summary: "crabserver {{ $labels.env }} is down" description: "{{ $labels.env }} has been down for more than 5m" - - alert: CRAB server service has large number of fds - expr: avg_open_fds_10m > 75 - for: 1m - labels: - severity: warning - tag: cmsweb - service: crab - host: "{{ $labels.host }}" - action: Please check CRAB server on {{ $labels.instance }} and possibly restart it - annotations: - summary: "CRAB {{ $labels.env }} environment" - description: "{{ $labels.env }} has large level of fds {{ $value }} (avg 10m) for more than 1m" - alert: CRAB server service has high number of fds - expr: avg_open_fds_30m > 100 + expr: avg_open_fds_8m > 50 for: 1m labels: severity: high @@ -40,4 +26,4 @@ groups: action: Please restart CRAB server on {{ $labels.instance }} annotations: summary: "CRAB {{ $labels.env }} environment" - description: "{{ $labels.env }} has high level of fds {{ $value }} (avg 30m) for more than 1m" + description: "{{ $labels.env }} has high level of fds {{ $value }} (avg 8m) for more than 1m" diff --git a/kubernetes/cmsweb/monitoring/prometheus/rules/crabserver.test b/kubernetes/cmsweb/monitoring/prometheus/rules/crabserver.test index 47898dd32..6c245ce8c 100644 --- a/kubernetes/cmsweb/monitoring/prometheus/rules/crabserver.test +++ b/kubernetes/cmsweb/monitoring/prometheus/rules/crabserver.test @@ -25,27 +25,8 @@ tests: description: "prod has been down for more than 5m" - interval: 1m input_series: - - series: 'avg_open_fds_10m{env="prod",instance="test-instance",host="k8s-test"}' - values: '76+1x100' - alert_rule_test: - - eval_time: 10m - alertname: CRAB server service has large number of fds - exp_alerts: - - exp_labels: - severity: warning - tag: cmsweb - service: crab - host: k8s-test - action: Please check CRAB server on test-instance and possibly restart it - instance: test-instance - env: prod - exp_annotations: - summary: "CRAB prod environment" - description: "prod has large level of fds 86 (avg 10m) for more than 1m" -- interval: 1m - input_series: - - series: 'avg_open_fds_30m{env="prod",instance="test-instance",host="k8s-test"}' - values: '101+1x100' + - series: 'avg_open_fds_8m{env="prod",instance="test-instance",host="k8s-test"}' + values: '51+1x100' alert_rule_test: - eval_time: 10m alertname: CRAB server service has high number of fds @@ -60,4 +41,4 @@ tests: env: prod exp_annotations: summary: "CRAB prod environment" - description: "prod has high level of fds 111 (avg 30m) for more than 1m" + description: "prod has high level of fds 61 (avg 8m) for more than 1m"