From 02f45d07fe580398b43f2eaf92812a2899501fc7 Mon Sep 17 00:00:00 2001 From: Stephan Hesselmann Date: Thu, 2 May 2024 19:31:17 +0200 Subject: [PATCH 1/6] feat: add generic tenant metrics (#238) --- resources/index.json | 3 +- .../pod_monitors/rhacs-scanner-metrics.yaml | 38 ------------------- ...metrics.yaml => rhacs-tenant-metrics.yaml} | 4 +- 3 files changed, 3 insertions(+), 42 deletions(-) delete mode 100644 resources/prometheus/pod_monitors/rhacs-scanner-metrics.yaml rename resources/prometheus/pod_monitors/{rhacs-central-metrics.yaml => rhacs-tenant-metrics.yaml} (92%) diff --git a/resources/index.json b/resources/index.json index f9f20e9d..73f1389f 100644 --- a/resources/index.json +++ b/resources/index.json @@ -4,10 +4,9 @@ "prometheus": { "pod_monitors": [ "prometheus/pod_monitors/prometheus-self-metrics.yaml", - "prometheus/pod_monitors/rhacs-central-metrics.yaml", "prometheus/pod_monitors/rhacs-cloudwatch-exporter.yaml", "prometheus/pod_monitors/rhacs-fleetshard-sync-metrics.yaml", - "prometheus/pod_monitors/rhacs-scanner-metrics.yaml" + "prometheus/pod_monitors/rhacs-tenant-metrics.yaml" ], "rules": [ "prometheus/billing-rules.yaml", diff --git a/resources/prometheus/pod_monitors/rhacs-scanner-metrics.yaml b/resources/prometheus/pod_monitors/rhacs-scanner-metrics.yaml deleted file mode 100644 index 8dcb01e9..00000000 --- a/resources/prometheus/pod_monitors/rhacs-scanner-metrics.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: rhacs-scanner-metrics - labels: - app: rhacs -spec: - selector: - matchLabels: - app: "scanner" - namespaceSelector: - any: true - podMetricsEndpoints: - - path: /metrics - port: monitoring - relabelings: - - action: labeldrop - regex: endpoint - - - sourceLabels: [container] - targetLabel: job - - - action: labelmap - regex: __meta_kubernetes_pod_annotation_rhacs_redhat_com_(.+) - replacement: rhacs_${1} - - - action: labelmap - regex: __meta_kubernetes_pod_label_rhacs_redhat_com_(.+) - replacement: rhacs_${1} - - - sourceLabels: [rhacs_tenant] - targetLabel: rhacs_instance_id - - - action: labeldrop - regex: rhacs_tenant - - - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_version] - targetLabel: rhacs_version diff --git a/resources/prometheus/pod_monitors/rhacs-central-metrics.yaml b/resources/prometheus/pod_monitors/rhacs-tenant-metrics.yaml similarity index 92% rename from resources/prometheus/pod_monitors/rhacs-central-metrics.yaml rename to resources/prometheus/pod_monitors/rhacs-tenant-metrics.yaml index 9d198015..a84287a0 100644 --- a/resources/prometheus/pod_monitors/rhacs-central-metrics.yaml +++ b/resources/prometheus/pod_monitors/rhacs-tenant-metrics.yaml @@ -1,13 +1,13 @@ apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: - name: rhacs-central-metrics + name: rhacs-tenant-metrics labels: app: rhacs spec: selector: matchLabels: - app: "central" + app.kubernetes.io/name: stackrox namespaceSelector: any: true podMetricsEndpoints: From 825ca29558f230dbd6c66868352a9b34d835d227 Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Mon, 6 May 2024 15:31:19 -0400 Subject: [PATCH 2/6] ROX-24055: Annotations for cluster upgrades --- .../dashboards/rhacs-cluster-overview-configmap.yaml | 11 +++++++++++ .../dashboards/rhacs-cluster-overview-dashboard.yaml | 11 +++++++++++ resources/grafana/sources/rhacs-cluster-overview.json | 11 +++++++++++ scripts/generate-federate-match.sh | 6 +++++- 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml b/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml index 8c10d07c..f5155355 100644 --- a/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml +++ b/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml @@ -29,6 +29,17 @@ data: "type": "dashboard" }, "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "expr": "count by(cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "iconColor": "purple", + "name": "Upgrade", + "textFormat": "Kubernetes Upgrade" } ] }, diff --git a/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml b/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml index e7e3091e..73e7cab9 100644 --- a/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml +++ b/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml @@ -29,6 +29,17 @@ spec: "type": "dashboard" }, "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "expr": "count by(cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "iconColor": "purple", + "name": "Upgrade", + "textFormat": "Kubernetes Upgrade" } ] }, diff --git a/resources/grafana/sources/rhacs-cluster-overview.json b/resources/grafana/sources/rhacs-cluster-overview.json index 0841eb42..e5768524 100644 --- a/resources/grafana/sources/rhacs-cluster-overview.json +++ b/resources/grafana/sources/rhacs-cluster-overview.json @@ -18,6 +18,17 @@ "type": "dashboard" }, "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "expr": "count by(cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "iconColor": "purple", + "name": "Upgrade", + "textFormat": "Kubernetes Upgrade" } ] }, diff --git a/scripts/generate-federate-match.sh b/scripts/generate-federate-match.sh index f6c57feb..cc5a29cc 100755 --- a/scripts/generate-federate-match.sh +++ b/scripts/generate-federate-match.sh @@ -1,7 +1,11 @@ #!/usr/bin/env bash set -eou pipefail -shopt -s inherit_errexit + +# only if not OSX +if [[ $(uname) != "Darwin" ]]; then + shopt -s inherit_errexit +fi function log() { echo "$@" >&2 From 229cc853ebc6c79aac301675ffffbc2e5ed20658 Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Tue, 7 May 2024 13:27:39 -0400 Subject: [PATCH 3/6] ROX-24055: Annotations for cluster upgrades --- .../generated/dashboards/rhacs-cluster-overview-configmap.yaml | 2 +- .../generated/dashboards/rhacs-cluster-overview-dashboard.yaml | 2 +- resources/grafana/sources/rhacs-cluster-overview.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml b/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml index f5155355..bb55db02 100644 --- a/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml +++ b/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml @@ -36,7 +36,7 @@ data: "uid": "PBFA97CFB590B2093" }, "enable": true, - "expr": "count by(cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", "iconColor": "purple", "name": "Upgrade", "textFormat": "Kubernetes Upgrade" diff --git a/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml b/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml index 73e7cab9..c2e162f8 100644 --- a/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml +++ b/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml @@ -36,7 +36,7 @@ spec: "uid": "PBFA97CFB590B2093" }, "enable": true, - "expr": "count by(cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", "iconColor": "purple", "name": "Upgrade", "textFormat": "Kubernetes Upgrade" diff --git a/resources/grafana/sources/rhacs-cluster-overview.json b/resources/grafana/sources/rhacs-cluster-overview.json index e5768524..3407c9a4 100644 --- a/resources/grafana/sources/rhacs-cluster-overview.json +++ b/resources/grafana/sources/rhacs-cluster-overview.json @@ -25,7 +25,7 @@ "uid": "PBFA97CFB590B2093" }, "enable": true, - "expr": "count by(cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", "iconColor": "purple", "name": "Upgrade", "textFormat": "Kubernetes Upgrade" From cb58b71712851b7ae2f1bc740c01fde1f8609413 Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Tue, 7 May 2024 13:38:24 -0400 Subject: [PATCH 4/6] ROX-24055: Annotations for cluster upgrades --- .../rhacs-cluster-overview-configmap.yaml | 15 +++++++++++++-- .../rhacs-cluster-overview-dashboard.yaml | 15 +++++++++++++-- .../grafana/sources/rhacs-cluster-overview.json | 15 +++++++++++++-- resources/prometheus/federation-config-base.yaml | 1 + resources/prometheus/federation-config.yaml | 1 + 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml b/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml index bb55db02..c0affb7f 100644 --- a/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml +++ b/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml @@ -36,10 +36,21 @@ data: "uid": "PBFA97CFB590B2093" }, "enable": true, - "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", "iconColor": "purple", - "name": "Upgrade", + "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "name": "Kubernetes Upgrade", "textFormat": "Kubernetes Upgrade" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "iconColor": "purple", + "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1", + "name": "Openshift Upgrade", + "textFormat": "Openshift Upgrade" } ] }, diff --git a/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml b/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml index c2e162f8..c95777e9 100644 --- a/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml +++ b/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml @@ -36,10 +36,21 @@ spec: "uid": "PBFA97CFB590B2093" }, "enable": true, - "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", "iconColor": "purple", - "name": "Upgrade", + "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "name": "Kubernetes Upgrade", "textFormat": "Kubernetes Upgrade" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "iconColor": "purple", + "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1", + "name": "Openshift Upgrade", + "textFormat": "Openshift Upgrade" } ] }, diff --git a/resources/grafana/sources/rhacs-cluster-overview.json b/resources/grafana/sources/rhacs-cluster-overview.json index 3407c9a4..442d4d23 100644 --- a/resources/grafana/sources/rhacs-cluster-overview.json +++ b/resources/grafana/sources/rhacs-cluster-overview.json @@ -25,10 +25,21 @@ "uid": "PBFA97CFB590B2093" }, "enable": true, - "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", "iconColor": "purple", - "name": "Upgrade", + "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "name": "Kubernetes Upgrade", "textFormat": "Kubernetes Upgrade" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "iconColor": "purple", + "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1", + "name": "Openshift Upgrade", + "textFormat": "Openshift Upgrade" } ] }, diff --git a/resources/prometheus/federation-config-base.yaml b/resources/prometheus/federation-config-base.yaml index 33a70c2f..60b16a5b 100644 --- a/resources/prometheus/federation-config-base.yaml +++ b/resources/prometheus/federation-config-base.yaml @@ -3,3 +3,4 @@ match[]: # This is an example. - up{job!~"central|scanner"} + - openshift_apiserver_build_info diff --git a/resources/prometheus/federation-config.yaml b/resources/prometheus/federation-config.yaml index c63da907..11b05d7c 100644 --- a/resources/prometheus/federation-config.yaml +++ b/resources/prometheus/federation-config.yaml @@ -154,6 +154,7 @@ match[]: - node_netstat_Tcp_RetransSegs{job!~"central|scanner"} - node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{job!~"central|scanner"} - obs_operator:prometheus_remote_storage_succeeded_samples:ratio_rate1h{job!~"central|scanner"} + - openshift_apiserver_build_info - process_cpu_seconds_total{job!~"central|scanner"} - process_resident_memory_bytes{job!~"central|scanner"} - prometheus_remote_storage_samples_failed_total{job!~"central|scanner"} From e673cf6957f71147e3c1bd5e4d63f281b43f7abb Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Tue, 7 May 2024 14:00:54 -0400 Subject: [PATCH 5/6] ROX-24055: Annotations for cluster upgrades --- .../dashboards/rhacs-cluster-overview-configmap.yaml | 6 +++--- .../dashboards/rhacs-cluster-overview-dashboard.yaml | 6 +++--- resources/grafana/sources/rhacs-cluster-overview.json | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml b/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml index c0affb7f..bfa11a11 100644 --- a/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml +++ b/resources/grafana/generated/dashboards/rhacs-cluster-overview-configmap.yaml @@ -47,10 +47,10 @@ data: "uid": "PBFA97CFB590B2093" }, "enable": true, - "iconColor": "purple", + "iconColor": "red", "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1", - "name": "Openshift Upgrade", - "textFormat": "Openshift Upgrade" + "name": "OpenShift Upgrade", + "textFormat": "OpenShift Upgrade" } ] }, diff --git a/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml b/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml index c95777e9..6ea35bf0 100644 --- a/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml +++ b/resources/grafana/generated/dashboards/rhacs-cluster-overview-dashboard.yaml @@ -47,10 +47,10 @@ spec: "uid": "PBFA97CFB590B2093" }, "enable": true, - "iconColor": "purple", + "iconColor": "red", "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1", - "name": "Openshift Upgrade", - "textFormat": "Openshift Upgrade" + "name": "OpenShift Upgrade", + "textFormat": "OpenShift Upgrade" } ] }, diff --git a/resources/grafana/sources/rhacs-cluster-overview.json b/resources/grafana/sources/rhacs-cluster-overview.json index 442d4d23..8969a623 100644 --- a/resources/grafana/sources/rhacs-cluster-overview.json +++ b/resources/grafana/sources/rhacs-cluster-overview.json @@ -36,10 +36,10 @@ "uid": "PBFA97CFB590B2093" }, "enable": true, - "iconColor": "purple", + "iconColor": "red", "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1", - "name": "Openshift Upgrade", - "textFormat": "Openshift Upgrade" + "name": "OpenShift Upgrade", + "textFormat": "OpenShift Upgrade" } ] }, From 97f63bd9d7e1f7692cc1c2dadf60c59d5ac0b849 Mon Sep 17 00:00:00 2001 From: Ludovic Cleroux Date: Fri, 10 May 2024 13:25:58 -0400 Subject: [PATCH 6/6] ROX-24055: Annotations for cluster upgrades (#241) --- .../rhacs-central-slo-configmap.yaml | 22 +++++++++++++++++++ .../rhacs-central-slo-dashboard.yaml | 22 +++++++++++++++++++ .../grafana/sources/rhacs-central-slo.json | 22 +++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/resources/grafana/generated/dashboards/rhacs-central-slo-configmap.yaml b/resources/grafana/generated/dashboards/rhacs-central-slo-configmap.yaml index b7b0833c..01962278 100644 --- a/resources/grafana/generated/dashboards/rhacs-central-slo-configmap.yaml +++ b/resources/grafana/generated/dashboards/rhacs-central-slo-configmap.yaml @@ -29,6 +29,28 @@ data: "type": "dashboard" }, "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "iconColor": "purple", + "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "name": "Kubernetes Upgrade", + "textFormat": "Kubernetes Upgrade" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "iconColor": "red", + "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1", + "name": "OpenShift Upgrade", + "textFormat": "OpenShift Upgrade" } ] }, diff --git a/resources/grafana/generated/dashboards/rhacs-central-slo-dashboard.yaml b/resources/grafana/generated/dashboards/rhacs-central-slo-dashboard.yaml index c89ecba2..ff27bad0 100644 --- a/resources/grafana/generated/dashboards/rhacs-central-slo-dashboard.yaml +++ b/resources/grafana/generated/dashboards/rhacs-central-slo-dashboard.yaml @@ -29,6 +29,28 @@ spec: "type": "dashboard" }, "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "iconColor": "purple", + "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "name": "Kubernetes Upgrade", + "textFormat": "Kubernetes Upgrade" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "iconColor": "red", + "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1", + "name": "OpenShift Upgrade", + "textFormat": "OpenShift Upgrade" } ] }, diff --git a/resources/grafana/sources/rhacs-central-slo.json b/resources/grafana/sources/rhacs-central-slo.json index 45db8f0d..caa3c5a3 100644 --- a/resources/grafana/sources/rhacs-central-slo.json +++ b/resources/grafana/sources/rhacs-central-slo.json @@ -18,6 +18,28 @@ "type": "dashboard" }, "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "iconColor": "purple", + "expr": "count (count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"}, \"git_version\", \"$1\", \"git_version\", \"(v[0-9]*.[0-9]*).*\"))) > 1", + "name": "Kubernetes Upgrade", + "textFormat": "Kubernetes Upgrade" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "enable": true, + "iconColor": "red", + "expr": "count (count by (gitVersion) (openshift_apiserver_build_info)) > 1", + "name": "OpenShift Upgrade", + "textFormat": "OpenShift Upgrade" } ] },