Adjust the promql query to support all the K8s versions

The promql expression:

    sum by (group, version, kind) (apiserver_registered_watchers)
  + on () group_left ()
    absent(apiserver_longrunning_requests) * 0
or
  sum by (group, version, resource) (apiserver_longrunning_requests)

returns the result of the newer metric `apiserver_longrunning_requests` (>=1.23)
if present, otherwise it will return the `apiserver_registered_watchers` (<1.23).

Note that the "total" query used the "count" aggregation which was semantically
not meaningful. This aspect is also fixed in this commit: the registered
watchers / long running requests need to be added up to get the total value.

Co-authored-by: Istvan Zoltan Ballok <[email protected]>
Co-authored-by: Jeremy Rickards <[email protected]>

charts/seed-monitoring/charts/grafana/dashboards/owners/apiserver-overview.json

@@ -2748,9 +2748,11 @@
       "steppedLine": true,
       "targets": [
         {
-          "expr": "sum(apiserver_registered_watchers{pod=~\"$apiserver\"})by(kind)",
+          "exemplar": true,
+          "expr": "    sum by (kind) (apiserver_registered_watchers{pod=~\"$apiserver\",kind!=\"\"})\n  + on () group_left ()\n    absent(apiserver_longrunning_requests) * 0\nor\n  sum by (resource) (apiserver_longrunning_requests{pod=~\"$apiserver\",verb=\"WATCH\"})",
+          "hide": false,
           "interval": "",
-          "legendFormat": "{{kind}}",
+          "legendFormat": "{{resource}}{{kind}}",
           "refId": "A"
         }
       ],

charts/seed-monitoring/charts/grafana/dashboards/owners/kubernetes-control-plane-status-dashboard.json

@@ -1319,17 +1319,21 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "apiserver_registered_watchers",
+          "exemplar": true,
+          "expr": "    sum by (group, version, kind) (apiserver_registered_watchers)\n  + on () group_left ()\n    absent(apiserver_longrunning_requests) * 0\nor\n  sum by (group, version, resource) (apiserver_longrunning_requests)",
           "format": "time_series",
+          "hide": false,
+          "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{group}}/{{version}}/{{kind}}",
+          "legendFormat": "{{group}}/{{version}}/{{resource}}{{kind}}",
           "refId": "A"
         },
         {
-          "expr": "count(apiserver_registered_watchers)\n",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "total",
+          "exemplar": true,
+          "expr": "sum(\n      sum by (group, version, kind) (apiserver_registered_watchers)\n    + on () group_left ()\n      absent(apiserver_longrunning_requests) * 0\n  or\n    sum by (group, version, resource) (apiserver_longrunning_requests)\n)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Total",
           "refId": "B"
         }
       ],
@@ -1353,15 +1357,17 @@
       },
       "yaxes": [
         {
+          "$$hashKey": "object:244",
           "decimals": 0,
           "format": "short",
           "label": "Count Watches",
-          "logBase": 10,
+          "logBase": 1,
           "max": null,
           "min": null,
           "show": true
         },
         {
+          "$$hashKey": "object:245",
           "format": "short",
           "label": null,
           "logBase": 1,