Adjust the promql query to support all the K8s versions

The promql expression:

  sum by (resource) (apiserver_longrunning_requests{pod=~".+",verb="WATCH"})
or
  sum by (resource) (apiserver_registered_watchers{pod=~".+",resource!=""})

queries both the new metric apiserver_longrunning_requests (>=1.23) and the old
metric apiserver_registered_watchers (<=1.23) and combines them with the "or"
set operator:

> vector1 or vector2 results in a vector that contains all original
> elements (label sets + values) of vector1 and additionally all elements
> of vector2 which do not have matching label sets in vector1.

The metric relabeling config in the previous commit guarantees that the label
sets (resource) matches, so that the result of the query is that of the new
metric if it exists, or that of the old metric if the new metric does not exist.

Note that the "total" query used the "count" aggregation which was
semantically not meaningful. This aspect is also fixed in this commit:
the registered watchers / long running requests need to be added up
to get the total value.

Co-authored-by: Istvan Zoltan Ballok <[email protected]>
Co-authored-by: Jeremy Rickards <[email protected]>

charts/seed-monitoring/charts/grafana/dashboards/owners/apiserver-overview.json

@@ -2748,9 +2748,11 @@
       "steppedLine": true,
       "targets": [
         {
-          "expr": "sum(apiserver_registered_watchers{pod=~\"$apiserver\"})by(kind)",
+          "exemplar": true,
+          "expr": "  sum by (resource) (apiserver_longrunning_requests{pod=~\"$apiserver\",verb=\"WATCH\"})\nor\n  sum by (resource) (apiserver_registered_watchers{pod=~\"$apiserver\",resource!=\"\"})",
+          "hide": false,
           "interval": "",
-          "legendFormat": "{{kind}}",
+          "legendFormat": "{{resource}}",
           "refId": "A"
         }
       ],

charts/seed-monitoring/charts/grafana/dashboards/owners/kubernetes-control-plane-status-dashboard.json

@@ -1319,14 +1319,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "apiserver_registered_watchers",
+          "expr": "  sum by (group, version, resource) (apiserver_longrunning_requests{verb=\"WATCH\"})\nor\n  sum by (group, version, resource) (apiserver_registered_watchers{resource!=\"\"})",
           "format": "time_series",
           "intervalFactor": 1,
-          "legendFormat": "{{group}}/{{version}}/{{kind}}",
+          "legendFormat": "{{group}}/{{version}}/{{resource}}",
           "refId": "A"
         },
         {
-          "expr": "count(apiserver_registered_watchers)\n",
+          "expr": "  sum (apiserver_longrunning_requests{verb=\"WATCH\"})\nor\n  sum (apiserver_registered_watchers{resource!=\"\"})",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "total",