From 544639414ec62174361a4bff683c60634003133a Mon Sep 17 00:00:00 2001
From: Mladen Todorovic <mtodor@redhat.com>
Date: Tue, 13 Feb 2024 16:16:21 +0100
Subject: [PATCH] Add option to define base federated metrics

---
 README.md                                   | 2 ++
 resources/prometheus/federation-config.yaml | 6 +++---
 scripts/generate-federate-match.sh          | 3 ++-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 912b80fd..bd323955 100644
--- a/README.md
+++ b/README.md
@@ -62,6 +62,8 @@ If you make changes to Alerts, Recording rules, or Grafana dashboards, and if th
 * Run `make generate-federate` to update federation config.
 * And commit changes in `resources/prometheus/federation-config.yaml` file to the repo.
 
+You can add additional federated metrics that are not used in any Alert, Recording rule, or Grafana dashboard to `resources/prometheus/federation-config-base.yaml,` and they will be merged with other metrics. Always add a comment with the reason why metrics are added to the base list.
+
 ### Pre-commit hook
 
 This repository makes use of [pre-commit](https://pre-commit.com/) framework. Refer to the [installation instructions](https://pre-commit.com/#installation) for further information.
diff --git a/resources/prometheus/federation-config.yaml b/resources/prometheus/federation-config.yaml
index bb4b4749..da3b635d 100644
--- a/resources/prometheus/federation-config.yaml
+++ b/resources/prometheus/federation-config.yaml
@@ -1,7 +1,7 @@
 match[]:
   - :node_memory_MemAvailable_bytes:sum{job!~"central|scanner"}
-  - aggregator_unavailable_apiservice{job!~"central|scanner"}
   - aggregator_unavailable_apiservice_total{job!~"central|scanner"}
+  - aggregator_unavailable_apiservice{job!~"central|scanner"}
   - apiserver_request:availability30d{job!~"central|scanner"}
   - apiserver_request:burnrate1d{job!~"central|scanner"}
   - apiserver_request:burnrate1h{job!~"central|scanner"}
@@ -95,9 +95,9 @@ match[]:
   - kube_statefulset_replicas{job!~"central|scanner"}
   - kube_statefulset_status_current_revision{job!~"central|scanner"}
   - kube_statefulset_status_observed_generation{job!~"central|scanner"}
-  - kube_statefulset_status_replicas{job!~"central|scanner"}
   - kube_statefulset_status_replicas_ready{job!~"central|scanner"}
   - kube_statefulset_status_replicas_updated{job!~"central|scanner"}
+  - kube_statefulset_status_replicas{job!~"central|scanner"}
   - kube_statefulset_status_update_revision{job!~"central|scanner"}
   - kubelet_certificate_manager_client_expiration_renew_errors{job!~"central|scanner"}
   - kubelet_cgroup_manager_duration_seconds_bucket{job!~"central|scanner"}
@@ -121,9 +121,9 @@ match[]:
   - kubelet_server_expiration_renew_errors{job!~"central|scanner"}
   - kubelet_volume_stats_available_bytes{job!~"central|scanner"}
   - kubelet_volume_stats_capacity_bytes{job!~"central|scanner"}
-  - kubelet_volume_stats_inodes{job!~"central|scanner"}
   - kubelet_volume_stats_inodes_free{job!~"central|scanner"}
   - kubelet_volume_stats_inodes_used{job!~"central|scanner"}
+  - kubelet_volume_stats_inodes{job!~"central|scanner"}
   - kubelet_volume_stats_used_bytes{job!~"central|scanner"}
   - kubeproxy_network_programming_duration_seconds_bucket{job!~"central|scanner"}
   - kubeproxy_network_programming_duration_seconds_count{job!~"central|scanner"}
diff --git a/scripts/generate-federate-match.sh b/scripts/generate-federate-match.sh
index 2cc77e92..f6c57feb 100755
--- a/scripts/generate-federate-match.sh
+++ b/scripts/generate-federate-match.sh
@@ -80,7 +80,8 @@ function main() {
     sort "${metrics_list_file}" | uniq | grep -v -E "^acs|^rox|^aws|^central:|acscs_worker_nodes" | awk '{ print $1 "{job!~\"central|scanner\"}" }' > "${metrics_list_file}.filter"
 
     # Create federation-config.yaml
-    sed -e 's/^/- /'  "${metrics_list_file}.filter" | yq '{ "match[]": . }' > "${repo_dir}/resources/prometheus/federation-config.yaml"
+    local yq_expression='. *+ load("'"${repo_dir}/resources/prometheus/federation-config-base.yaml"'")."match[]" | unique | sort | { "match[]": . }'
+    sed -e 's/^/- /'  "${metrics_list_file}.filter" | yq "${yq_expression}" > "${repo_dir}/resources/prometheus/federation-config.yaml"
 
     # Clean up the temp directory with all transient files
     rm -rf "${working_tmp_dir}"