Add kube-proxy to the list of ignored targets (#1379)

giantswarm · Sep 21, 2023 · 76c1412 · 76c1412
1 parent b8a10ce
commit 76c1412
Show file tree

Hide file tree

Showing 9 changed files with 14 additions and 454 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed 
+
+- Ignore kube-proxy target on EKS or clusters with observability bundle >= 0.8.3 (where the kube-proxy service monitor is enabled).
+
 ## [4.49.0] - 2023-09-21
 
 ### Changed

diff --git a/files/templates/scrapeconfigs/additional-scrape-configs.template.yaml b/files/templates/scrapeconfigs/additional-scrape-configs.template.yaml
@@ -338,6 +338,7 @@
     regex: rest_client_(rate_limiter_duration_seconds_bucket|request_size_bytes_bucket|response_size_bytes_bucket)
     action: drop
 [[- end ]]
+[[- if not (contains "kube-proxy" .IgnoredTargets) ]]
 # kube-proxy
 - job_name: [[ .ClusterID ]]-prometheus/kube-proxy-[[ .ClusterID ]]/0
   honor_labels: true
@@ -371,6 +372,7 @@
   - source_labels: [__name__]
     regex: rest_client_(rate_limiter_duration_seconds_bucket|request_size_bytes_bucket|response_size_bytes_bucket)
     action: drop
+[[- end ]]
 [[- if not (contains "coredns" .IgnoredTargets) ]]
 # coredns
 - job_name: [[ .ClusterID ]]-prometheus/coredns-[[ .ClusterID ]]/0

diff --git a/service/controller/resource/monitoring/scrapeconfigs/resource.go b/service/controller/resource/monitoring/scrapeconfigs/resource.go
@@ -273,6 +273,7 @@ func listTargetsToIgnore(ctx context.Context, ctrlClient client.Client, cluster
 			"kube-scheduler",
 			"node-exporter",
 			"kubelet",
+			"kube-proxy",
 			"coredns",
 			"kube-state-metrics",
 			"etcd")
@@ -296,6 +297,10 @@ func listTargetsToIgnore(ctx context.Context, ctrlClient client.Client, cluster
 		if err != nil {
 			return nil, microerror.Mask(err)
 		}
+		bundleWithKubeProxyExporterVersion, err := semver.Parse("0.8.3")
+		if err != nil {
+			return nil, microerror.Mask(err)
+		}
 
 		if version.GTE(initialBundleVersion) {
 			ignoredTargets = append(ignoredTargets, "prometheus-operator-app", "kube-apiserver", "kube-controller-manager", "kube-scheduler", "node-exporter")
@@ -308,6 +313,9 @@ func listTargetsToIgnore(ctx context.Context, ctrlClient client.Client, cluster
 				ignoredTargets = append(ignoredTargets, "etcd")
 			}
 		}
+		if version.GTE(bundleWithKubeProxyExporterVersion) {
+			ignoredTargets = append(ignoredTargets, "kube-proxy")
+		}
 	}
 	// Vintage WC
 	if !key.IsCAPIManagementCluster(config.Provider) && !key.IsManagementCluster(config.Installation, cluster) {

diff --git a/service/controller/resource/monitoring/scrapeconfigs/test/aws/case-6-cluster-api-eks.golden b/service/controller/resource/monitoring/scrapeconfigs/test/aws/case-6-cluster-api-eks.golden
@@ -119,80 +119,6 @@
   # Add customer label.
   - target_label: customer
     replacement: pmo
-# kube-proxy
-- job_name: eks-sample-prometheus/kube-proxy-eks-sample/0
-  honor_labels: true
-  scheme: https
-  kubernetes_sd_configs:
-  - role: pod
-    namespaces:
-      names:
-      - kube-system
-    api_server: https://master.eks-sample:443
-    bearer_token_file: /etc/prometheus/secrets/cluster-certificates/token
-    tls_config:
-      ca_file: /etc/prometheus/secrets/cluster-certificates/ca
-      insecure_skip_verify: false
-  bearer_token_file: /etc/prometheus/secrets/cluster-certificates/token
-  tls_config:
-    ca_file: /etc/prometheus/secrets/cluster-certificates/ca
-    insecure_skip_verify: true
-  relabel_configs:
-  - source_labels: [__address__]
-    replacement: $1:10249
-    target_label: instance
-  - source_labels: [__meta_kubernetes_pod_name]
-    regex: (kube-proxy.*)
-    action: keep
-  - target_label: __address__
-    replacement: master.eks-sample:443
-  - source_labels: [__meta_kubernetes_pod_name]
-    regex: (kube-proxy.*)
-    target_label: __metrics_path__
-    replacement: /api/v1/namespaces/kube-system/pods/${1}:10249/proxy/metrics
-  - target_label: app
-    replacement: kube-proxy
-  # Add namespace label.
-  - source_labels: [__meta_kubernetes_namespace]
-    target_label: namespace
-  # Add pod label.
-  - source_labels: [__meta_kubernetes_pod_name]
-    target_label: pod
-  # Add container label.
-  - source_labels: [__meta_kubernetes_pod_container_name]
-    target_label: container
-  # Add node label.
-  - source_labels: [__meta_kubernetes_pod_node_name]
-    target_label: node
-  # Add role label.
-  - source_labels: [__meta_kubernetes_node_label_role]
-    target_label: role
-  # Add cluster_id label.
-  - target_label: cluster_id
-    replacement: eks-sample
-  # Add cluster_type label.
-  - target_label: cluster_type
-    replacement: workload_cluster
-  # Add provider label.
-  - target_label: provider
-    replacement: aws
-  # Add installation label.
-  - target_label: installation
-    replacement: test-installation
-  # Add priority label.
-  - target_label: service_priority
-    replacement: highest
-  # Add organization label.
-  - target_label: organization
-    replacement: my-organization
-  # Add customer label.
-  - target_label: customer
-    replacement: pmo
-  metric_relabel_configs:
-  # drop unused rest client metrics
-  - source_labels: [__name__]
-    regex: rest_client_(rate_limiter_duration_seconds_bucket|request_size_bytes_bucket|response_size_bytes_bucket)
-    action: drop
 # cert-exporter
 - job_name: eks-sample-prometheus/cert-exporter-eks-sample/0
   honor_labels: true

diff --git a/...ice/controller/resource/monitoring/scrapeconfigs/test/azure/case-6-cluster-api-eks.golden b/...ice/controller/resource/monitoring/scrapeconfigs/test/azure/case-6-cluster-api-eks.golden
@@ -123,82 +123,6 @@
   # Add customer label.
   - target_label: customer
     replacement: pmo
-# kube-proxy
-- job_name: eks-sample-prometheus/kube-proxy-eks-sample/0
-  honor_labels: true
-  scheme: https
-  kubernetes_sd_configs:
-  - role: pod
-    namespaces:
-      names:
-      - kube-system
-    api_server: https://master.eks-sample:443
-    tls_config:
-      ca_file: /etc/prometheus/secrets/cluster-certificates/ca
-      cert_file: /etc/prometheus/secrets/cluster-certificates/crt
-      key_file: /etc/prometheus/secrets/cluster-certificates/key
-      insecure_skip_verify: false
-  tls_config:
-    ca_file: /etc/prometheus/secrets/cluster-certificates/ca
-    cert_file: /etc/prometheus/secrets/cluster-certificates/crt
-    key_file: /etc/prometheus/secrets/cluster-certificates/key
-    insecure_skip_verify: true
-  relabel_configs:
-  - source_labels: [__address__]
-    replacement: $1:10249
-    target_label: instance
-  - source_labels: [__meta_kubernetes_pod_name]
-    regex: (kube-proxy.*)
-    action: keep
-  - target_label: __address__
-    replacement: master.eks-sample:443
-  - source_labels: [__meta_kubernetes_pod_name]
-    regex: (kube-proxy.*)
-    target_label: __metrics_path__
-    replacement: /api/v1/namespaces/kube-system/pods/${1}:10249/proxy/metrics
-  - target_label: app
-    replacement: kube-proxy
-  # Add namespace label.
-  - source_labels: [__meta_kubernetes_namespace]
-    target_label: namespace
-  # Add pod label.
-  - source_labels: [__meta_kubernetes_pod_name]
-    target_label: pod
-  # Add container label.
-  - source_labels: [__meta_kubernetes_pod_container_name]
-    target_label: container
-  # Add node label.
-  - source_labels: [__meta_kubernetes_pod_node_name]
-    target_label: node
-  # Add role label.
-  - source_labels: [__meta_kubernetes_node_label_role]
-    target_label: role
-  # Add cluster_id label.
-  - target_label: cluster_id
-    replacement: eks-sample
-  # Add cluster_type label.
-  - target_label: cluster_type
-    replacement: workload_cluster
-  # Add provider label.
-  - target_label: provider
-    replacement: azure
-  # Add installation label.
-  - target_label: installation
-    replacement: test-installation
-  # Add priority label.
-  - target_label: service_priority
-    replacement: highest
-  # Add organization label.
-  - target_label: organization
-    replacement: my-organization
-  # Add customer label.
-  - target_label: customer
-    replacement: pmo
-  metric_relabel_configs:
-  # drop unused rest client metrics
-  - source_labels: [__name__]
-    regex: rest_client_(rate_limiter_duration_seconds_bucket|request_size_bytes_bucket|response_size_bytes_bucket)
-    action: drop
 # cert-exporter
 - job_name: eks-sample-prometheus/cert-exporter-eks-sample/0
   honor_labels: true

diff --git a/service/controller/resource/monitoring/scrapeconfigs/test/capa/case-6-cluster-api-eks.golden b/service/controller/resource/monitoring/scrapeconfigs/test/capa/case-6-cluster-api-eks.golden
@@ -67,82 +67,6 @@
   # Add customer label.
   - target_label: customer
     replacement: pmo
-# kube-proxy
-- job_name: eks-sample-prometheus/kube-proxy-eks-sample/0
-  honor_labels: true
-  scheme: https
-  kubernetes_sd_configs:
-  - role: pod
-    namespaces:
-      names:
-      - kube-system
-    api_server: https://master.eks-sample:443
-    tls_config:
-      ca_file: /etc/prometheus/secrets/cluster-certificates/ca
-      cert_file: /etc/prometheus/secrets/cluster-certificates/crt
-      key_file: /etc/prometheus/secrets/cluster-certificates/key
-      insecure_skip_verify: false
-  tls_config:
-    ca_file: /etc/prometheus/secrets/cluster-certificates/ca
-    cert_file: /etc/prometheus/secrets/cluster-certificates/crt
-    key_file: /etc/prometheus/secrets/cluster-certificates/key
-    insecure_skip_verify: true
-  relabel_configs:
-  - source_labels: [__address__]
-    replacement: $1:10249
-    target_label: instance
-  - source_labels: [__meta_kubernetes_pod_name]
-    regex: (kube-proxy.*)
-    action: keep
-  - target_label: __address__
-    replacement: master.eks-sample:443
-  - source_labels: [__meta_kubernetes_pod_name]
-    regex: (kube-proxy.*)
-    target_label: __metrics_path__
-    replacement: /api/v1/namespaces/kube-system/pods/${1}:10249/proxy/metrics
-  - target_label: app
-    replacement: kube-proxy
-  # Add namespace label.
-  - source_labels: [__meta_kubernetes_namespace]
-    target_label: namespace
-  # Add pod label.
-  - source_labels: [__meta_kubernetes_pod_name]
-    target_label: pod
-  # Add container label.
-  - source_labels: [__meta_kubernetes_pod_container_name]
-    target_label: container
-  # Add node label.
-  - source_labels: [__meta_kubernetes_pod_node_name]
-    target_label: node
-  # Add role label.
-  - source_labels: [__meta_kubernetes_node_label_role]
-    target_label: role
-  # Add cluster_id label.
-  - target_label: cluster_id
-    replacement: eks-sample
-  # Add cluster_type label.
-  - target_label: cluster_type
-    replacement: workload_cluster
-  # Add provider label.
-  - target_label: provider
-    replacement: capa
-  # Add installation label.
-  - target_label: installation
-    replacement: test-installation
-  # Add priority label.
-  - target_label: service_priority
-    replacement: highest
-  # Add organization label.
-  - target_label: organization
-    replacement: my-organization
-  # Add customer label.
-  - target_label: customer
-    replacement: pmo
-  metric_relabel_configs:
-  # drop unused rest client metrics
-  - source_labels: [__name__]
-    regex: rest_client_(rate_limiter_duration_seconds_bucket|request_size_bytes_bucket|response_size_bytes_bucket)
-    action: drop
 # cert-exporter
 - job_name: eks-sample-prometheus/cert-exporter-eks-sample/0
   honor_labels: true

diff --git a/service/controller/resource/monitoring/scrapeconfigs/test/gcp/case-6-cluster-api-eks.golden b/service/controller/resource/monitoring/scrapeconfigs/test/gcp/case-6-cluster-api-eks.golden
@@ -67,82 +67,6 @@
   # Add customer label.
   - target_label: customer
     replacement: pmo
-# kube-proxy
-- job_name: eks-sample-prometheus/kube-proxy-eks-sample/0
-  honor_labels: true
-  scheme: https
-  kubernetes_sd_configs:
-  - role: pod
-    namespaces:
-      names:
-      - kube-system
-    api_server: https://master.eks-sample:443
-    tls_config:
-      ca_file: /etc/prometheus/secrets/cluster-certificates/ca
-      cert_file: /etc/prometheus/secrets/cluster-certificates/crt
-      key_file: /etc/prometheus/secrets/cluster-certificates/key
-      insecure_skip_verify: false
-  tls_config:
-    ca_file: /etc/prometheus/secrets/cluster-certificates/ca
-    cert_file: /etc/prometheus/secrets/cluster-certificates/crt
-    key_file: /etc/prometheus/secrets/cluster-certificates/key
-    insecure_skip_verify: true
-  relabel_configs:
-  - source_labels: [__address__]
-    replacement: $1:10249
-    target_label: instance
-  - source_labels: [__meta_kubernetes_pod_name]
-    regex: (kube-proxy.*)
-    action: keep
-  - target_label: __address__
-    replacement: master.eks-sample:443
-  - source_labels: [__meta_kubernetes_pod_name]
-    regex: (kube-proxy.*)
-    target_label: __metrics_path__
-    replacement: /api/v1/namespaces/kube-system/pods/${1}:10249/proxy/metrics
-  - target_label: app
-    replacement: kube-proxy
-  # Add namespace label.
-  - source_labels: [__meta_kubernetes_namespace]
-    target_label: namespace
-  # Add pod label.
-  - source_labels: [__meta_kubernetes_pod_name]
-    target_label: pod
-  # Add container label.
-  - source_labels: [__meta_kubernetes_pod_container_name]
-    target_label: container
-  # Add node label.
-  - source_labels: [__meta_kubernetes_pod_node_name]
-    target_label: node
-  # Add role label.
-  - source_labels: [__meta_kubernetes_node_label_role]
-    target_label: role
-  # Add cluster_id label.
-  - target_label: cluster_id
-    replacement: eks-sample
-  # Add cluster_type label.
-  - target_label: cluster_type
-    replacement: workload_cluster
-  # Add provider label.
-  - target_label: provider
-    replacement: gcp
-  # Add installation label.
-  - target_label: installation
-    replacement: test-installation
-  # Add priority label.
-  - target_label: service_priority
-    replacement: highest
-  # Add organization label.
-  - target_label: organization
-    replacement: my-organization
-  # Add customer label.
-  - target_label: customer
-    replacement: pmo
-  metric_relabel_configs:
-  # drop unused rest client metrics
-  - source_labels: [__name__]
-    regex: rest_client_(rate_limiter_duration_seconds_bucket|request_size_bytes_bucket|response_size_bytes_bucket)
-    action: drop
 # cert-exporter
 - job_name: eks-sample-prometheus/cert-exporter-eks-sample/0
   honor_labels: true