Update the log level for metric scrape failures of the smartagent/kubernetes-proxy receiver from error to debug

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Upgrade the Splunk OpenTelemetry Collector for Kubernetes subchart dependencies [#828](https://github.com/signalfx/splunk-otel-collector-chart/pull/828)
   - cert-manager upgraded from 1.11.1 to [1.12.2](https://github.com/cert-manager/cert-manager/releases/tag/v1.12.2)
   - opentelemetry-operator upgraded from 0.28.0 to [0.32.0)](https://github.com/open-telemetry/opentelemetry-helm-charts/releases/tag/opentelemetry-operator-0.32.0)
+- Update the log level of metric scrape failures of control plane smart agent receivers from error to debug [#832](https://github.com/signalfx/splunk-otel-collector-chart/pull/832)
 
 ## [0.79.1] - 2023-06-22
 

docs/advanced-configuration.md

@@ -379,14 +379,19 @@ agent:
 ### Known issues
 
 Kube Proxy
-* https://github.com/kubernetes/kops/issues/6472
-  * Problem
-    * When using a kops created Kubernetes cluster, a network connectivity issue has been reported that prevents proxy
-      metrics from being collected.
+* `10249: connect: connection refuse`
+  * Issue
+    * When using a Kubernetes cluster with non-default configurations for kube proxy, there is a reported network connectivity issue that prevents the collection of proxy metrics.
   * Solution
-    * This issue can be addressed updating the kubeProxy metric bind address in the kops cluster spec:
-      * Set "kubeProxy.metricsBindAddress: 0.0.0.0" in the kops cluster spec.
-      * Deploy the change with "kops update cluster {cluster_name}" and "kops rolling-update cluster {cluster_name}".
+    * Update the kube proxy metric bind address (--metrics-bind-address) in the cluster spec.
+Set the kubeProxy metrics bind address to 0.0.0.0 or another value based on your Kubernetes cluster distribution.
+For this particular issue, the solution may vary depending on the Kubernetes cluster distribution. It is recommended to research what your Kubernetes distribution recommends for addressing this issue.
+  * Related Issue Links
+    * [kubernetes - Expose kube-proxy metrics on 0.0.0.0 by default ](https://github.com/kubernetes/kubernetes/pull/74300)
+    * [kubernetes - kube-proxy TLS support](https://github.com/kubernetes/kubernetes/issues/106870)
+    * [splunk-otel-collector-chart - Error connecting to kubernetes-proxy](https://github.com/signalfx/splunk-otel-collector-chart/issues/758)
+    * [kops - expose metrics-bind-address configuration for kube-proxy](https://github.com/kubernetes/kops/issues/6472)
+    * [prometheus - prometheus-kube-stack - kube-proxy metrics status with connection refused](https://github.com/prometheus-community/helm-charts/issues/977)
 
 ## Logs collection
 

examples/add-filter-processor/rendered_manifests/configmap-agent.yaml

@@ -242,13 +242,15 @@ data:
               extraDimensions:
                 metric_source: k8s-coredns
               port: 9153
+              scrapeFailureLogLevel: debug
               type: coredns
             rule: type == "pod" && labels["k8s-app"] == "kube-dns"
           smartagent/kube-controller-manager:
             config:
               extraDimensions:
                 metric_source: kubernetes-controller-manager
               port: 10257
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kube-controller-manager
               useHTTPS: true
@@ -258,6 +260,7 @@ data:
             config:
               extraDimensions:
                 metric_source: kubernetes-apiserver
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-apiserver
               useHTTPS: true
@@ -268,13 +271,15 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:
             config:
               extraDimensions:
                 metric_source: kubernetes-scheduler
               port: 10259
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-scheduler
               useHTTPS: true

examples/add-filter-processor/rendered_manifests/daemonset.yaml

@@ -30,7 +30,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: e68f45dc5d5ad899a71a006ace9941aaa37d88abf3b8d2680fda8672c3681acd
+        checksum/config: 9910c9dd5a2833ecdf545ae6738ea328ef15a0a9d5c564319414824c921d1b67
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/add-receiver-creator/rendered_manifests/configmap-agent.yaml

@@ -189,13 +189,15 @@ data:
               extraDimensions:
                 metric_source: k8s-coredns
               port: 9153
+              scrapeFailureLogLevel: debug
               type: coredns
             rule: type == "pod" && labels["k8s-app"] == "kube-dns"
           smartagent/kube-controller-manager:
             config:
               extraDimensions:
                 metric_source: kubernetes-controller-manager
               port: 10257
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kube-controller-manager
               useHTTPS: true
@@ -205,6 +207,7 @@ data:
             config:
               extraDimensions:
                 metric_source: kubernetes-apiserver
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-apiserver
               useHTTPS: true
@@ -215,13 +218,15 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:
             config:
               extraDimensions:
                 metric_source: kubernetes-scheduler
               port: 10259
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-scheduler
               useHTTPS: true

examples/add-receiver-creator/rendered_manifests/daemonset.yaml

@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 87c798151a01dd47d4706373a0b092d87104a41f5b6f2c415c763ccdc43d6e1b
+        checksum/config: aaab76b0ba4409fc761e642ded78e0998d4e3622d52357f6dd86ac5ad92f12e1
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/add-sampler/rendered_manifests/configmap-agent.yaml

@@ -186,13 +186,15 @@ data:
               extraDimensions:
                 metric_source: k8s-coredns
               port: 9153
+              scrapeFailureLogLevel: debug
               type: coredns
             rule: type == "pod" && labels["k8s-app"] == "kube-dns"
           smartagent/kube-controller-manager:
             config:
               extraDimensions:
                 metric_source: kubernetes-controller-manager
               port: 10257
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kube-controller-manager
               useHTTPS: true
@@ -202,6 +204,7 @@ data:
             config:
               extraDimensions:
                 metric_source: kubernetes-apiserver
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-apiserver
               useHTTPS: true
@@ -212,13 +215,15 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:
             config:
               extraDimensions:
                 metric_source: kubernetes-scheduler
               port: 10259
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-scheduler
               useHTTPS: true

examples/add-sampler/rendered_manifests/daemonset.yaml

@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 5e482b805db2c896dfb2132882c8483c3eeedd75b44adbf0ef3864e602a17b7a
+        checksum/config: 42668ca7fe744e89fc0d7c421c90357a91f10c036b8261bd439101d0b44e6335
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/autodetect-istio/rendered_manifests/configmap-agent.yaml

@@ -234,13 +234,15 @@ data:
               extraDimensions:
                 metric_source: k8s-coredns
               port: 9153
+              scrapeFailureLogLevel: debug
               type: coredns
             rule: type == "pod" && labels["k8s-app"] == "kube-dns"
           smartagent/kube-controller-manager:
             config:
               extraDimensions:
                 metric_source: kubernetes-controller-manager
               port: 10257
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kube-controller-manager
               useHTTPS: true
@@ -250,6 +252,7 @@ data:
             config:
               extraDimensions:
                 metric_source: kubernetes-apiserver
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-apiserver
               useHTTPS: true
@@ -260,13 +263,15 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:
             config:
               extraDimensions:
                 metric_source: kubernetes-scheduler
               port: 10259
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-scheduler
               useHTTPS: true

examples/autodetect-istio/rendered_manifests/daemonset.yaml

@@ -30,7 +30,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 87750c2383c72b2201b89caa4951c926b837abf514b7f599b4debb43e37fa22a
+        checksum/config: 3493d3098462df5040526c597dd0f0a1260a943e038ee543e93113df78b39788
         kubectl.kubernetes.io/default-container: otel-collector
         sidecar.istio.io/inject: "false"
     spec:

examples/collector-agent-only/rendered_manifests/configmap-agent.yaml

@@ -183,13 +183,15 @@ data:
               extraDimensions:
                 metric_source: k8s-coredns
               port: 9153
+              scrapeFailureLogLevel: debug
               type: coredns
             rule: type == "pod" && labels["k8s-app"] == "kube-dns"
           smartagent/kube-controller-manager:
             config:
               extraDimensions:
                 metric_source: kubernetes-controller-manager
               port: 10257
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kube-controller-manager
               useHTTPS: true
@@ -199,6 +201,7 @@ data:
             config:
               extraDimensions:
                 metric_source: kubernetes-apiserver
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-apiserver
               useHTTPS: true
@@ -209,13 +212,15 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:
             config:
               extraDimensions:
                 metric_source: kubernetes-scheduler
               port: 10259
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-scheduler
               useHTTPS: true

examples/collector-agent-only/rendered_manifests/daemonset.yaml

@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: e99d38672d7af00809472d0007d39b7347034ee8ce712211c1618b97541b1dc6
+        checksum/config: f4c8840646246bfa2f9a248aa23949069d484766086a9b0e95ed238140595894
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/collector-all-modes/rendered_manifests/configmap-agent.yaml

@@ -169,13 +169,15 @@ data:
               extraDimensions:
                 metric_source: k8s-coredns
               port: 9153
+              scrapeFailureLogLevel: debug
               type: coredns
             rule: type == "pod" && labels["k8s-app"] == "kube-dns"
           smartagent/kube-controller-manager:
             config:
               extraDimensions:
                 metric_source: kubernetes-controller-manager
               port: 10257
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kube-controller-manager
               useHTTPS: true
@@ -185,6 +187,7 @@ data:
             config:
               extraDimensions:
                 metric_source: kubernetes-apiserver
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-apiserver
               useHTTPS: true
@@ -195,13 +198,15 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:
             config:
               extraDimensions:
                 metric_source: kubernetes-scheduler
               port: 10259
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-scheduler
               useHTTPS: true

examples/collector-all-modes/rendered_manifests/daemonset.yaml

@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: b31f3459bf195bd6989c1ce5664661e08afc35ff5baf721cf9c78a26cf5258c0
+        checksum/config: 3664f885e167fee6d3443db28b3e739f37f3f462eb3c9e8f4768546e9ae513cf
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/crio-logging/rendered_manifests/configmap-agent.yaml

@@ -183,13 +183,15 @@ data:
               extraDimensions:
                 metric_source: k8s-coredns
               port: 9153
+              scrapeFailureLogLevel: debug
               type: coredns
             rule: type == "pod" && labels["k8s-app"] == "kube-dns"
           smartagent/kube-controller-manager:
             config:
               extraDimensions:
                 metric_source: kubernetes-controller-manager
               port: 10257
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kube-controller-manager
               useHTTPS: true
@@ -199,6 +201,7 @@ data:
             config:
               extraDimensions:
                 metric_source: kubernetes-apiserver
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-apiserver
               useHTTPS: true
@@ -209,13 +212,15 @@ data:
               extraDimensions:
                 metric_source: kubernetes-proxy
               port: 10249
+              scrapeFailureLogLevel: debug
               type: kubernetes-proxy
             rule: type == "pod" && labels["k8s-app"] == "kube-proxy"
           smartagent/kubernetes-scheduler:
             config:
               extraDimensions:
                 metric_source: kubernetes-scheduler
               port: 10259
+              scrapeFailureLogLevel: debug
               skipVerify: true
               type: kubernetes-scheduler
               useHTTPS: true

examples/crio-logging/rendered_manifests/daemonset.yaml

@@ -29,7 +29,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: e99d38672d7af00809472d0007d39b7347034ee8ce712211c1618b97541b1dc6
+        checksum/config: f4c8840646246bfa2f9a248aa23949069d484766086a9b0e95ed238140595894
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true