Set default send_batch_max_size on k8s cluster receiver

Big k8s clusters with many pods cause k8s cluster receiver to fetch metrics batches with huge number of datapoints that are timing to be sent on signalfx exporter. This change introduces maximum limit of datapoints that one metrics batch can contain. batches with more than 64k datapoints will be broken in several batches
signalfx · Apr 2, 2023 · d380b76 · d380b76
1 parent d29b28e
commit d380b76
Show file tree

Hide file tree

Showing 40 changed files with 62 additions and 38 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Update the Kubernetes scheduler monitor to stop using insecure port 10251 and start using secure port 10259 with authentication [#711](https://github.com/signalfx/splunk-otel-collector-chart/pull/711)
 
+### Fixed
+
+- Sending request timeouts in k8s cluster receiver deployment on big k8s clusters [#717](https://github.com/signalfx/splunk-otel-collector-chart/pull/717)
+
 ### Added
 
 - Add experimental support for deploying OpenTelemetry Operator as a subchart [#691](https://github.com/signalfx/splunk-otel-collector-chart/pull/691)

diff --git a/examples/add-receiver-creator/rendered_manifests/configmap-cluster-receiver.yaml b/examples/add-receiver-creator/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/add-receiver-creator/rendered_manifests/deployment-cluster-receiver.yaml b/examples/add-receiver-creator/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/add-sampler/rendered_manifests/configmap-cluster-receiver.yaml b/examples/add-sampler/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/add-sampler/rendered_manifests/deployment-cluster-receiver.yaml b/examples/add-sampler/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml b/examples/collector-cluster-receiver-only/rendered_manifests/configmap-cluster-receiver.yaml
@@ -33,7 +33,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml b/examples/collector-cluster-receiver-only/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 0c43c73ed8b95ee3bf6bdfbe3b60357b0fcb2c8b5306c5a525e3c3a04db55d8b
+        checksum/config: a1be4b2a3f836c3d03dc67eebcefd37e76aee79ee605b02eb53ccaf2cc6eaa21
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/crio-logging/rendered_manifests/configmap-cluster-receiver.yaml b/examples/crio-logging/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/crio-logging/rendered_manifests/deployment-cluster-receiver.yaml b/examples/crio-logging/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/default/rendered_manifests/configmap-cluster-receiver.yaml b/examples/default/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/default/rendered_manifests/deployment-cluster-receiver.yaml b/examples/default/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/distribution-aks/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-aks/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/distribution-aks/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-aks/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: f92818798cea443e8009c342ec0ae2e6cc3418e8301ab666bfef7b45eacea47a
+        checksum/config: 766af110ef6da6a4a474fc7724819d67e831f2eef429dfb0f69d48a127ff23de
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/distribution-eks-fargate/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-eks-fargate/rendered_manifests/configmap-cluster-receiver.yaml
@@ -31,7 +31,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/distribution-eks-fargate/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-eks-fargate/rendered_manifests/deployment-cluster-receiver.yaml
@@ -32,7 +32,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 1b359e355f48e0001516d21cd16e7d9896e4fb03e6b626a988fab488d126bffb
+        checksum/config: b5c9f6008d6ff36f6aea6388f4ba7e5a4dc8653d17f68b8fa1864f255ff129a3
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/distribution-eks/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-eks/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/distribution-eks/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-eks/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 420c3462317cca26689c016cfff600b5ec3529e07e3a758890989d93e976538b
+        checksum/config: 76bfb8d08cf2cb4ff14e34c69e98cc102116a619030a0430ffcb0254cbbb8b9b
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/distribution-gke-autopilot/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-gke-autopilot/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/distribution-gke-autopilot/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-gke-autopilot/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 9f4e518a28726a74ce0d156f944339ad0541a14640151748d28e1757bcd13620
+        checksum/config: 42ac50d7aaac2a656c40808a2591939df84ec3b650e27e384328aa4f9e3f2d71
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/distribution-gke/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-gke/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/distribution-gke/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-gke/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 9f4e518a28726a74ce0d156f944339ad0541a14640151748d28e1757bcd13620
+        checksum/config: 42ac50d7aaac2a656c40808a2591939df84ec3b650e27e384328aa4f9e3f2d71
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/distribution-openshift/rendered_manifests/configmap-cluster-receiver.yaml b/examples/distribution-openshift/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/distribution-openshift/rendered_manifests/deployment-cluster-receiver.yaml b/examples/distribution-openshift/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 707ed21d7355ed365b199c2b04321cca18bd4410add5c21a4265c4516503d6fd
+        checksum/config: 10ac8d5708dd8d894fd97c6ec0b32d7abebd7061ca1deb80acf0116e0d676286
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/enable-network-explorer/rendered_manifests/configmap-cluster-receiver.yaml b/examples/enable-network-explorer/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/enable-network-explorer/rendered_manifests/deployment-cluster-receiver.yaml b/examples/enable-network-explorer/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/enabled-pprof-extension/rendered_manifests/configmap-cluster-receiver.yaml b/examples/enabled-pprof-extension/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/enabled-pprof-extension/rendered_manifests/deployment-cluster-receiver.yaml b/examples/enabled-pprof-extension/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/filter-container-metrics/rendered_manifests/configmap-cluster-receiver.yaml b/examples/filter-container-metrics/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       filter/exclude_containers:
         metrics:
           exclude:

diff --git a/examples/filter-container-metrics/rendered_manifests/deployment-cluster-receiver.yaml b/examples/filter-container-metrics/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 77bff56590873e36c7b0a53d91393ccbd719e2afabdb60bece57ce70f559f09c
+        checksum/config: 00d8b18dea859c37ab2a4b148a753ef21cb670c32fc36b734103502d3da19e11
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/...uentd-multiline-logs-java-stack-traces/rendered_manifests/configmap-cluster-receiver.yaml b/...uentd-multiline-logs-java-stack-traces/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/...entd-multiline-logs-java-stack-traces/rendered_manifests/deployment-cluster-receiver.yaml b/...entd-multiline-logs-java-stack-traces/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/kubernetes-windows-nodes/rendered_manifests/configmap-cluster-receiver.yaml b/examples/kubernetes-windows-nodes/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/kubernetes-windows-nodes/rendered_manifests/deployment-cluster-receiver.yaml b/examples/kubernetes-windows-nodes/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/only-metrics/rendered_manifests/configmap-cluster-receiver.yaml b/examples/only-metrics/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/only-metrics/rendered_manifests/deployment-cluster-receiver.yaml b/examples/only-metrics/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/...ta-through-gateway-deployed-separately/rendered_manifests/configmap-cluster-receiver.yaml b/...ta-through-gateway-deployed-separately/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/...a-through-gateway-deployed-separately/rendered_manifests/deployment-cluster-receiver.yaml b/...a-through-gateway-deployed-separately/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 5ef6a2963f553a09f4a6f64d586b25bb2564602dfea2b170a4b31a3069797734
+        checksum/config: 2f10fcbfbaa3132a3fdc7ea9eb1e1f43f1cb0425ec6e91ef3fdbaa0f4e26ad97
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/examples/use-proxy/rendered_manifests/configmap-cluster-receiver.yaml b/examples/use-proxy/rendered_manifests/configmap-cluster-receiver.yaml
@@ -27,7 +27,8 @@ data:
       memory_ballast:
         size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     processors:
-      batch: null
+      batch:
+        send_batch_max_size: 32768
       memory_limiter:
         check_interval: 2s
         limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}

diff --git a/examples/use-proxy/rendered_manifests/deployment-cluster-receiver.yaml b/examples/use-proxy/rendered_manifests/deployment-cluster-receiver.yaml
@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: d737da63ea26b6752c9debbc9ddfd8e427cd4722d5a78011dc124b1379ea244d
+        checksum/config: 253afa4f3efc59ac1fb8f9cd821574fc081cf8ceba7f1492bcfb7d8f4e23cf38
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl
@@ -83,6 +83,7 @@ processors:
   {{- include "splunk-otel-collector.otelMemoryLimiterConfig" . | nindent 2 }}
 
   batch:
+    send_batch_max_size: 32768
 
   {{- include "splunk-otel-collector.resourceDetectionProcessor" . | nindent 2 }}
-Original file line number
+Diff line change
@@ Expand Up / @@ -83,6 +83,7 @@ processors: @@
       {{- include "splunk-otel-collector.otelMemoryLimiterConfig" . | nindent 2 }}
       batch:
+        send_batch_max_size: 32768
       {{- include "splunk-otel-collector.resourceDetectionProcessor" . | nindent 2 }}
@@ Expand Down @@