signalfx · dmitryax · Feb 2, 2022 · Feb 2, 2022 · Feb 2, 2022
@@ -4,6 +4,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+[Upgrade
+guidelines](https://github.com/signalfx/splunk-otel-collector-chart/blob/main/UPGRADING.md#0431-to-0432)
+
+### Changed
+
+- [BREAKING CHANGE] OTel Collector Agent now overrides host and cloud attributes
+  of logs, metrics and traces that are sent through it (#375)
+
 ## [0.43.1] - 2022-02-01
 
 ### Added

@@ -1,5 +1,21 @@
 # Upgrade guidelines
 
+## 0.43.1 to 0.43.2
+
+[#375 Resource detection processor is configured to override all host and cloud
+attributes](https://github.com/signalfx/splunk-otel-collector-chart/pull/375)
+
+If you still want to keep the previous behavior, use the following custom
+values.yaml configuration:
+
+```yaml
+agent:
+  config:
+    processors:
+      resourcedetection:
+        override: false
+```
+
 ## 0.41.0 to 0.42.0
 
 [#357 Double expansion issue in splunk-otel-collector is

@@ -60,8 +60,6 @@ splunk.com/exclude
 Common config for resourcedetection processor
 */}}
 {{- define "splunk-otel-collector.resourceDetectionProcessor" -}}
-# Resource detection processor picks attributes from host environment.
-# https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor
 resourcedetection:
   detectors:
     # Note: Kubernetes distro detectors need to come first so they set the proper cloud.platform
@@ -84,8 +82,7 @@ resourcedetection:
     # The `system` detector goes last so it can't preclude cloud detectors from setting host/os info.
     # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering
     - system
-  # Don't override existing resource attributes to maintain identification of data sources
-  override: false
+  override: true
   timeout: 10s
 {{- end }}
 

@@ -386,6 +386,10 @@ processors:
 
   batch:
 
+  # Resource detection processor is configured to override all host and cloud
+  # attributes because OTel Collector Agent is the source of truth for all host
+  # and cloud metadata, and instrumentation libraries can send wrong host
+  # attributes from container environments.
   {{- include "splunk-otel-collector.resourceDetectionProcessor" . | nindent 2 }}
 
   resource:
@@ -394,7 +398,7 @@ processors:
       - action: insert
         key: k8s.node.name
         value: "${K8S_NODE_NAME}"
-      - action: insert
+      - action: upsert
         key: k8s.cluster.name
         value: {{ .Values.clusterName }}
       {{- range .Values.extraAttributes.custom }}
@@ -511,11 +515,11 @@ service:
         {{- if not $gateway.enabled }}
         - filter/logs
         {{- end }}
-        - resource
         {{- if not $gateway.enabled }}
         - resource/logs
         {{- end }}
         - resourcedetection
+        - resource
         {{- if .Values.environment }}
         - resource/add_environment
         {{- end }}
@@ -569,8 +573,8 @@ service:
         - memory_limiter
         - k8sattributes
         - batch
-        - resource
         - resourcedetection
+        - resource
         {{- if .Values.environment }}
         - resource/add_environment
         {{- end }}
@@ -593,8 +597,8 @@ service:
       processors:
         - memory_limiter
         - batch
-        - resource
         - resourcedetection
+        - resource
         {{- if .Values.isWindows }}
         - metricstransform
         {{- end }}
@@ -618,9 +622,9 @@ service:
       processors:
         - memory_limiter
         - batch
-        - resource
         - resource/add_agent_k8s
         - resourcedetection
+        - resource
       exporters:
         {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }}
         # Use signalfx instead of otlp even if collector is enabled

@@ -236,9 +236,9 @@ service:
       processors:
         - memory_limiter
         - batch
-        - resource/add_cluster_name
         - resource/add_collector_k8s
         - resourcedetection
+        - resource/add_cluster_name
       exporters:
         {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }}
         - signalfx

@@ -194,9 +194,9 @@ service:
       processors:
         - memory_limiter
         - batch
-        - resource
         - resource/add_collector_k8s
         - resourcedetection
+        - resource
       exporters:
         {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }}
         - signalfx

@@ -100,7 +100,7 @@ data:
         - action: insert
           key: k8s.node.name
           value: ${K8S_NODE_NAME}
-        - action: insert
+        - action: upsert
           key: k8s.cluster.name
           value: CHANGEME
       resource/add_agent_k8s:
@@ -127,7 +127,7 @@ data:
         detectors:
         - env
         - system
-        override: false
+        override: true
         timeout: 10s
     receivers:
       hostmetrics:
@@ -204,8 +204,8 @@ data:
           processors:
           - memory_limiter
           - batch
-          - resource
           - resourcedetection
+          - resource
           receivers:
           - hostmetrics
           - kubeletstats
@@ -218,9 +218,9 @@ data:
           processors:
           - memory_limiter
           - batch
-          - resource
           - resource/add_agent_k8s
           - resourcedetection
+          - resource
           receivers:
           - prometheus/agent
         traces:
@@ -231,8 +231,8 @@ data:
           - memory_limiter
           - k8sattributes
           - batch
-          - resource
           - resourcedetection
+          - resource
           receivers:
           - otlp
           - jaeger

@@ -62,7 +62,7 @@ data:
         detectors:
         - env
         - system
-        override: false
+        override: true
         timeout: 10s
     receivers:
       k8s_cluster:
@@ -98,9 +98,9 @@ data:
           processors:
           - memory_limiter
           - batch
-          - resource
           - resource/add_collector_k8s
           - resourcedetection
+          - resource
           receivers:
           - prometheus/k8s_cluster_receiver
       telemetry:

@@ -27,7 +27,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: b56cd78f8c87dbf690836cbb3d1f9b91e62d8e837b89d15df85acd0ff96eddc4
+        checksum/config: 6b534182fcb3dc246c6adfe58c3b6fa1c92d4a93c59a61791807667994ebcc89
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

@@ -30,7 +30,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 925fbd72db4ee177ec316038f01d37486cab4a1c4a90ad62d9da01f7be5aaa2e
+        checksum/config: 123553cb36ed2465fa6415be93afb6feb2cb91846d593998367b72fd3a18756a
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

@@ -68,7 +68,7 @@ data:
         - eks
         - ec2
         - system
-        override: false
+        override: true
         timeout: 10s
     receivers:
       k8s_cluster:
@@ -121,9 +121,9 @@ data:
           processors:
           - memory_limiter
           - batch
-          - resource
           - resource/add_collector_k8s
           - resourcedetection
+          - resource
           receivers:
           - prometheus/k8s_cluster_receiver
         metrics/eks:

@@ -111,7 +111,7 @@ data:
         - eks
         - ec2
         - system
-        override: false
+        override: true
         timeout: 10s
     receivers:
       jaeger:
@@ -170,9 +170,9 @@ data:
           processors:
           - memory_limiter
           - batch
-          - resource/add_cluster_name
           - resource/add_collector_k8s
           - resourcedetection
+          - resource/add_cluster_name
           receivers:
           - prometheus/collector
         traces:

@@ -32,7 +32,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 7b000008a1667ac932745c9f3a4cc3a22cff2aa17ab84f5a45ac95a665e076f7
+        checksum/config: ba21de43588c0a91ee1ab7aeebd4adcbbcb3c76167ead48bcdebc160d78ce0aa
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

@@ -30,7 +30,7 @@ spec:
         component: otel-collector
         release: default
       annotations:
-        checksum/config: 372fb715d4550c53e1329faf387c84ada22f5e1bbacdc5fd1fe6e8f9ce0f65ba
+        checksum/config: a422285772fbd0ac554df75718da49348c3d670d011151a62fbc7f695b6d4459
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

@@ -109,7 +109,7 @@ data:
         detectors:
         - env
         - system
-        override: false
+        override: true
         timeout: 10s
     receivers:
       jaeger:
@@ -168,9 +168,9 @@ data:
           processors:
           - memory_limiter
           - batch
-          - resource/add_cluster_name
           - resource/add_collector_k8s
           - resourcedetection
+          - resource/add_cluster_name
           receivers:
           - prometheus/collector
         traces:

@@ -30,7 +30,7 @@ spec:
         component: otel-collector
         release: default
       annotations:
-        checksum/config: 48bf9d4adc2bdf2883cbcbfc9f813c04a9864efae7ac683fc71d92b9fed805f4
+        checksum/config: 4abdef7c45b9264e6f1fbfd7fb527b5605db39406b5bfc0e50d7b73786d93e74
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

@@ -100,7 +100,7 @@ data:
         - action: insert
           key: k8s.node.name
           value: ${K8S_NODE_NAME}
-        - action: insert
+        - action: upsert
           key: k8s.cluster.name
           value: CHANGEME
       resource/add_agent_k8s:
@@ -127,7 +127,7 @@ data:
         detectors:
         - env
         - system
-        override: false
+        override: true
         timeout: 10s
     receivers:
       fluentforward:
@@ -162,9 +162,9 @@ data:
           - k8sattributes
           - batch
           - filter/logs
-          - resource
           - resource/logs
           - resourcedetection
+          - resource
           receivers:
           - fluentforward
           - otlp
@@ -174,9 +174,9 @@ data:
           processors:
           - memory_limiter
           - batch
-          - resource
           - resource/add_agent_k8s
           - resourcedetection
+          - resource
           receivers:
           - prometheus/agent
       telemetry:

@@ -28,7 +28,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: a78099692ac8e69140978f832900b059e12684a43299ec9a5e983e7015d1b854
+        checksum/config: 1683bbbac9a4569a4ca96864da4a93e517b318be16691424907be1d181749699
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true