diff --git a/CHANGELOG.md b/CHANGELOG.md index 450a951140..ba19e26b6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +[Upgrade +guidelines](https://github.com/signalfx/splunk-otel-collector-chart/blob/main/UPGRADING.md#0431-to-0432) + +### Changed + +- [BREAKING CHANGE] OTel Collector Agent now overrides host and cloud attributes + of logs, metrics and traces that are sent through it (#375) + ## [0.43.1] - 2022-02-01 ### Added diff --git a/UPGRADING.md b/UPGRADING.md index 3a96e62f7f..5a432d296e 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,5 +1,21 @@ # Upgrade guidelines +## 0.43.1 to 0.43.2 + +[#375 Resource detection processor is configured to override all host and cloud +attributes](https://github.com/signalfx/splunk-otel-collector-chart/pull/375) + +If you still want to keep the previous behavior, use the following custom +values.yaml configuration: + +```yaml +agent: + config: + processors: + resourcedetection: + override: false +``` + ## 0.41.0 to 0.42.0 [#357 Double expansion issue in splunk-otel-collector is diff --git a/helm-charts/splunk-otel-collector/templates/config/_common.tpl b/helm-charts/splunk-otel-collector/templates/config/_common.tpl index daa1207df3..7aae69c6fe 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_common.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_common.tpl @@ -60,8 +60,6 @@ splunk.com/exclude Common config for resourcedetection processor */}} {{- define "splunk-otel-collector.resourceDetectionProcessor" -}} -# Resource detection processor picks attributes from host environment. -# https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor resourcedetection: detectors: # Note: Kubernetes distro detectors need to come first so they set the proper cloud.platform @@ -84,8 +82,7 @@ resourcedetection: # The `system` detector goes last so it can't preclude cloud detectors from setting host/os info. # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering - system - # Don't override existing resource attributes to maintain identification of data sources - override: false + override: true timeout: 10s {{- end }} diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl index b4f969b73b..d3cc4048f0 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl @@ -386,6 +386,10 @@ processors: batch: + # Resource detection processor is configured to override all host and cloud + # attributes because OTel Collector Agent is the source of truth for all host + # and cloud metadata, and instrumentation libraries can send wrong host + # attributes from container environments. {{- include "splunk-otel-collector.resourceDetectionProcessor" . | nindent 2 }} resource: @@ -394,7 +398,7 @@ processors: - action: insert key: k8s.node.name value: "${K8S_NODE_NAME}" - - action: insert + - action: upsert key: k8s.cluster.name value: {{ .Values.clusterName }} {{- range .Values.extraAttributes.custom }} @@ -511,11 +515,11 @@ service: {{- if not $gateway.enabled }} - filter/logs {{- end }} - - resource {{- if not $gateway.enabled }} - resource/logs {{- end }} - resourcedetection + - resource {{- if .Values.environment }} - resource/add_environment {{- end }} @@ -569,8 +573,8 @@ service: - memory_limiter - k8sattributes - batch - - resource - resourcedetection + - resource {{- if .Values.environment }} - resource/add_environment {{- end }} @@ -593,8 +597,8 @@ service: processors: - memory_limiter - batch - - resource - resourcedetection + - resource {{- if .Values.isWindows }} - metricstransform {{- end }} @@ -618,9 +622,9 @@ service: processors: - memory_limiter - batch - - resource - resource/add_agent_k8s - resourcedetection + - resource exporters: {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }} # Use signalfx instead of otlp even if collector is enabled diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl index d52d781e1b..bcf42af577 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl @@ -236,9 +236,9 @@ service: processors: - memory_limiter - batch - - resource/add_cluster_name - resource/add_collector_k8s - resourcedetection + - resource/add_cluster_name exporters: {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }} - signalfx diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl index 478301a1e2..c63b7d1e31 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl @@ -194,9 +194,9 @@ service: processors: - memory_limiter - batch - - resource - resource/add_collector_k8s - resourcedetection + - resource exporters: {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }} - signalfx diff --git a/rendered/manifests/agent-only/configmap-agent.yaml b/rendered/manifests/agent-only/configmap-agent.yaml index eea232282a..a0d94a0f86 100644 --- a/rendered/manifests/agent-only/configmap-agent.yaml +++ b/rendered/manifests/agent-only/configmap-agent.yaml @@ -100,7 +100,7 @@ data: - action: insert key: k8s.node.name value: ${K8S_NODE_NAME} - - action: insert + - action: upsert key: k8s.cluster.name value: CHANGEME resource/add_agent_k8s: @@ -127,7 +127,7 @@ data: detectors: - env - system - override: false + override: true timeout: 10s receivers: hostmetrics: @@ -204,8 +204,8 @@ data: processors: - memory_limiter - batch - - resource - resourcedetection + - resource receivers: - hostmetrics - kubeletstats @@ -218,9 +218,9 @@ data: processors: - memory_limiter - batch - - resource - resource/add_agent_k8s - resourcedetection + - resource receivers: - prometheus/agent traces: @@ -231,8 +231,8 @@ data: - memory_limiter - k8sattributes - batch - - resource - resourcedetection + - resource receivers: - otlp - jaeger diff --git a/rendered/manifests/agent-only/configmap-cluster-receiver.yaml b/rendered/manifests/agent-only/configmap-cluster-receiver.yaml index 404aa4ea55..761c512885 100644 --- a/rendered/manifests/agent-only/configmap-cluster-receiver.yaml +++ b/rendered/manifests/agent-only/configmap-cluster-receiver.yaml @@ -62,7 +62,7 @@ data: detectors: - env - system - override: false + override: true timeout: 10s receivers: k8s_cluster: @@ -98,9 +98,9 @@ data: processors: - memory_limiter - batch - - resource - resource/add_collector_k8s - resourcedetection + - resource receivers: - prometheus/k8s_cluster_receiver telemetry: diff --git a/rendered/manifests/agent-only/daemonset.yaml b/rendered/manifests/agent-only/daemonset.yaml index 2d6104bd41..d58cde7dbd 100644 --- a/rendered/manifests/agent-only/daemonset.yaml +++ b/rendered/manifests/agent-only/daemonset.yaml @@ -27,7 +27,7 @@ spec: app: splunk-otel-collector release: default annotations: - checksum/config: b56cd78f8c87dbf690836cbb3d1f9b91e62d8e837b89d15df85acd0ff96eddc4 + checksum/config: 6b534182fcb3dc246c6adfe58c3b6fa1c92d4a93c59a61791807667994ebcc89 kubectl.kubernetes.io/default-container: otel-collector spec: hostNetwork: true diff --git a/rendered/manifests/agent-only/deployment-cluster-receiver.yaml b/rendered/manifests/agent-only/deployment-cluster-receiver.yaml index cc95ba9316..7031bb6628 100644 --- a/rendered/manifests/agent-only/deployment-cluster-receiver.yaml +++ b/rendered/manifests/agent-only/deployment-cluster-receiver.yaml @@ -30,7 +30,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: 925fbd72db4ee177ec316038f01d37486cab4a1c4a90ad62d9da01f7be5aaa2e + checksum/config: 123553cb36ed2465fa6415be93afb6feb2cb91846d593998367b72fd3a18756a spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/rendered/manifests/eks-fargate/configmap-cluster-receiver.yaml b/rendered/manifests/eks-fargate/configmap-cluster-receiver.yaml index 6b4d2b386a..0ba877636d 100644 --- a/rendered/manifests/eks-fargate/configmap-cluster-receiver.yaml +++ b/rendered/manifests/eks-fargate/configmap-cluster-receiver.yaml @@ -68,7 +68,7 @@ data: - eks - ec2 - system - override: false + override: true timeout: 10s receivers: k8s_cluster: @@ -121,9 +121,9 @@ data: processors: - memory_limiter - batch - - resource - resource/add_collector_k8s - resourcedetection + - resource receivers: - prometheus/k8s_cluster_receiver metrics/eks: diff --git a/rendered/manifests/eks-fargate/configmap-gateway.yaml b/rendered/manifests/eks-fargate/configmap-gateway.yaml index 9b776c1fd1..be513a9b70 100644 --- a/rendered/manifests/eks-fargate/configmap-gateway.yaml +++ b/rendered/manifests/eks-fargate/configmap-gateway.yaml @@ -111,7 +111,7 @@ data: - eks - ec2 - system - override: false + override: true timeout: 10s receivers: jaeger: @@ -170,9 +170,9 @@ data: processors: - memory_limiter - batch - - resource/add_cluster_name - resource/add_collector_k8s - resourcedetection + - resource/add_cluster_name receivers: - prometheus/collector traces: diff --git a/rendered/manifests/eks-fargate/deployment-cluster-receiver.yaml b/rendered/manifests/eks-fargate/deployment-cluster-receiver.yaml index 365f160145..a93f01c80c 100644 --- a/rendered/manifests/eks-fargate/deployment-cluster-receiver.yaml +++ b/rendered/manifests/eks-fargate/deployment-cluster-receiver.yaml @@ -32,7 +32,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: 7b000008a1667ac932745c9f3a4cc3a22cff2aa17ab84f5a45ac95a665e076f7 + checksum/config: ba21de43588c0a91ee1ab7aeebd4adcbbcb3c76167ead48bcdebc160d78ce0aa spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/rendered/manifests/eks-fargate/deployment-gateway.yaml b/rendered/manifests/eks-fargate/deployment-gateway.yaml index aba475002b..6ba6df5948 100644 --- a/rendered/manifests/eks-fargate/deployment-gateway.yaml +++ b/rendered/manifests/eks-fargate/deployment-gateway.yaml @@ -30,7 +30,7 @@ spec: component: otel-collector release: default annotations: - checksum/config: 372fb715d4550c53e1329faf387c84ada22f5e1bbacdc5fd1fe6e8f9ce0f65ba + checksum/config: a422285772fbd0ac554df75718da49348c3d670d011151a62fbc7f695b6d4459 spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/rendered/manifests/gateway-only/configmap-gateway.yaml b/rendered/manifests/gateway-only/configmap-gateway.yaml index 419aced634..3d1c4bc33d 100644 --- a/rendered/manifests/gateway-only/configmap-gateway.yaml +++ b/rendered/manifests/gateway-only/configmap-gateway.yaml @@ -109,7 +109,7 @@ data: detectors: - env - system - override: false + override: true timeout: 10s receivers: jaeger: @@ -168,9 +168,9 @@ data: processors: - memory_limiter - batch - - resource/add_cluster_name - resource/add_collector_k8s - resourcedetection + - resource/add_cluster_name receivers: - prometheus/collector traces: diff --git a/rendered/manifests/gateway-only/deployment-gateway.yaml b/rendered/manifests/gateway-only/deployment-gateway.yaml index 8c6dd800cf..16208c35e8 100644 --- a/rendered/manifests/gateway-only/deployment-gateway.yaml +++ b/rendered/manifests/gateway-only/deployment-gateway.yaml @@ -30,7 +30,7 @@ spec: component: otel-collector release: default annotations: - checksum/config: 48bf9d4adc2bdf2883cbcbfc9f813c04a9864efae7ac683fc71d92b9fed805f4 + checksum/config: 4abdef7c45b9264e6f1fbfd7fb527b5605db39406b5bfc0e50d7b73786d93e74 spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/rendered/manifests/logs-only/configmap-agent.yaml b/rendered/manifests/logs-only/configmap-agent.yaml index f7b772f58d..b2f8182e71 100644 --- a/rendered/manifests/logs-only/configmap-agent.yaml +++ b/rendered/manifests/logs-only/configmap-agent.yaml @@ -100,7 +100,7 @@ data: - action: insert key: k8s.node.name value: ${K8S_NODE_NAME} - - action: insert + - action: upsert key: k8s.cluster.name value: CHANGEME resource/add_agent_k8s: @@ -127,7 +127,7 @@ data: detectors: - env - system - override: false + override: true timeout: 10s receivers: fluentforward: @@ -162,9 +162,9 @@ data: - k8sattributes - batch - filter/logs - - resource - resource/logs - resourcedetection + - resource receivers: - fluentforward - otlp @@ -174,9 +174,9 @@ data: processors: - memory_limiter - batch - - resource - resource/add_agent_k8s - resourcedetection + - resource receivers: - prometheus/agent telemetry: diff --git a/rendered/manifests/logs-only/daemonset.yaml b/rendered/manifests/logs-only/daemonset.yaml index 1c07db2ab7..d2ea947c8a 100644 --- a/rendered/manifests/logs-only/daemonset.yaml +++ b/rendered/manifests/logs-only/daemonset.yaml @@ -28,7 +28,7 @@ spec: app: splunk-otel-collector release: default annotations: - checksum/config: a78099692ac8e69140978f832900b059e12684a43299ec9a5e983e7015d1b854 + checksum/config: 1683bbbac9a4569a4ca96864da4a93e517b318be16691424907be1d181749699 kubectl.kubernetes.io/default-container: otel-collector spec: hostNetwork: true diff --git a/rendered/manifests/metrics-only/configmap-agent.yaml b/rendered/manifests/metrics-only/configmap-agent.yaml index ef7de75ac4..8df3e83983 100644 --- a/rendered/manifests/metrics-only/configmap-agent.yaml +++ b/rendered/manifests/metrics-only/configmap-agent.yaml @@ -97,7 +97,7 @@ data: - action: insert key: k8s.node.name value: ${K8S_NODE_NAME} - - action: insert + - action: upsert key: k8s.cluster.name value: CHANGEME resource/add_agent_k8s: @@ -124,7 +124,7 @@ data: detectors: - env - system - override: false + override: true timeout: 10s receivers: hostmetrics: @@ -190,8 +190,8 @@ data: processors: - memory_limiter - batch - - resource - resourcedetection + - resource receivers: - hostmetrics - kubeletstats @@ -204,9 +204,9 @@ data: processors: - memory_limiter - batch - - resource - resource/add_agent_k8s - resourcedetection + - resource receivers: - prometheus/agent telemetry: diff --git a/rendered/manifests/metrics-only/configmap-cluster-receiver.yaml b/rendered/manifests/metrics-only/configmap-cluster-receiver.yaml index 404aa4ea55..761c512885 100644 --- a/rendered/manifests/metrics-only/configmap-cluster-receiver.yaml +++ b/rendered/manifests/metrics-only/configmap-cluster-receiver.yaml @@ -62,7 +62,7 @@ data: detectors: - env - system - override: false + override: true timeout: 10s receivers: k8s_cluster: @@ -98,9 +98,9 @@ data: processors: - memory_limiter - batch - - resource - resource/add_collector_k8s - resourcedetection + - resource receivers: - prometheus/k8s_cluster_receiver telemetry: diff --git a/rendered/manifests/metrics-only/daemonset.yaml b/rendered/manifests/metrics-only/daemonset.yaml index e3951d62ed..e79eda46db 100644 --- a/rendered/manifests/metrics-only/daemonset.yaml +++ b/rendered/manifests/metrics-only/daemonset.yaml @@ -27,7 +27,7 @@ spec: app: splunk-otel-collector release: default annotations: - checksum/config: 07616385282c9684bc17f3a9101f30fbab5af541a49138c7d490a2f2b804d713 + checksum/config: 263aa7cd62ec0e9d61fd49954e265e128b1467e705c4c84de14d0e1ed305445c kubectl.kubernetes.io/default-container: otel-collector spec: hostNetwork: true diff --git a/rendered/manifests/metrics-only/deployment-cluster-receiver.yaml b/rendered/manifests/metrics-only/deployment-cluster-receiver.yaml index cc95ba9316..7031bb6628 100644 --- a/rendered/manifests/metrics-only/deployment-cluster-receiver.yaml +++ b/rendered/manifests/metrics-only/deployment-cluster-receiver.yaml @@ -30,7 +30,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: 925fbd72db4ee177ec316038f01d37486cab4a1c4a90ad62d9da01f7be5aaa2e + checksum/config: 123553cb36ed2465fa6415be93afb6feb2cb91846d593998367b72fd3a18756a spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/rendered/manifests/otel-logs/configmap-agent.yaml b/rendered/manifests/otel-logs/configmap-agent.yaml index 4e71af2a51..072386ee1a 100644 --- a/rendered/manifests/otel-logs/configmap-agent.yaml +++ b/rendered/manifests/otel-logs/configmap-agent.yaml @@ -94,7 +94,7 @@ data: - action: insert key: k8s.node.name value: ${K8S_NODE_NAME} - - action: insert + - action: upsert key: k8s.cluster.name value: CHANGEME resource/add_agent_k8s: @@ -121,7 +121,7 @@ data: detectors: - env - system - override: false + override: true timeout: 10s receivers: filelog: @@ -293,9 +293,9 @@ data: - k8sattributes - batch - filter/logs - - resource - resource/logs - resourcedetection + - resource receivers: - filelog - fluentforward @@ -306,8 +306,8 @@ data: processors: - memory_limiter - batch - - resource - resourcedetection + - resource receivers: - hostmetrics - kubeletstats @@ -320,9 +320,9 @@ data: processors: - memory_limiter - batch - - resource - resource/add_agent_k8s - resourcedetection + - resource receivers: - prometheus/agent traces: @@ -333,8 +333,8 @@ data: - memory_limiter - k8sattributes - batch - - resource - resourcedetection + - resource receivers: - otlp - jaeger diff --git a/rendered/manifests/otel-logs/configmap-cluster-receiver.yaml b/rendered/manifests/otel-logs/configmap-cluster-receiver.yaml index 404aa4ea55..761c512885 100644 --- a/rendered/manifests/otel-logs/configmap-cluster-receiver.yaml +++ b/rendered/manifests/otel-logs/configmap-cluster-receiver.yaml @@ -62,7 +62,7 @@ data: detectors: - env - system - override: false + override: true timeout: 10s receivers: k8s_cluster: @@ -98,9 +98,9 @@ data: processors: - memory_limiter - batch - - resource - resource/add_collector_k8s - resourcedetection + - resource receivers: - prometheus/k8s_cluster_receiver telemetry: diff --git a/rendered/manifests/otel-logs/daemonset.yaml b/rendered/manifests/otel-logs/daemonset.yaml index 8c4b2bb873..bde16333e5 100644 --- a/rendered/manifests/otel-logs/daemonset.yaml +++ b/rendered/manifests/otel-logs/daemonset.yaml @@ -27,7 +27,7 @@ spec: app: splunk-otel-collector release: default annotations: - checksum/config: b478b3835575050aead2af193c536089e5e89e0017dab925e0e00964b7bea73b + checksum/config: 366c78798d8b71d26a64f089e81717af5be07f691a110f45d0304ca35f9890be kubectl.kubernetes.io/default-container: otel-collector spec: hostNetwork: true diff --git a/rendered/manifests/otel-logs/deployment-cluster-receiver.yaml b/rendered/manifests/otel-logs/deployment-cluster-receiver.yaml index cc95ba9316..7031bb6628 100644 --- a/rendered/manifests/otel-logs/deployment-cluster-receiver.yaml +++ b/rendered/manifests/otel-logs/deployment-cluster-receiver.yaml @@ -30,7 +30,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: 925fbd72db4ee177ec316038f01d37486cab4a1c4a90ad62d9da01f7be5aaa2e + checksum/config: 123553cb36ed2465fa6415be93afb6feb2cb91846d593998367b72fd3a18756a spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/rendered/manifests/traces-only/configmap-agent.yaml b/rendered/manifests/traces-only/configmap-agent.yaml index a53a3ff8f3..e1bad960a4 100644 --- a/rendered/manifests/traces-only/configmap-agent.yaml +++ b/rendered/manifests/traces-only/configmap-agent.yaml @@ -100,7 +100,7 @@ data: - action: insert key: k8s.node.name value: ${K8S_NODE_NAME} - - action: insert + - action: upsert key: k8s.cluster.name value: CHANGEME resource/add_agent_k8s: @@ -127,7 +127,7 @@ data: detectors: - env - system - override: false + override: true timeout: 10s receivers: jaeger: @@ -168,9 +168,9 @@ data: processors: - memory_limiter - batch - - resource - resource/add_agent_k8s - resourcedetection + - resource receivers: - prometheus/agent traces: @@ -180,8 +180,8 @@ data: - memory_limiter - k8sattributes - batch - - resource - resourcedetection + - resource receivers: - otlp - jaeger diff --git a/rendered/manifests/traces-only/daemonset.yaml b/rendered/manifests/traces-only/daemonset.yaml index 7a60f37f61..c96194aa79 100644 --- a/rendered/manifests/traces-only/daemonset.yaml +++ b/rendered/manifests/traces-only/daemonset.yaml @@ -27,7 +27,7 @@ spec: app: splunk-otel-collector release: default annotations: - checksum/config: 4cfe57d697bf6a8d85972b26473a2284203c120468cb8f702735a2135b90149f + checksum/config: 890c31871e984d10e0c16706b816212920bb05cfb0241990414d00f10f0e5801 kubectl.kubernetes.io/default-container: otel-collector spec: hostNetwork: true