Do not send clusterReceiver metrics through gateway (#491)

* Do not send clusterReceiver metrics through gateway

Reasoning for this change:
- Currently platform metrics are not being sent through the gateway even if the gateway is enabled, and k8s event collection is broken if gateway is enabled. It has to be consistent: send everything from cluster receiver through gateway or don't send anything.
- Sending telemetry though gateway from one pod looks like doesn't provide any real value except for adding overhead:
  - If gateway is assigned on some special nodes that have internet connection, the same way cluster receiver can be assigned to the same pods.
  - Cluster receiver has to talk a lot to k8s API, that part cannot be moved to the gateway as it's done for agents. Gateway also talks a lot to k8s API, so it makes sense to collocate them together.

* Update CHANGELOG.md

CHANGELOG.md

@@ -4,6 +4,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### Changed
+
+- Do not send clusterReceiver metrics through gateway (#491)
+
 ## [0.57.0] - 2022-08-05
 
 ### Changed

helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl

@@ -3,7 +3,6 @@ Config for the otel-collector k8s cluster receiver deployment.
 The values can be overridden in .Values.clusterReceiver.config
 */}}
 {{- define "splunk-otel-collector.clusterReceiverConfig" -}}
-{{ $gateway := fromYaml (include "splunk-otel-collector.gateway" .) -}}
 {{ $clusterReceiver := fromYaml (include "splunk-otel-collector.clusterReceiver" .) -}}
 extensions:
   health_check:
@@ -153,13 +152,8 @@ processors:
 exporters:
   {{- if or (eq (include "splunk-otel-collector.o11yMetricsEnabled" $) "true") (eq (include "splunk-otel-collector.o11yInfraMonEventsEnabled" .) "true") }}
   signalfx:
-    {{- if $gateway.enabled }}
-    ingest_url: http://{{ include "splunk-otel-collector.fullname" . }}:9943
-    api_url: http://{{ include "splunk-otel-collector.fullname" . }}:6060
-    {{- else }}
     ingest_url: {{ include "splunk-otel-collector.o11yIngestUrl" . }}
     api_url: {{ include "splunk-otel-collector.o11yApiUrl" . }}
-    {{- end }}
     access_token: ${SPLUNK_OBSERVABILITY_ACCESS_TOKEN}
     timeout: 10s
   {{- end }}
@@ -249,16 +243,12 @@ service:
         - resource/add_environment
         {{- end }}
       exporters:
-        {{- if $gateway.enabled }}
-        - otlp
-        {{- else }}
         {{- if (eq (include "splunk-otel-collector.o11yLogsEnabled" .) "true") }}
         - splunk_hec/o11y
         {{- end }}
         {{- if (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") }}
         - splunk_hec/platform_logs
         {{- end }}
-        {{- end }}
     {{- end }}
 
     {{- if eq (include "splunk-otel-collector.o11yInfraMonEventsEnabled" .) "true" }}

rendered/manifests/eks-fargate/configmap-cluster-receiver.yaml

@@ -19,8 +19,8 @@ data:
     exporters:
       signalfx:
         access_token: ${SPLUNK_OBSERVABILITY_ACCESS_TOKEN}
-        api_url: http://default-splunk-otel-collector:6060
-        ingest_url: http://default-splunk-otel-collector:9943
+        api_url: https://api.CHANGEME.signalfx.com
+        ingest_url: https://ingest.CHANGEME.signalfx.com
         timeout: 10s
     extensions:
       health_check: null

rendered/manifests/eks-fargate/deployment-cluster-receiver.yaml

@@ -32,7 +32,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: ec22f19255dde9193fc3aaa545cede469187fbf493a503f30d366bef83a648e3
+        checksum/config: 57ea7798d82ef59176be54fdbc6c1ecfa393a6e9a0615e533de11b6f936d4cea
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector: