From d0320392cc74657e8e9bcfff5793b6b25b67d21a Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Thu, 24 Aug 2023 15:43:14 +0100 Subject: [PATCH] feat: add remote write support to the Open Telemetry configuration Signed-off-by: Jeromy Cannon --- .../config-files/otel-collector-config.yaml | 26 ++++++++++++++++--- .../templates/services/network-node-svc.yaml | 4 +++ .../templates/sidecars/_otel-collector.tpl | 5 +++- .../telemetry/prometheus-svc-monitor.yaml | 2 ++ charts/hedera-network/values.yaml | 5 +++- dev/scripts/telemetry.sh | 2 +- dev/telemetry/prometheus/prometheus.yaml | 1 + 7 files changed, 38 insertions(+), 7 deletions(-) diff --git a/charts/hedera-network/config-files/otel-collector-config.yaml b/charts/hedera-network/config-files/otel-collector-config.yaml index 764e6256d..791127bd0 100644 --- a/charts/hedera-network/config-files/otel-collector-config.yaml +++ b/charts/hedera-network/config-files/otel-collector-config.yaml @@ -18,9 +18,19 @@ exporters: prometheus: endpoint: "0.0.0.0:8889" const_labels: - label1: "{{ .otelDefaults.nameOverride }}-label1" + source: p-{{ default "otel-collector" .otelDefaults.nameOverride }} # PromQL: {source="p-otel-collector"} + {{- if eq .otelDefaults.prometheusRemoteWrite.enable "true" }} + prometheusremotewrite: + endpoint: "{{ .otelDefaults.prometheusRemoteWrite.endpoint }}" + tls: + insecure: true + external_labels: + source: prw-{{ default "otel-collector" .otelDefaults.nameOverride }} # PromQL: {source="prw-otel-collector"} + {{- end }} +# logging: +# verbosity: detailed +# sampling_initial: 5 - logging: # jaeger: # endpoint: jaeger-all-in-one:14250 @@ -34,11 +44,14 @@ exporters: processors: batch: - extensions: health_check: service: +# telemetry: +# logs: +# level: "debug" +# development: true extensions: [health_check] pipelines: #traces: @@ -48,4 +61,9 @@ service: metrics: receivers: [otlp, hostmetrics] processors: [batch] - exporters: [logging, prometheus] + exporters: + - prometheus + {{- if eq .otelDefaults.prometheusRemoteWrite.enable "true" }} + - prometheusremotewrite + {{- end }} +# exporters: [logging, prometheus{{ if eq .otelDefaults.prometheusRemoteWrite.enable "true" }}, prometheusremotewrite{{ end }}] diff --git a/charts/hedera-network/templates/services/network-node-svc.yaml b/charts/hedera-network/templates/services/network-node-svc.yaml index d2d05585f..9757a9192 100644 --- a/charts/hedera-network/templates/services/network-node-svc.yaml +++ b/charts/hedera-network/templates/services/network-node-svc.yaml @@ -27,4 +27,8 @@ spec: protocol: TCP port: 8888 targetPort: 8888 + - name: prometheus + protocol: TCP + port: 8889 + targetPort: 8889 {{- end }} diff --git a/charts/hedera-network/templates/sidecars/_otel-collector.tpl b/charts/hedera-network/templates/sidecars/_otel-collector.tpl index e2db47db7..6935db96c 100644 --- a/charts/hedera-network/templates/sidecars/_otel-collector.tpl +++ b/charts/hedera-network/templates/sidecars/_otel-collector.tpl @@ -17,6 +17,9 @@ - name: otel-otlp containerPort: 4317 protocol: TCP + - name: prometheus + containerPort: 8889 + protocol: TCP {{- with default $defaults.livenessProbe $otel.livenessProbe }} livenessProbe: {{- toYaml . | nindent 4 }} @@ -27,7 +30,7 @@ {{- end }} volumeMounts: - name: otel-collector-volume - mountPath: /etc/otel-collector-config.yaml + mountPath: /etc/otelcol-contrib/config.yaml subPath: config.yaml #key in the configmap readOnly: true {{- with default $defaults.resources $otel.resources }} diff --git a/charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml b/charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml index e43b08209..241a3db43 100644 --- a/charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml +++ b/charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml @@ -12,4 +12,6 @@ spec: endpoints: - port: otel-metrics interval: 5s + - port: prometheus + interval: 5s {{- end }} diff --git a/charts/hedera-network/values.yaml b/charts/hedera-network/values.yaml index d7d86e5c3..8fca71216 100644 --- a/charts/hedera-network/values.yaml +++ b/charts/hedera-network/values.yaml @@ -157,7 +157,7 @@ defaults: resources: {} otelCollector: enable: "true" - nameOverride: "otel-collector" + nameOverride: "" image: registry: "docker.io" repository: "otel/opentelemetry-collector-contrib" @@ -172,6 +172,9 @@ defaults: path: / port: otel-health resources: {} + prometheusRemoteWrite: + enable: "false" + endpoint: "http://prometheus.default.svc:9090/api/v1/write" # This configures the minio tenant subchart # Reference for configuration: https://github.com/minio/operator/blob/master/helm/tenant/values.yaml diff --git a/dev/scripts/telemetry.sh b/dev/scripts/telemetry.sh index 3a2306ee6..82121b54d 100644 --- a/dev/scripts/telemetry.sh +++ b/dev/scripts/telemetry.sh @@ -81,7 +81,7 @@ function deploy-prometheus-example-app() { echo "PROMETHEUS_EXAMPLE_APP_YAML: ${PROMETHEUS_EXAMPLE_APP_YAML}" echo "-----------------------------------------------------------------------------------------------------" kubectl create -f "${PROMETHEUS_EXAMPLE_APP_YAML}" - kubectl wait --for=condition=Ready pods -l app=prometheus-example-app -n default --timeout 60 + kubectl wait --for=condition=Ready pods -l app=prometheus-example-app -n default --timeout 60s } function destroy-prometheus-example-app() { diff --git a/dev/telemetry/prometheus/prometheus.yaml b/dev/telemetry/prometheus/prometheus.yaml index ab66607c5..8bf4a42a0 100644 --- a/dev/telemetry/prometheus/prometheus.yaml +++ b/dev/telemetry/prometheus/prometheus.yaml @@ -11,6 +11,7 @@ spec: requests: memory: 400Mi enableAdminAPI: false + enableRemoteWriteReceiver: true --- apiVersion: v1 kind: Service