feat: add the Elastic Collector distribution (#31)

* add the Elastic Collector distribution * feat: add Elastic distro to Helm deployment * remove attributes processors for k8s config * chore: bump elastic-agent snapshot version * docs: add k8s configuration deployment command * chore: remove otel header from Elastic config * use Bearer for apm authorization header * Update kubernetes/elastic-helm/values.yaml Co-authored-by: Chris Mark <[email protected]> * Update kubernetes/elastic-helm/configmap-elastic.yaml Co-authored-by: Chris Mark <[email protected]> --------- Co-authored-by: Chris Mark <[email protected]>
elastic · Jul 3, 2024 · bffcf2c · bffcf2c
1 parent af241f7
commit bffcf2c
Show file tree

Hide file tree

Showing 9 changed files with 152 additions and 59 deletions.
diff --git a/.env.override b/.env.override
@@ -10,3 +10,10 @@ IMAGE_NAME=ghcr.io/elastic/opentelemetry-demo
 # *********************
 AD_SERVICE_DOCKERFILE=./src/adservice/Dockerfile.elastic
 FRAUD_SERVICE_DOCKERFILE=./src/frauddetectionservice/Dockerfile.elastic
+
+# *********************
+# Elastic Collector
+# *********************
+COLLECTOR_CONTRIB_IMAGE=docker.elastic.co/beats/elastic-agent:8.15.0-7b611e39-SNAPSHOT
+OTEL_COLLECTOR_CONFIG=./src/otelcollector/otelcol-elastic-config.yaml
+OTEL_COLLECTOR_CONFIG_EXTRAS=./src/otelcollector/otelcol-elastic-config-extras.yaml
diff --git a/.github/README.md b/.github/README.md
@@ -7,15 +7,17 @@ The following guide describes how to setup the OpenTelemetry demo with Elastic O
 - The .NET agent within the [Cart service](../src/cartservice/src/Directory.Build.props) has been replaced with the Elastic distribution of the OpenTelemetry .NET Agent. You can find more information about the Elastic distribution in [this blog post](https://www.elastic.co/observability-labs/blog/elastic-opentelemetry-distribution-dotnet-applications).
 - The Elastic distribution of the OpenTelemetry Node.js Agent has replaced the OpenTelemetry Node.js agent in the [Payment service](../src/paymentservice/package.json). Additional details about the Elastic distribution are available in [this blog post](https://www.elastic.co/observability-labs/blog/elastic-opentelemetry-distribution-node-js).
 
+Additionally, the OpenTelemetry Contrib collector has also been changed to the [Elastic OpenTelemetry Collector distribution](https://github.com/elastic/elastic-agent/blob/main/internal/pkg/otel/README.md). This ensures a more integrated and optimized experience with Elastic Observability.
+
 ## Docker compose
 
 1. Start a free trial on [Elastic Cloud](https://cloud.elastic.co/) and copy the `endpoint` and `secretToken` from the Elastic APM setup instructions in your Kibana.
-1. Open the file `src/otelcollector/otelcol-config-extras.yml` in an editor and replace the following two placeholders:
+1. Open the file `src/otelcollector/otelcol-elastic-config-extras.yaml` in an editor and replace the following two placeholders:
    - `YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX`: your Elastic APM endpoint (*without* `https://` prefix) that *must* also include the port (example: `1234567.apm.us-west2.gcp.elastic-cloud.com:443`).
    - `YOUR_APM_SECRET_TOKEN`: your Elastic APM secret token.
 1. Start the demo with the following command from the repository's root directory:
    ```
-   docker-compose up -d
+   make start
    ```
 
 ## Kubernetes
@@ -46,6 +48,9 @@ The following guide describes how to setup the OpenTelemetry demo with Elastic O
    # !(when an older helm open-telemetry repo exists) update the open-telemetry helm repo
    helm repo update open-telemetry
 
+   # deploy the configuration for the Elastic OpenTelemetry collector distribution
+   kubectl apply -f configmap-elastic.yaml
+
    # deploy the demo through helm install
    helm install -f values.yaml my-otel-demo open-telemetry/opentelemetry-demo
    ```

diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml
@@ -582,7 +582,8 @@ services:
         limits:
           memory: 200M
     restart: unless-stopped
-    command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ]
+    command: ["otel", "--config", "/etc/otelcol-config.yml", "--config", "/etc/otelcol-config-extras.yml" ]
+    entrypoint: [ /usr/share/elastic-agent/elastic-agent ]
     user: 0:0
     volumes:
       - ${DOCKER_SOCK}:/var/run/docker.sock:ro

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -692,7 +692,8 @@ services:
         limits:
           memory: 200M
     restart: unless-stopped
-    command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ]
+    command: ["otel", "--config", "/etc/otelcol-config.yml", "--config", "/etc/otelcol-config-extras.yml" ]
+    entrypoint: [ /usr/share/elastic-agent/elastic-agent ]
     user: 0:0
     volumes:
       - ${DOCKER_SOCK}:/var/run/docker.sock:ro

diff --git a/kubernetes/elastic-helm/configmap-elastic.yaml b/kubernetes/elastic-helm/configmap-elastic.yaml
@@ -0,0 +1,71 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elastic-otelcol-agent
+  namespace: default
+  labels:
+    app.kubernetes.io/name: otelcol
+
+data:
+  relay: |
+    connectors:
+      spanmetrics: {}
+    exporters:
+      debug: {}
+      otlp/elastic:
+        endpoint: ${env:ELASTIC_APM_ENDPOINT}
+        compression: none
+        headers:
+          Authorization: Bearer ${ELASTIC_APM_SECRET_TOKEN}
+    extensions:
+    processors:
+      batch: {}
+    receivers:
+      httpcheck/frontendproxy:
+        targets:
+        - endpoint: http://example-frontendproxy:8080
+      otlp:
+        protocols:
+          grpc:
+            endpoint: ${env:MY_POD_IP}:4317
+          http:
+            cors:
+              allowed_origins:
+              - http://*
+              - https://*
+            endpoint: ${env:MY_POD_IP}:4318
+    service:
+      extensions:
+      pipelines:
+        logs:
+          exporters:
+          - debug
+          - otlp/elastic
+          processors:
+          - batch
+          receivers:
+          - otlp
+        metrics:
+          exporters:
+          - otlp/elastic
+          - debug
+          processors:
+          - batch
+          receivers:
+          - httpcheck/frontendproxy
+          - otlp
+          - spanmetrics
+        traces:
+          exporters:
+          - otlp/elastic
+          - debug
+          - spanmetrics
+          processors:
+          - batch
+          receivers:
+          - otlp
+      telemetry:
+        metrics:
+          address: ${env:MY_POD_IP}:8888
+
diff --git a/kubernetes/elastic-helm/values.yaml b/kubernetes/elastic-helm/values.yaml
@@ -4,6 +4,12 @@ default:
     tag: 1.10.1
 
 opentelemetry-collector:
+  image:
+    repository: docker.elastic.co/beats/elastic-agent
+    tag: 8.15.0-SNAPSHOT
+  command:
+    name: "/usr/share/elastic-agent/elastic-agent"
+    extraArgs: ["otel", "-c", "/etc/elastic-agent/otel.yaml"]
   mode: "deployment"
   presets:
     kubernetesAttributes:
@@ -24,38 +30,23 @@ opentelemetry-collector:
         secretKeyRef:
           name: elastic-secret
           key: elastic_apm_secret_token
-  config:
-    exporters:
-      otlp/elastic:
-        endpoint: ${ELASTIC_APM_ENDPOINT}
-        compression: none
-        headers:
-          Authorization: Bearer ${ELASTIC_APM_SECRET_TOKEN}
-    receivers:
-      otlp:
-        protocols:
-          grpc:
-            endpoint: ${MY_POD_IP}:4317
-          http:
-            cors:
-              allowed_origins:
-              - http://*
-              - https://*
-            endpoint: ${MY_POD_IP}:4318
-    service:
-      pipelines:
-        traces:
-          receivers: [otlp]
-          processors: [batch]
-          exporters: [otlp/elastic]
-        metrics:
-          receivers: [otlp]
-          processors: [batch]
-          exporters: [otlp/elastic]
-        logs:
-          receivers: [otlp]
-          processors: [batch]
-          exporters: [otlp/elastic]
+
+  configMap:
+    create: false
+    existingName: elastic-otelcol-agent
+
+  livenessProbe:
+    httpGet:
+      port: 8888
+      path: metrics
+  readinessProbe:
+    httpGet:
+      port: 8888
+      path: metrics
+
+opensearch:
+  enabled: false
+
 grafana:
   enabled: false
 

diff --git a/src/otelcollector/otelcol-config-extras.yml b/src/otelcollector/otelcol-config-extras.yml
@@ -16,26 +16,3 @@
 #    pipelines:
 #      traces:
 #        exporters: [spanmetrics, otlphttp/example]
-
-exporters:
-  otlp/elastic:
-    # !!! Elastic APM https endpoint WITHOUT the "https://" prefix
-    endpoint: "YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX"
-    compression: none
-    headers:
-      Authorization: "Bearer YOUR_APM_SECRET_TOKEN"
-
-service:
-  pipelines:
-    traces:
-      receivers: [otlp]
-      processors: [batch]
-      exporters: [spanmetrics, otlp/elastic]
-    metrics:
-      receivers: [otlp, spanmetrics]
-      processors: [batch]
-      exporters: [otlp/elastic]
-    logs:
-      receivers: [otlp]
-      processors: [batch]
-      exporters: [otlp/elastic]
diff --git a/src/otelcollector/otelcol-elastic-config-extras.yaml b/src/otelcollector/otelcol-elastic-config-extras.yaml
@@ -0,0 +1,22 @@
+exporters:
+  otlp/elastic:
+    # !!! Elastic APM https endpoint WITHOUT the "https://" prefix
+    endpoint: "YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX"
+    compression: none
+    headers:
+      Authorization: "Bearer YOUR_APM_SECRET_TOKEN"
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [spanmetrics, otlp/elastic]
+    metrics:
+      receivers: [otlp, spanmetrics]
+      processors: [batch]
+      exporters: [otlp/elastic]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/elastic]
diff --git a/src/otelcollector/otelcol-elastic-config.yaml b/src/otelcollector/otelcol-elastic-config.yaml
@@ -0,0 +1,18 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+        cors:
+          allowed_origins:
+            - "http://*"
+            - "https://*"
+
+exporters:
+  debug:
+
+processors:
+  batch:
+
+connectors:
+  spanmetrics: