diff --git a/.env.override b/.env.override index 3c6c909469..ad0661922a 100644 --- a/.env.override +++ b/.env.override @@ -10,3 +10,10 @@ IMAGE_NAME=ghcr.io/elastic/opentelemetry-demo # ********************* AD_SERVICE_DOCKERFILE=./src/adservice/Dockerfile.elastic FRAUD_SERVICE_DOCKERFILE=./src/frauddetectionservice/Dockerfile.elastic + +# ********************* +# Elastic Collector +# ********************* +COLLECTOR_CONTRIB_IMAGE=docker.elastic.co/beats/elastic-agent:8.15.0-7b611e39-SNAPSHOT +OTEL_COLLECTOR_CONFIG=./src/otelcollector/otelcol-elastic-config.yaml +OTEL_COLLECTOR_CONFIG_EXTRAS=./src/otelcollector/otelcol-elastic-config-extras.yaml diff --git a/.github/README.md b/.github/README.md index 37d5f8e4b3..4958b2bd73 100644 --- a/.github/README.md +++ b/.github/README.md @@ -7,15 +7,17 @@ The following guide describes how to setup the OpenTelemetry demo with Elastic O - The .NET agent within the [Cart service](../src/cartservice/src/Directory.Build.props) has been replaced with the Elastic distribution of the OpenTelemetry .NET Agent. You can find more information about the Elastic distribution in [this blog post](https://www.elastic.co/observability-labs/blog/elastic-opentelemetry-distribution-dotnet-applications). - The Elastic distribution of the OpenTelemetry Node.js Agent has replaced the OpenTelemetry Node.js agent in the [Payment service](../src/paymentservice/package.json). Additional details about the Elastic distribution are available in [this blog post](https://www.elastic.co/observability-labs/blog/elastic-opentelemetry-distribution-node-js). +Additionally, the OpenTelemetry Contrib collector has also been changed to the [Elastic OpenTelemetry Collector distribution](https://github.com/elastic/elastic-agent/blob/main/internal/pkg/otel/README.md). This ensures a more integrated and optimized experience with Elastic Observability. + ## Docker compose 1. Start a free trial on [Elastic Cloud](https://cloud.elastic.co/) and copy the `endpoint` and `secretToken` from the Elastic APM setup instructions in your Kibana. -1. Open the file `src/otelcollector/otelcol-config-extras.yml` in an editor and replace the following two placeholders: +1. Open the file `src/otelcollector/otelcol-elastic-config-extras.yaml` in an editor and replace the following two placeholders: - `YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX`: your Elastic APM endpoint (*without* `https://` prefix) that *must* also include the port (example: `1234567.apm.us-west2.gcp.elastic-cloud.com:443`). - `YOUR_APM_SECRET_TOKEN`: your Elastic APM secret token. 1. Start the demo with the following command from the repository's root directory: ``` - docker-compose up -d + make start ``` ## Kubernetes @@ -46,6 +48,9 @@ The following guide describes how to setup the OpenTelemetry demo with Elastic O # !(when an older helm open-telemetry repo exists) update the open-telemetry helm repo helm repo update open-telemetry + # deploy the configuration for the Elastic OpenTelemetry collector distribution + kubectl apply -f configmap-elastic.yaml + # deploy the demo through helm install helm install -f values.yaml my-otel-demo open-telemetry/opentelemetry-demo ``` diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml index b4925dd742..4282259c07 100644 --- a/docker-compose.minimal.yml +++ b/docker-compose.minimal.yml @@ -582,7 +582,8 @@ services: limits: memory: 200M restart: unless-stopped - command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ] + command: ["otel", "--config", "/etc/otelcol-config.yml", "--config", "/etc/otelcol-config-extras.yml" ] + entrypoint: [ /usr/share/elastic-agent/elastic-agent ] user: 0:0 volumes: - ${DOCKER_SOCK}:/var/run/docker.sock:ro diff --git a/docker-compose.yml b/docker-compose.yml index 1f54b8d960..c53a747ece 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -692,7 +692,8 @@ services: limits: memory: 200M restart: unless-stopped - command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ] + command: ["otel", "--config", "/etc/otelcol-config.yml", "--config", "/etc/otelcol-config-extras.yml" ] + entrypoint: [ /usr/share/elastic-agent/elastic-agent ] user: 0:0 volumes: - ${DOCKER_SOCK}:/var/run/docker.sock:ro diff --git a/kubernetes/elastic-helm/configmap-elastic.yaml b/kubernetes/elastic-helm/configmap-elastic.yaml new file mode 100644 index 0000000000..5799301baa --- /dev/null +++ b/kubernetes/elastic-helm/configmap-elastic.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastic-otelcol-agent + namespace: default + labels: + app.kubernetes.io/name: otelcol + +data: + relay: | + connectors: + spanmetrics: {} + exporters: + debug: {} + otlp/elastic: + endpoint: ${env:ELASTIC_APM_ENDPOINT} + compression: none + headers: + Authorization: Bearer ${ELASTIC_APM_SECRET_TOKEN} + extensions: + processors: + batch: {} + receivers: + httpcheck/frontendproxy: + targets: + - endpoint: http://example-frontendproxy:8080 + otlp: + protocols: + grpc: + endpoint: ${env:MY_POD_IP}:4317 + http: + cors: + allowed_origins: + - http://* + - https://* + endpoint: ${env:MY_POD_IP}:4318 + service: + extensions: + pipelines: + logs: + exporters: + - debug + - otlp/elastic + processors: + - batch + receivers: + - otlp + metrics: + exporters: + - otlp/elastic + - debug + processors: + - batch + receivers: + - httpcheck/frontendproxy + - otlp + - spanmetrics + traces: + exporters: + - otlp/elastic + - debug + - spanmetrics + processors: + - batch + receivers: + - otlp + telemetry: + metrics: + address: ${env:MY_POD_IP}:8888 + diff --git a/kubernetes/elastic-helm/values.yaml b/kubernetes/elastic-helm/values.yaml index 616bbda572..d490086495 100644 --- a/kubernetes/elastic-helm/values.yaml +++ b/kubernetes/elastic-helm/values.yaml @@ -4,6 +4,12 @@ default: tag: 1.10.1 opentelemetry-collector: + image: + repository: docker.elastic.co/beats/elastic-agent + tag: 8.15.0-SNAPSHOT + command: + name: "/usr/share/elastic-agent/elastic-agent" + extraArgs: ["otel", "-c", "/etc/elastic-agent/otel.yaml"] mode: "deployment" presets: kubernetesAttributes: @@ -24,38 +30,23 @@ opentelemetry-collector: secretKeyRef: name: elastic-secret key: elastic_apm_secret_token - config: - exporters: - otlp/elastic: - endpoint: ${ELASTIC_APM_ENDPOINT} - compression: none - headers: - Authorization: Bearer ${ELASTIC_APM_SECRET_TOKEN} - receivers: - otlp: - protocols: - grpc: - endpoint: ${MY_POD_IP}:4317 - http: - cors: - allowed_origins: - - http://* - - https://* - endpoint: ${MY_POD_IP}:4318 - service: - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [otlp/elastic] - metrics: - receivers: [otlp] - processors: [batch] - exporters: [otlp/elastic] - logs: - receivers: [otlp] - processors: [batch] - exporters: [otlp/elastic] + + configMap: + create: false + existingName: elastic-otelcol-agent + + livenessProbe: + httpGet: + port: 8888 + path: metrics + readinessProbe: + httpGet: + port: 8888 + path: metrics + +opensearch: + enabled: false + grafana: enabled: false diff --git a/src/otelcollector/otelcol-config-extras.yml b/src/otelcollector/otelcol-config-extras.yml index ce2395d485..73dd011b2c 100644 --- a/src/otelcollector/otelcol-config-extras.yml +++ b/src/otelcollector/otelcol-config-extras.yml @@ -16,26 +16,3 @@ # pipelines: # traces: # exporters: [spanmetrics, otlphttp/example] - -exporters: - otlp/elastic: - # !!! Elastic APM https endpoint WITHOUT the "https://" prefix - endpoint: "YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX" - compression: none - headers: - Authorization: "Bearer YOUR_APM_SECRET_TOKEN" - -service: - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [spanmetrics, otlp/elastic] - metrics: - receivers: [otlp, spanmetrics] - processors: [batch] - exporters: [otlp/elastic] - logs: - receivers: [otlp] - processors: [batch] - exporters: [otlp/elastic] diff --git a/src/otelcollector/otelcol-elastic-config-extras.yaml b/src/otelcollector/otelcol-elastic-config-extras.yaml new file mode 100644 index 0000000000..530d47ad15 --- /dev/null +++ b/src/otelcollector/otelcol-elastic-config-extras.yaml @@ -0,0 +1,22 @@ +exporters: + otlp/elastic: + # !!! Elastic APM https endpoint WITHOUT the "https://" prefix + endpoint: "YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX" + compression: none + headers: + Authorization: "Bearer YOUR_APM_SECRET_TOKEN" + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [spanmetrics, otlp/elastic] + metrics: + receivers: [otlp, spanmetrics] + processors: [batch] + exporters: [otlp/elastic] + logs: + receivers: [otlp] + processors: [batch] + exporters: [otlp/elastic] diff --git a/src/otelcollector/otelcol-elastic-config.yaml b/src/otelcollector/otelcol-elastic-config.yaml new file mode 100644 index 0000000000..63ec2b97bd --- /dev/null +++ b/src/otelcollector/otelcol-elastic-config.yaml @@ -0,0 +1,18 @@ +receivers: + otlp: + protocols: + grpc: + http: + cors: + allowed_origins: + - "http://*" + - "https://*" + +exporters: + debug: + +processors: + batch: + +connectors: + spanmetrics: