feat: add ECS mode for Helm deployment (#33)

* remove unused daemonset.yaml file * feat: add elasticsearch k8s secrets * docs: add elasticsearch endpoint secrets * set daemonset as default Helm deployment method * Revert "set daemonset as default Helm deployment method" This reverts commit 80612a5. * Revert "remove unused daemonset.yaml file" This reverts commit 8de4003. * decouple elasticsearch exporter from deployment mode * feat: add configmap for daemonset deployment * rollback filelog pods exclusion * feat: add deployment.environment attribute * remove extra volumes in deployment mode * remove security context from deployment mode * add security context for daemonset deployment
elastic · Jul 26, 2024 · f93eabe · f93eabe
1 parent aba39ea
commit f93eabe
Show file tree

Hide file tree

Showing 6 changed files with 322 additions and 45 deletions.
diff --git a/.env.override b/.env.override
@@ -14,6 +14,6 @@ FRAUD_SERVICE_DOCKERFILE=./src/frauddetectionservice/Dockerfile.elastic
 # *********************
 # Elastic Collector
 # *********************
-COLLECTOR_CONTRIB_IMAGE=docker.elastic.co/beats/elastic-agent:8.15.0-7b611e39-SNAPSHOT
+COLLECTOR_CONTRIB_IMAGE=docker.elastic.co/beats/elastic-agent:8.15.0-SNAPSHOT
 OTEL_COLLECTOR_CONFIG=./src/otelcollector/otelcol-elastic-config.yaml
 OTEL_COLLECTOR_CONFIG_EXTRAS=./src/otelcollector/otelcol-elastic-config-extras.yaml
diff --git a/.github/README.md b/.github/README.md
@@ -49,21 +49,38 @@ Additionally, the OpenTelemetry Contrib collector has also been changed to the [
    helm repo update open-telemetry
 
    # deploy the configuration for the Elastic OpenTelemetry collector distribution
-   kubectl apply -f configmap-elastic.yaml
+   kubectl apply -f configmap-deployment.yaml
 
    # deploy the demo through helm install
-   helm install -f values.yaml my-otel-demo open-telemetry/opentelemetry-demo
+   helm install -f deployment.yaml my-otel-demo open-telemetry/opentelemetry-demo
    ```
 
 #### Kubernetes monitoring
 
 This demo already enables cluster level metrics collection with `clusterMetrics` and
 Kubernetes events collection with `kubernetesEvents`.
 
-In order to add Node level metrics collection and autodiscovery for Redis Pods
-we can run an additional Otel collector Daemonset with the following:
+In order to add Node level metrics collection we can run an additional Otel collector Daemonset with the following:
 
-`helm install daemonset open-telemetry/opentelemetry-collector --values daemonset.yaml`
+1. Create a secret in Kubernetes with the following command.
+   ```
+   kubectl create secret generic elastic-secret-ds \
+     --from-literal=elastic_endpoint='YOUR_ELASTICSEARCH_ENDPOINT' \
+     --from-literal=elastic_api_key='YOUR_ELASTICSEARCH_API_KEY'
+   ```
+   Don't forget to replace
+   - `YOUR_ELASTICSEARCH_ENDPOINT`: your Elasticsearch endpoint (example: `1234567.us-west2.gcp.elastic-cloud.com:443`).
+   - `YOUR_ELASTICSEARCH_API_KEY`: your Elasticsearch API Key
+
+2. Execute the following command to deploy the OpenTelemetry Collector to your Kubernetes cluster:
+
+```
+# deploy the configuration for the Elastic OpenTelemetry collector distribution
+kubectl apply -f configmap-daemonset.yaml
+
+# deploy the Elastic OpenTelemetry collector distribution through helm install
+helm install otel-daemonset open-telemetry/opentelemetry-collector --values daemonset.yaml
+```
 
 ## Explore and analyze the data With Elastic
 

diff --git a/kubernetes/elastic-helm/configmap-daemonset.yaml b/kubernetes/elastic-helm/configmap-daemonset.yaml
@@ -0,0 +1,265 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elastic-otelcol-agent-ds
+  namespace: default
+  labels:
+    app.kubernetes.io/name: otelcol
+
+data:
+  relay: |
+    exporters:
+      debug:
+      elasticsearch:
+        endpoints:
+        - ${env:ELASTIC_ENDPOINT}
+        api_key: ${env:ELASTIC_API_KEY}
+        logs_dynamic_index:
+          enabled: true
+        metrics_dynamic_index:
+          enabled: true
+        mapping:
+          mode: ecs
+    processors:
+      batch: {}
+      elasticinframetrics:
+        add_system_metrics: true
+        add_k8s_metrics: true
+      resourcedetection/eks:
+        detectors: [env, eks]
+        timeout: 15s
+        override: true
+        eks:
+          resource_attributes:
+            k8s.cluster.name:
+              enabled: true
+      resourcedetection/gcp:
+        detectors: [env, gcp]
+        timeout: 2s
+        override: false
+      resource/k8s:
+        attributes:
+          - key: service.name
+            from_attribute: app.label.component
+            action: insert
+      attributes/k8s_logs_dataset:
+        actions:
+          - key: data_stream.dataset
+            value: "kubernetes.container_logs"
+            action: upsert
+      attributes/dataset:
+        actions:
+          - key: event.dataset
+            from_attribute: data_stream.dataset
+            action: upsert
+      resource/cloud:
+        attributes:
+          - key: cloud.instance.id
+            from_attribute: host.id
+            action: insert
+      resource/demo:
+        attributes:
+          - key: deployment.environment
+            value: "opentelemetry-demo"
+            action: upsert
+      resource/process:
+        attributes:
+          - key: process.executable.name
+            action: delete
+          - key: process.executable.path
+            action: delete
+      resourcedetection/system:
+        detectors: ["system", "ec2"]
+        system:
+          hostname_sources: [ "os" ]
+          resource_attributes:
+            host.name:
+              enabled: true
+            host.id:
+              enabled: false
+            host.arch:
+              enabled: true
+            host.ip:
+              enabled: true
+            host.mac:
+              enabled: true
+            host.cpu.vendor.id:
+              enabled: true
+            host.cpu.family:
+              enabled: true
+            host.cpu.model.id:
+              enabled: true
+            host.cpu.model.name:
+              enabled: true
+            host.cpu.stepping:
+              enabled: true
+            host.cpu.cache.l2.size:
+              enabled: true
+            os.description:
+              enabled: true
+            os.type:
+              enabled: true
+        ec2:
+          resource_attributes:
+            host.name:
+              enabled: false
+            host.id:
+              enabled: true
+      k8sattributes:
+        filter:
+          node_from_env_var: K8S_NODE_NAME
+        passthrough: false
+        pod_association:
+          - sources:
+              - from: resource_attribute
+                name: k8s.pod.ip
+          - sources:
+              - from: resource_attribute
+                name: k8s.pod.uid
+          - sources:
+              - from: connection
+        extract:
+          metadata:
+            - "k8s.namespace.name"
+            - "k8s.deployment.name"
+            - "k8s.statefulset.name"
+            - "k8s.daemonset.name"
+            - "k8s.cronjob.name"
+            - "k8s.job.name"
+            - "k8s.node.name"
+            - "k8s.pod.name"
+            - "k8s.pod.uid"
+            - "k8s.pod.start_time"
+          labels:
+            - tag_name: app.label.component
+              key: app.kubernetes.io/component
+              from: pod
+    receivers:
+      filelog:
+        retry_on_failure:
+          enabled: true
+        start_at: end
+        exclude:
+        # exlude collector logs
+        - /var/log/pods/default_otel-daemonset-opentelemetry-collector-agent*_*/opentelemetry-collector/*.log
+        include:
+        - /var/log/pods/*/*/*.log
+        include_file_name: false
+        include_file_path: true
+        operators:
+        - id: container-parser
+          type: container
+      hostmetrics:
+        collection_interval: 10s
+        root_path: /hostfs
+        scrapers:
+          cpu:
+            metrics:
+              system.cpu.utilization:
+                enabled: true
+              system.cpu.logical.count:
+                enabled: true
+          memory:
+            metrics:
+              system.memory.utilization:
+                enabled: true
+          process:
+            mute_process_exe_error: true
+            mute_process_io_error: true
+            mute_process_user_error: true
+            metrics:
+              process.threads:
+                enabled: true
+              process.open_file_descriptors:
+                enabled: true
+              process.memory.utilization:
+                enabled: true
+              process.disk.operations:
+                enabled: true
+          network:
+          processes:
+          load:
+          disk:
+          filesystem:
+            exclude_mount_points:
+              mount_points:
+                - /dev/*
+                - /proc/*
+                - /sys/*
+                - /run/k3s/containerd/*
+                - /var/lib/docker/*
+                - /var/lib/kubelet/*
+                - /snap/*
+              match_type: regexp
+            exclude_fs_types:
+              fs_types:
+                - autofs
+                - binfmt_misc
+                - bpf
+                - cgroup2
+                - configfs
+                - debugfs
+                - devpts
+                - devtmpfs
+                - fusectl
+                - hugetlbfs
+                - iso9660
+                - mqueue
+                - nsfs
+                - overlay
+                - proc
+                - procfs
+                - pstore
+                - rpc_pipefs
+                - securityfs
+                - selinuxfs
+                - squashfs
+                - sysfs
+                - tracefs
+              match_type: strict
+      kubeletstats:
+        auth_type: serviceAccount
+        collection_interval: 20s
+        endpoint: ${env:K8S_NODE_NAME}:10250
+        node: '${env:K8S_NODE_NAME}'
+        # Required to work for all CSPs without an issue
+        insecure_skip_verify: true
+        k8s_api_config:
+          auth_type: serviceAccount
+        metrics:
+          k8s.pod.cpu.node.utilization:
+            enabled: true
+          k8s.container.cpu_limit_utilization:
+            enabled: true
+          k8s.pod.cpu_limit_utilization:
+            enabled: true
+          k8s.container.cpu_request_utilization:
+            enabled: true
+          k8s.container.memory_limit_utilization:
+            enabled: true
+          k8s.pod.memory_limit_utilization:
+            enabled: true
+          k8s.container.memory_request_utilization:
+            enabled: true
+          k8s.node.uptime:
+            enabled: true
+          k8s.node.cpu.usage:
+            enabled: true
+          k8s.pod.cpu.usage:
+            enabled: true
+        extra_metadata_labels:
+          - container.id
+    service:
+      pipelines:
+        logs:
+          receivers: [filelog]
+          processors: [batch, k8sattributes, resourcedetection/system, resourcedetection/eks, resourcedetection/gcp, resource/demo, resource/k8s, resource/cloud, attributes/k8s_logs_dataset]
+          exporters: [debug, elasticsearch]
+        metrics:
+          receivers: [hostmetrics, kubeletstats]
+          processors: [batch, k8sattributes, elasticinframetrics, resourcedetection/system, resource/demo, resourcedetection/eks, resourcedetection/gcp, resource/k8s, resource/cloud, attributes/dataset, resource/process]
+          exporters: [debug, elasticsearch]
+      telemetry:
+        metrics:
+          address: ${env:MY_POD_IP}:8888
diff --git a/...netes/elastic-helm/configmap-elastic.yaml → ...es/elastic-helm/configmap-deployment.yaml b/...netes/elastic-helm/configmap-elastic.yaml → ...es/elastic-helm/configmap-deployment.yaml
@@ -49,6 +49,7 @@ data:
           - otlp/elastic
           processors:
           - batch
+          - resource
           receivers:
           - otlp
         metrics:
@@ -57,6 +58,7 @@ data:
           - debug
           processors:
           - batch
+          - resource
           receivers:
           - httpcheck/frontendproxy
           - otlp