From 89fa6a5f0836b655dec43e0d5dd7ba99ebee305d Mon Sep 17 00:00:00 2001 From: Jacob Aronoff Date: Wed, 1 May 2024 17:56:38 -0400 Subject: [PATCH 01/36] Add presets --- .../templates/_config.tpl | 466 ++++++++++++++++++ .../templates/_helpers.tpl | 23 - .../templates/collector.yaml | 45 +- charts/opentelemetry-kube-stack/test.yaml | 1 + .../values.schema.json | 10 + charts/opentelemetry-kube-stack/values.yaml | 171 ++++++- 6 files changed, 688 insertions(+), 28 deletions(-) create mode 100644 charts/opentelemetry-kube-stack/templates/_config.tpl create mode 100644 charts/opentelemetry-kube-stack/test.yaml diff --git a/charts/opentelemetry-kube-stack/templates/_config.tpl b/charts/opentelemetry-kube-stack/templates/_config.tpl new file mode 100644 index 000000000..b644b3ff9 --- /dev/null +++ b/charts/opentelemetry-kube-stack/templates/_config.tpl @@ -0,0 +1,466 @@ +{{/* +Constructs the final config for the given collector + +This allows a user to supply a scrape_configs_file. This file is templated and loaded as a yaml array. +If a user has already supplied a prometheus receiver config, the file's config is appended. Finally, +the config is written as YAML. +*/}} +{{- define "opentelemetry-kube-stack.config" -}} +{{- $collector := .collector }} +{{- $config := .collector.config }} +{{- if .collector.scrape_configs_file }} +{{- $config = (include "opentelemetry-kube-stack.collector.appendPrometheusScrapeFile" . | fromYaml) }} +{{- $_ := set $collector "config" $config }} +{{- end }} +{{- if .collector.presets.kubernetesAttributes.enabled }} +{{- $config = (include "opentelemetry-kube-stack.collector.applyKubernetesAttributesConfig" (dict "collector" $collector) | fromYaml) }} +{{- $_ := set $collector "config" $config }} +{{- end }} +{{- if .collector.presets.logsCollection.enabled }} +{{- $config = (include "opentelemetry-kube-stack.collector.applyLogsCollectionConfig" (dict "collector" $collector) | fromYaml) -}} +{{- $_ := set $collector "config" $config }} +{{- end }} +{{- if .collector.presets.hostMetrics.enabled }} +{{- $config = (include "opentelemetry-kube-stack.collector.applyHostMetricsConfig" (dict "collector" $collector) | fromYaml) -}} +{{- $_ := set $collector "config" $config }} +{{- end }} +{{- if .collector.presets.kubernetesAttributes.enabled }} +{{- $config = (include "opentelemetry-kube-stack.collector.applyKubernetesAttributesConfig" (dict "collector" $collector) | fromYaml) -}} +{{- $_ := set $collector "config" $config }} +{{- end }} +{{- if .collector.presets.kubeletMetrics.enabled }} +{{- $config = (include "opentelemetry-kube-stack.collector.applyKubeletMetricsConfig" (dict "collector" $collector) | fromYaml) -}} +{{- $_ := set $collector "config" $config }} +{{- end }} +{{- if .collector.presets.kubernetesEvents.enabled }} +{{- $config = (include "opentelemetry-kube-stack.collector.applyKubernetesEventsConfig" (dict "collector" $collector) | fromYaml) -}} +{{- $_ := set $collector "config" $config }} +{{- end }} +{{- if .collector.presets.clusterMetrics.enabled }} +{{- $config = (include "opentelemetry-kube-stack.collector.applyClusterMetricsConfig" (dict "collector" $collector) | fromYaml) -}} +{{- $_ := set $collector "config" $config }} +{{- end }} +{{- if .collector.presets.batchProcessor.enabled }} +{{- $config = (include "opentelemetry-kube-stack.collector.applyBatchProcessorConfig" (dict "collector" $collector) | fromYaml) -}} +{{- $_ := set $collector "config" $config }} +{{- end }} +{{- if .collector.presets.otlpExporter.enabled }} +{{- $config = (include "opentelemetry-kube-stack.collector.applyOTLPExporter" (dict "collector" $collector) | fromYaml) -}} +{{- $_ := set $collector "config" $config }} +{{- end }} +{{- toYaml $collector.config | nindent 4 }} +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.appendPrometheusScrapeFile" -}} +{{- $loaded_file := (.Files.Get .collector.scrape_configs_file) }} +{{- $loaded_config := (fromYamlArray (tpl $loaded_file .)) }} +{{- $prom_override := (dict "receivers" (dict "prometheus" (dict "config" (dict "scrape_configs" $loaded_config)))) }} +{{- if (dig "receivers" "prometheus" "config" "scrape_configs" false .collector.config) }} +{{- $merged_prom_scrape_configs := (concat .collector.config.receivers.prometheus.config.scrape_configs $loaded_config) }} +{{- $prom_override = (dict "receivers" (dict "prometheus" (dict "config" (dict "scrape_configs" $merged_prom_scrape_configs)))) }} +{{- end }} +{{- (mergeOverwrite .collector.config $prom_override) | toYaml }} +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.applyKubernetesAttributesConfig" -}} +{{- $config := mustMergeOverwrite (include "opentelemetry-kube-stack.collector.kubernetesAttributesConfig" .collector | fromYaml) .collector.config }} +{{- if and ($config.service.pipelines.logs) (not (has "k8sattributes" $config.service.pipelines.logs.processors)) }} +{{- $_ := set $config.service.pipelines.logs "processors" (prepend $config.service.pipelines.logs.processors "k8sattributes" | uniq) }} +{{- end }} +{{- if and ($config.service.pipelines.metrics) (not (has "k8sattributes" $config.service.pipelines.metrics.processors)) }} +{{- $_ := set $config.service.pipelines.metrics "processors" (prepend $config.service.pipelines.metrics.processors "k8sattributes" | uniq) }} +{{- end }} +{{- if and ($config.service.pipelines.traces) (not (has "k8sattributes" $config.service.pipelines.traces.processors)) }} +{{- $_ := set $config.service.pipelines.traces "processors" (prepend $config.service.pipelines.traces.processors "k8sattributes" | uniq) }} +{{- end }} +{{- $config | toYaml }} +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.kubernetesAttributesConfig" -}} +processors: + k8sattributes: + {{- if eq .mode "daemonset" }} + filter: + node_from_env_var: K8S_NODE_NAME + {{- end }} + passthrough: false + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.uid + - sources: + - from: resource_attribute + name: k8s.pod.name + - from: resource_attribute + name: k8s.namespace.name + - from: resource_attribute + name: k8s.node.name + - sources: + - from: resource_attribute + name: k8s.pod.ip + - sources: + - from: resource_attribute + name: k8s.pod.name + - from: resource_attribute + name: k8s.namespace.name + - sources: + - from: connection + extract: + metadata: + - k8s.namespace.name + - k8s.pod.name + - k8s.pod.uid + - k8s.node.name + - k8s.pod.start_time + - k8s.deployment.name + - k8s.replicaset.name + - k8s.replicaset.uid + - k8s.daemonset.name + - k8s.daemonset.uid + - k8s.job.name + - k8s.job.uid + - k8s.container.name + - k8s.cronjob.name + - k8s.statefulset.name + - k8s.statefulset.uid + - container.image.tag + - container.image.name + - k8s.cluster.uid + labels: + - tag_name: service.name + key: app.kubernetes.io/name + from: pod + - tag_name: service.name + key: k8s-app + from: pod + - tag_name: k8s.app.instance + key: app.kubernetes.io/instance + from: pod + - tag_name: service.version + key: app.kubernetes.io/version + from: pod + - tag_name: k8s.app.component + key: app.kubernetes.io/component + from: pod + {{- if .presets.kubernetesAttributes.extractAllPodLabels }} + - tag_name: $$1 + key_regex: (.*) + from: pod + {{- end }} + {{- if .presets.kubernetesAttributes.extractAllPodAnnotations }} + annotations: + - tag_name: $$1 + key_regex: (.*) + from: pod + {{- end }} +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.applyHostMetricsConfig" -}} +{{- $config := mustMergeOverwrite (include "opentelemetry-kube-stack.collector.hostMetricsConfig" .collector | fromYaml) .collector.config }} +{{- if and ($config.service.pipelines.metrics) (not (has "hostmetrics" $config.service.pipelines.metrics.receivers)) }} +{{- $_ := set $config.service.pipelines.metrics "receivers" (append $config.service.pipelines.metrics.receivers "hostmetrics" | uniq) }} +{{- end }} +{{- $config | toYaml }} +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.hostMetricsConfig" -}} +receivers: + hostmetrics: + root_path: /hostfs + collection_interval: 10s + scrapers: + cpu: + metrics: + system.cpu.utilization: + enabled: true + load: + memory: + metrics: + system.memory.utilization: + enabled: true + disk: + filesystem: + metrics: + system.filesystem.utilization: + enabled: true + exclude_mount_points: + mount_points: + - /dev/* + - /proc/* + - /sys/* + - /run/k3s/containerd/* + - /var/lib/docker/* + - /var/lib/kubelet/* + - /snap/* + match_type: regexp + exclude_fs_types: + fs_types: + - autofs + - binfmt_misc + - bpf + - cgroup2 + - configfs + - debugfs + - devpts + - devtmpfs + - fusectl + - hugetlbfs + - iso9660 + - mqueue + - nsfs + - overlay + - proc + - procfs + - pstore + - rpc_pipefs + - securityfs + - selinuxfs + - squashfs + - sysfs + - tracefs + match_type: strict + network: +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.applyClusterMetricsConfig" -}} +{{- $config := mustMergeOverwrite (include "opentelemetry-kube-stack.collector.clusterMetricsConfig" .collector | fromYaml) .collector.config }} +{{- if and ($config.service.pipelines.metrics) (not (has "k8s_cluster" $config.service.pipelines.metrics.receivers)) }} +{{- $_ := set $config.service.pipelines.metrics "receivers" (append $config.service.pipelines.metrics.receivers "k8s_cluster" | uniq) }} +{{- end }} +{{- $config | toYaml }} +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.clusterMetricsConfig" -}} +receivers: + k8s_cluster: + collection_interval: 10s + auth_type: serviceAccount + node_conditions_to_report: [Ready, MemoryPressure, DiskPressure, NetworkUnavailable] + allocatable_types_to_report: [cpu, memory, storage] +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.applyKubeletMetricsConfig" -}} +{{- $config := mustMergeOverwrite (include "opentelemetry-kube-stack.collector.kubeletMetricsConfig" .collector | fromYaml) .collector.config }} +{{- if and ($config.service.pipelines.metrics) (not (has "kubeletstats" $config.service.pipelines.metrics.receivers)) }} +{{- $_ := set $config.service.pipelines.metrics "receivers" (append $config.service.pipelines.metrics.receivers "kubeletstats" | uniq) }} +{{- end }} +{{- $config | toYaml }} +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.kubeletMetricsConfig" -}} +receivers: + kubeletstats: + collection_interval: "15s" + auth_type: "serviceAccount" + insecure_skip_verify: true + # For this scrape to work, the RBAC must have `nodes/stats` GET access. + endpoint: "https://${env:OTEL_K8S_NODE_IP}:10250" + extra_metadata_labels: + - container.id + - k8s.volume.type + metric_groups: + - node + - pod + - volume + - container + k8s_api_config: + auth_type: serviceAccount + metrics: + # k8s.pod.cpu.utilization is being deprecated + k8s.pod.cpu.usage: + enabled: true + k8s.node.uptime: + enabled: true + k8s.pod.uptime: + enabled: true +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.applyLogsCollectionConfig" -}} +{{- $config := mustMergeOverwrite (include "opentelemetry-kube-stack.collector.logsCollectionConfig" .collector | fromYaml) .collector.config }} +{{- if and ($config.service.pipelines.logs) (not (has "filelog" $config.service.pipelines.logs.receivers)) }} +{{- $_ := set $config.service.pipelines.logs "receivers" (append $config.service.pipelines.logs.receivers "filelog" | uniq) }} +{{- end }} +{{- if .collector.presets.logsCollection.storeCheckpoints}} +{{- $_ := set $config.service "extensions" (append $config.service.extensions "file_storage" | uniq) }} +{{- end }} +{{- $config | toYaml }} +{{- end }} + +{{- define "opentelemetry-kube-stack.collector.logsCollectionConfig" -}} +{{- if .presets.logsCollection.storeCheckpoints }} +extensions: + file_storage: + directory: /var/lib/otelcol +{{- end }} +receivers: + filelog: + include: + - /var/log/pods/*/*/*.log + start_at: beginning + include_file_path: true + include_file_name: false + operators: + - type: router + id: get-format + routes: + - output: parser-docker + expr: 'body matches "^\\{"' + - output: parser-crio + expr: 'body matches "^[^ Z]+ "' + - output: parser-containerd + expr: 'body matches "^[^ Z]+Z"' + # Parse CRI-O format + - type: regex_parser + id: parser-crio + regex: "^(?P