signalfx · dmitryax · Jan 21, 2022 · Nov 16, 2021 · Nov 16, 2021 · Nov 16, 2021
@@ -235,7 +235,37 @@ receivers:
               from: log
               to: $$
   {{- end }}
-
+  # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/journaldreceiver
+  {{- if (eq (include "splunk-otel-collector.logsEnabled" .) "true") }}
+  {{- if and (eq .Values.logsEngine "otel") .Values.logsCollection.journaldLogs.enabled }}
+  {{- if .Values.logsCollection.journaldLogs.units }}
+  {{- range $_, $unit := .Values.logsCollection.journaldLogs.units }}
+  {{- printf "journald/%s:" $unit.name | nindent 2 }}
+    directory: {{ $.Values.logsCollection.journaldLogs.directory }}
+    units: [{{ $unit.name }}]
+    priority: {{ $unit.priority | default $.Values.logsCollection.journaldLogs.defaultPriority }}
+    resource:
+      com.splunk.source: {{ $.Values.logsCollection.journaldLogs.directory }}
+      com.splunk.sourcetype: 'EXPR("kube:"+$$._SYSTEMD_UNIT)'
+      com.splunk.index: {{ $.Values.logsCollection.journaldLogs.index | default $.Values.splunkPlatform.index}}
+      host.name: 'EXPR(env("K8S_NODE_NAME"))'
+      {{- if $.Values.clusterName }}
+      k8s.cluster.name: {{ $.Values.clusterName }}
+      {{- end }}
+      {{- if $.Values.environment }}
+      deployment.environment: {{ $.Values.environment }}
+      {{- end }}
+      {{- if $.Values.extraAttributes.custom }}
+      {{- toYaml $.Values.extraAttributes.custom | nindent 6 }}
+      {{- end }}
+  {{- end }}
+  {{- end }}
+  {{- else }}
+  journald:
+    directory: {{- toYaml .Values.logsCollection.journaldLogs.directory | nindent 6 }}
+    priority: {{ .Values.logsCollection.journaldLogs.defaultPriority }}
+  {{- end }}
+  {{- end }}
 # By default k8sattributes and batch processors enabled.
 processors:
   # k8sattributes enriches traces and metrics with k8s metadata
@@ -532,4 +562,31 @@ service:
         {{- end }}
         {{- end }}
     {{- end }}
+
+    {{- if (eq (include "splunk-otel-collector.logsEnabled" .) "true") }}
+    {{- if and (eq .Values.logsEngine "otel") .Values.logsCollection.journaldLogs.enabled }}
+    logs/journald:
+      receivers:
+        {{- if .Values.logsCollection.journaldLogs.units }}
+        {{- range $_, $unit := .Values.logsCollection.journaldLogs.units }}
+        {{- printf "- journald/%s" $unit.name | nindent 8 }}
+        {{- end }}
+        {{- else }}
+        - journald
+        {{- end }}
+      processors:
+        - batch
+      exporters:
+        {{- if .Values.otelCollector.enabled }}
+        - otlp
+        {{- else }}
+        {{- if (eq (include "splunk-otel-collector.o11yLogsEnabled" .) "true") }}
+        - splunk_hec/o11y
+        {{- end }}
+        {{- if (eq (include "splunk-otel-collector.platformLogsEnabled" .) "true") }}
+        - splunk_hec/platform_logs
+        {{- end }}
+        {{- end }}
+    {{- end }}
+    {{- end }}
 {{- end }}
@@ -322,6 +322,11 @@ spec:
           readOnly: true
         - name: checkpoint
           mountPath: {{ .Values.logsCollection.checkpointPath }}
+        {{- if .Values.logsCollection.journaldLogs.enabled}}
+        - mountPath: {{.Values.logsCollection.journaldLogs.directory}}
+          name: journaldlogs-otel
+          readOnly: true
+        {{- end }}
         {{- end }}
         {{- if .Values.otelAgent.extraVolumeMounts }}
         {{- toYaml .Values.otelAgent.extraVolumeMounts | nindent 8 }}
@@ -365,6 +370,11 @@ spec:
         hostPath:
           path: {{ .Values.logsCollection.checkpointPath }}
           type: DirectoryOrCreate
+      {{- if .Values.logsCollection.journaldLogs.enabled}}
+      - name: journaldlogs-otel
+        hostPath:
+          path: {{.Values.logsCollection.journaldLogs.directory}}
+      {{- end}}
       {{- end}}
       {{- end}}
       {{- if eq (include "splunk-otel-collector.metricsEnabled" $) "true" }}

@@ -489,6 +489,12 @@
             }
           }
         },
+        "journaldLogs": {
+          "type": "object",
+          "items": {
+            "type": "object"
+          }
+        },
         "checkpointPath": {
           "type": "string"
         }

@@ -372,6 +372,21 @@ logsCollection:
     #       value: server
     #     firstEntryRegex: ^[^\s].*
 
+# Configuration for collecting journald logs using otel collector
+  journaldLogs:
+    enabled: true
+    directory: /run/log/journal
+    defaultPriority: info
+    # List of service units to collect and configuration for each. To collect all journald services logs, leave units blank.
+    units:
+      - name: kubelet
+        priority: info
+      - name: docker
+        priority: info
+      - name: containerd
+        priority: info
+    # Route journald logs to its own Splunk Index by specifying the index value below, else leave it blank. Please make sure the index exist in Splunk and is configured to receive HEC traffic.
+    index:
   checkpointPath: "/var/lib/otel_pos"
 
 ################################################################################

diff --git a/rendered/manifests/agent-only/configmap-otel-agent.yaml b/rendered/manifests/agent-only/configmap-otel-agent.yaml
@@ -152,6 +152,9 @@ data:
             endpoint: 0.0.0.0:14250
           thrift_http:
             endpoint: 0.0.0.0:14268
+      journald:
+        directory: /run/log/journal
+        priority: info
       kubeletstats:
         auth_type: serviceAccount
         collection_interval: 10s

diff --git a/rendered/manifests/agent-only/daemonset.yaml b/rendered/manifests/agent-only/daemonset.yaml
@@ -28,7 +28,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 4eeca5afdb8456588b3f21440b9833d2f723f657ae167126f7f5182eea44bf30
+        checksum/config: 30edcde59e9ca392a04630e41bc491ed9317e597cf8b1c3aa36ea98b92f09cd9
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/rendered/manifests/logs-only/configmap-otel-agent.yaml b/rendered/manifests/logs-only/configmap-otel-agent.yaml
@@ -132,6 +132,9 @@ data:
     receivers:
       fluentforward:
         endpoint: 0.0.0.0:8006
+      journald:
+        directory: /run/log/journal
+        priority: info
       otlp:
         protocols:
           grpc:

diff --git a/rendered/manifests/logs-only/daemonset.yaml b/rendered/manifests/logs-only/daemonset.yaml
@@ -28,7 +28,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: a31970065aabdf55dec4f5397d2d338f6b988f1599e0dca4ecf168fd294eb945
+        checksum/config: 69cb918c344b7fd787c77333133cc73371243be9050cd4691ea788298685ca11
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

diff --git a/rendered/manifests/otel-logs/configmap-otel-agent.yaml b/rendered/manifests/otel-logs/configmap-otel-agent.yaml
@@ -233,6 +233,39 @@ data:
             endpoint: 0.0.0.0:14250
           thrift_http:
             endpoint: 0.0.0.0:14268
+      journald/containerd:
+        directory: /run/log/journal
+        priority: info
+        resource:
+          com.splunk.index: null
+          com.splunk.source: /run/log/journal
+          com.splunk.sourcetype: EXPR("kube:"+$$._SYSTEMD_UNIT)
+          host.name: EXPR(env("K8S_NODE_NAME"))
+          k8s.cluster.name: CHANGEME
+        units:
+        - containerd
+      journald/docker:
+        directory: /run/log/journal
+        priority: info
+        resource:
+          com.splunk.index: null
+          com.splunk.source: /run/log/journal
+          com.splunk.sourcetype: EXPR("kube:"+$$._SYSTEMD_UNIT)
+          host.name: EXPR(env("K8S_NODE_NAME"))
+          k8s.cluster.name: CHANGEME
+        units:
+        - docker
+      journald/kubelet:
+        directory: /run/log/journal
+        priority: info
+        resource:
+          com.splunk.index: null
+          com.splunk.source: /run/log/journal
+          com.splunk.sourcetype: EXPR("kube:"+$$._SYSTEMD_UNIT)
+          host.name: EXPR(env("K8S_NODE_NAME"))
+          k8s.cluster.name: CHANGEME
+        units:
+        - kubelet
       kubeletstats:
         auth_type: serviceAccount
         collection_interval: 10s
@@ -291,6 +324,15 @@ data:
           - filelog
           - fluentforward
           - otlp
+        logs/journald:
+          exporters:
+          - splunk_hec/o11y
+          processors:
+          - batch
+          receivers:
+          - journald/kubelet
+          - journald/docker
+          - journald/containerd
         metrics:
           exporters:
           - signalfx

diff --git a/rendered/manifests/otel-logs/daemonset.yaml b/rendered/manifests/otel-logs/daemonset.yaml
@@ -27,7 +27,7 @@ spec:
         app: splunk-otel-collector
         release: default
       annotations:
-        checksum/config: 79fbef3ed17c5716a0d28f36d47d1f28fa123718a75dd9865dec2160707b0b47
+        checksum/config: d9a2677e16887f2d810df5c2dbac2fb3a0abf4ffa6d7b910dad19480f9c13dbf
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true
@@ -204,6 +204,9 @@ spec:
           readOnly: true
         - name: checkpoint
           mountPath: /var/lib/otel_pos
+        - mountPath: /run/log/journal
+          name: journaldlogs-otel
+          readOnly: true
       terminationGracePeriodSeconds: 600
       volumes:
       - name: varlog
@@ -216,6 +219,9 @@ spec:
         hostPath:
           path: /var/lib/otel_pos
           type: DirectoryOrCreate
+      - name: journaldlogs-otel
+        hostPath:
+          path: /run/log/journal
       - name: host-dev
         hostPath:
           path: /dev