Add journald config

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
+- Journald support (#290)
 - Auto collect metrics for the apiserver control plane component
 - Add native OTel logs collection for the Windows node (#361)
 

docs/advanced-configuration.md

@@ -195,6 +195,29 @@ logsCollection:
 
 Use https://regex101.com/ to find a golang regex that works for your format and specify it in the config file for the config option `firstEntryRegex`.
 
+
+### Collect journald events
+
+Splunk OpenTelemetry Collector for Kubernetes can collect journald events from kubernetes environment.
+Process journald events by configuring `logsCollection.journald` section in values.yaml.
+
+```yaml
+logsCollection:
+  journald:
+    enabled: true
+    directory: /run/log/journal
+    # List of service units to collect and configuration for each. Please update the list as needed.
+    units:
+      - name: kubelet
+        priority: info
+      - name: docker
+        priority: info
+      - name: containerd
+       priority: info
+    # Route journald logs to its own Splunk Index by specifying the index value below, else leave it blank. Please make sure the index exist in Splunk and is configured to receive HEC traffic.
+    index: ""
+```
+
 ### Performance of native OpenTelemetry logs collection
 
 Some configurations used with the OpenTelemetry Collector (as set using the Splunk OpenTelemetry Collector for Kubernetes helm chart) can have an impact on overall performance of log ingestion. The more receivers, processors, exporters, and extensions that are added to any of the pipelines, the greater the performance impact.

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -128,7 +128,8 @@ receivers:
     listenAddress: 0.0.0.0:9080
   {{- end }}
 
-  {{- if and (eq .Values.logsEngine "otel") .Values.logsCollection.containers.enabled }}
+  {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (eq .Values.logsEngine "otel") }}
+  {{- if .Values.logsCollection.containers.enabled }}
   filelog:
     {{- if .Values.isWindows }}
     include: ["C:\\var\\log\\pods\\*\\*\\*.log"]
@@ -272,10 +273,38 @@ receivers:
           - move:
               from: log
               to: $$
+  {{- end }}
 
   {{- if .Values.logsCollection.extraFileLogs }}
   {{- toYaml .Values.logsCollection.extraFileLogs | nindent 2 }}
   {{- end }}
+
+  # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/journaldreceiver
+  {{- if .Values.logsCollection.journald.enabled }}
+  {{- range $_, $unit := .Values.logsCollection.journald.units }}
+  {{- printf "journald/%s:" $unit.name | nindent 2 }}
+    directory: {{ $.Values.logsCollection.journald.directory }}
+    units: [{{ $unit.name }}]
+    priority: {{ $unit.priority }}
+    operators:
+    - type: metadata
+      resource:
+        com.splunk.source: {{ $.Values.logsCollection.journald.directory }}
+        com.splunk.sourcetype: 'EXPR("kube:"+$$._SYSTEMD_UNIT)'
+        com.splunk.index: {{ $.Values.logsCollection.journald.index | default $.Values.splunkPlatform.index}}
+        host.name: 'EXPR(env("K8S_NODE_NAME"))'
+        # adding journald priority and unit as attributes
+        journald.priority.number: 'EXPR($$.PRIORITY)'
+        journald.unit.name: 'EXPR($$._SYSTEMD_UNIT)'
+    # extract MESSAGE field into the log body and discard rest of the fields
+    - type: restructure
+      id: set-body
+      ops:
+        - move:
+            from: MESSAGE
+            to: $$
+  {{- end }}
+  {{- end }}
   {{- end }}
 
 # By default k8sattributes and batch processors enabled.
@@ -499,12 +528,19 @@ service:
         {{- end }}
         {{- end }}
 
-    {{- if .Values.logsCollection.extraFileLogs }}
-    logs/extraFiles:
+    {{- if and (eq .Values.logsEngine "otel") (or .Values.logsCollection.extraFileLogs .Values.logsCollection.journald.enabled) }}
+    logs/host:
       receivers:
+        {{- if .Values.logsCollection.extraFileLogs }}
         {{- range $key, $exporterData := .Values.logsCollection.extraFileLogs }}
         - {{ $key }}
-        {{ end }}
+        {{- end }}
+        {{- end }}
+        {{- if (.Values.logsCollection.journald.enabled)}}
+        {{- range $_, $unit := .Values.logsCollection.journald.units }}
+        {{- printf "- journald/%s" $unit.name | nindent 8 }}
+        {{- end }}
+        {{- end }}
       processors:
         - memory_limiter
         - batch
@@ -519,7 +555,7 @@ service:
         {{- if eq (include "splunk-otel-collector.o11yLogsEnabled" .) "true" }}
         - splunk_hec/o11y
         {{- end }}
-    {{- end }}
+        {{- end }}
     {{- end }}
 
     {{- if (eq (include "splunk-otel-collector.tracesEnabled" .) "true") }}

helm-charts/splunk-otel-collector/templates/daemonset.yaml

@@ -367,6 +367,11 @@ spec:
         {{- end }}
         - name: checkpoint
           mountPath: {{ .Values.logsCollection.checkpointPath }}
+        {{- if .Values.logsCollection.journald.enabled}}
+        - mountPath: {{.Values.logsCollection.journald.directory}}
+          name: journaldlogs
+          readOnly: true
+        {{- end }}
         {{- end }}
         {{- if $agent.extraVolumeMounts }}
         {{- toYaml $agent.extraVolumeMounts | nindent 8 }}
@@ -416,6 +421,15 @@ spec:
         hostPath:
           path: {{ .Values.logsCollection.checkpointPath }}
           type: DirectoryOrCreate
+      {{- if .Values.logsCollection.journald.enabled}}
+      - name: journaldlogs
+        hostPath:
+          path: {{.Values.logsCollection.journald.directory}}
+      {{- end}}
+      - name: old-checkpoint
+        hostPath:
+          path: /var/lib/otel_pos
+          type: DirectoryOrCreate
       {{- end}}
       {{- end}}
       {{- if eq (include "splunk-otel-collector.metricsEnabled" $) "true" }}

helm-charts/splunk-otel-collector/values.schema.json

@@ -532,6 +532,28 @@
             }
           }
         },
+        "journald": {
+          "description": "Configuration to collect journald logs",
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "enabled": {
+              "type": "boolean"
+            },
+            "directory": {
+              "type": "string"
+            },
+            "index": {
+              "type": "string"
+            },
+            "units": {
+              "type": "array",
+              "items": {
+                "type": "object"
+              }
+            }
+          }
+        },
         "checkpointPath": {
           "type": "string"
         },

helm-charts/splunk-otel-collector/values.yaml

@@ -406,6 +406,22 @@ logsCollection:
     # All other logs will be ignored.
     useSplunkIncludeAnnotation: false
 
+# Configuration for collecting journald logs using otel collector
+  journald:
+    enabled: false
+    # Please update directory path for journald if it's different from below default value "/var/log/journal"
+    directory: /var/log/journal
+    # List of service units to collect journald logs for and configuration for each.
+    units:
+      - name: kubelet
+        priority: info
+      - name: docker
+        priority: info
+      - name: containerd
+        priority: info
+    # Route journald logs to its own Splunk Index by specifying the index value below, else leave it blank. Please make sure the index exist in Splunk and is configured to receive HEC traffic. Not applicable to Splunk Observability.
+    index: ""
+
   checkpointPath: "/var/addon/splunk/otel_pos"
 
   # Files on k8s nodes to tail.

rendered/manifests/otel-logs/daemonset.yaml

@@ -216,6 +216,10 @@ spec:
         hostPath:
           path: /var/addon/splunk/otel_pos
           type: DirectoryOrCreate
+      - name: old-checkpoint
+        hostPath:
+          path: /var/lib/otel_pos
+          type: DirectoryOrCreate
       - name: host-dev
         hostPath:
           path: /dev