From a9f24c089c57be8195f24ecc9ced59ad68e27abc Mon Sep 17 00:00:00 2001 From: wojtekzyla <108660584+wojtekzyla@users.noreply.github.com> Date: Thu, 19 Jan 2023 00:37:38 +0100 Subject: [PATCH] Fix/security issues fix (#617) * fix: enable running otel and fluentd with the following security settings: readOnlyRootFilesystem, allowPrivilegeEscalation, seccompProfile, runAsNonRoot. * fix: add condition for adding fluentd-checkpoint-dir volume in post delete hook * fix: comment about issues with cri-o runtime * Update helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml Co-authored-by: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> * fix: run pre-commit * Address review comments * fix pre-commit * fix: change default user and group to 999 in patch-log-dirs init container Co-authored-by: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Co-authored-by: harshit-splunk Co-authored-by: Antoine Toulme --- .DS_Store | Bin 0 -> 6148 bytes .../templates/configmap-fluentd.yaml | 2 +- .../templates/daemonset.yaml | 87 +++++++++++++++--- .../templates/revert-patch-log-dirs-hook.yaml | 82 +++++++++++++++++ .../templates/securityContextConstraints.yaml | 2 + helm-charts/splunk-otel-collector/values.yaml | 33 +++++-- rendered/manifests/agent-only/daemonset.yaml | 8 ++ .../cluster-receiver-objects/daemonset.yaml | 47 ++++++++-- .../logs-only/configmap-fluentd.yaml | 8 +- rendered/manifests/logs-only/daemonset.yaml | 42 ++++++++- .../manifests/metrics-only/daemonset.yaml | 8 ++ rendered/manifests/otel-logs/daemonset.yaml | 47 ++++++++-- rendered/manifests/traces-only/daemonset.yaml | 8 ++ 13 files changed, 335 insertions(+), 39 deletions(-) create mode 100644 .DS_Store create mode 100644 helm-charts/splunk-otel-collector/templates/revert-patch-log-dirs-hook.yaml diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 @type local persistent true - path /var/log/splunkd-fluentd-journald-{{ $name }}.pos.json + path {{ $.Values.fluentd.config.posFilePrefix }}-journald-{{ $name }}.pos.json field_map {"MESSAGE": "log", "_SYSTEMD_UNIT": "source"} diff --git a/helm-charts/splunk-otel-collector/templates/daemonset.yaml b/helm-charts/splunk-otel-collector/templates/daemonset.yaml index 4bfbbbc12a..361b64c3b4 100644 --- a/helm-charts/splunk-otel-collector/templates/daemonset.yaml +++ b/helm-charts/splunk-otel-collector/templates/daemonset.yaml @@ -70,7 +70,31 @@ spec: {{- end }} {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (not .Values.isWindows) }} initContainers: - {{- if and (eq .Values.logsEngine "fluentd") (not (eq .Values.distribution "gke/autopilot")) }} + {{- if ne .Values.distribution "gke/autopilot" }} + # Previously, fluentd checkpoints were written to /var/log directory. + # So, /var/log directory could not be mounted as read-only. + # Now, default fluentd checkpoint is moved to /var/addon/splunk/fluent_pos directory + # move-fluent-checkpoint will handle 2 scenarios + # - When upgrading from fluentd to fluentd, it will move checkpoint, so fluentd can continue from where it left + # - When migrating from fluentd to otel, it will move checkpoint, and then migrate to otel checkpoint. So, it + # covers both previous and current fluentd checkpoint directory. + {{- if ne .Values.fluentd.config.posFilePrefix "/var/log/splunk-fluentd" }} + - name: move-fluent-checkpoint + image: {{ template "splunk-otel-collector.image.initPatchLogDirs" . }} + imagePullPolicy: {{ .Values.image.initPatchLogDirs.pullPolicy }} + command: ['sh', '-c', ' + mkdir -p {{ dir .Values.fluentd.config.posFilePrefix }}; + find /var/log -maxdepth 1 -name *.json | xargs -I{} mv {} -t {{ dir .Values.fluentd.config.posFilePrefix }}; + find /var/log -maxdepth 1 -name *.pos | xargs -I{} mv {} -t {{ dir .Values.fluentd.config.posFilePrefix }};'] + securityContext: + runAsUser: 0 + volumeMounts: + - name: fluentd-checkpoint-dir + mountPath: {{ dir .Values.fluentd.config.posFilePrefix }} + - name: varlog + mountPath: /var/log + {{- end }} + {{- if eq .Values.logsEngine "fluentd" }} - name: prepare-fluentd-config image: {{ template "splunk-otel-collector.image.fluentd" . }} imagePullPolicy: {{ .Values.image.fluentd.pullPolicy }} @@ -97,7 +121,6 @@ spec: - name: fluentd-config-json mountPath: /fluentd/etc/json {{- else }} - {{- if not (eq .Values.distribution "gke/autopilot") }} - name: migrate-checkpoint image: {{ template "splunk-otel-collector.image.otelcol" . }} imagePullPolicy: {{ .Values.image.otelcol.pullPolicy }} @@ -128,27 +151,40 @@ spec: mountPath: /var/log - name: varlibdockercontainers mountPath: /var/lib/docker/containers + {{- if ne .Values.fluentd.config.posFilePrefix "/var/log/splunk-fluentd" }} + - name: fluentd-checkpoint-dir + mountPath: {{ dir .Values.fluentd.config.posFilePrefix -}} + {{- end }} {{- end }} - {{- if and $agent.securityContext.runAsUser $agent.securityContext.runAsGroup }} + {{- end }} + {{- if or (and (.Values.fluentd.securityContext.runAsUser) (.Values.fluentd.securityContext.runAsGroup)) (and ($agent.securityContext.runAsUser) ($agent.securityContext.runAsGroup)) }} - name: patch-log-dirs image: {{ template "splunk-otel-collector.image.initPatchLogDirs" . }} imagePullPolicy: {{ .Values.image.initPatchLogDirs.pullPolicy }} command: ['sh', '-c', ' mkdir -p {{ .Values.logsCollection.checkpointPath }}; - chown -Rv {{ $agent.securityContext.runAsUser | default 20000 }}:{{ $agent.securityContext.runAsGroup | default 20000 }} {{ .Values.logsCollection.checkpointPath }}; - chmod -v g+rwxs {{ .Values.logsCollection.checkpointPath }}; + setfacl -n -Rm d:m::rwx,m::rwx,d:g:{{ $agent.securityContext.runAsGroup | default 999 }}:rwx,g:{{ $agent.securityContext.runAsGroup | default 999 }}:rwx {{ .Values.logsCollection.checkpointPath }}; + {{- if ne .Values.fluentd.config.posFilePrefix "/var/log/splunk-fluentd" }} + setfacl -n -Rm d:m::rwx,m::rwx,d:g:{{ .Values.fluentd.securityContext.runAsGroup | default 999 }}:rwx,g:{{ .Values.fluentd.securityContext.runAsGroup | default 999 }}:rwx {{ dir .Values.fluentd.config.posFilePrefix }}; + {{- end }} {{ if .Values.logsCollection.containers.enabled -}} if [ -d "/var/lib/docker/containers" ]; then - setfacl -n -Rm d:g:{{ $agent.securityContext.runAsGroup | default 20000 }}:rx,g:{{ $agent.securityContext.runAsGroup | default 20000 }}:rx /var/lib/docker/containers; + setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx,g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx /var/lib/docker/containers; fi; if [ -d "/var/log/crio/pods" ]; then - setfacl -n -Rm d:g:{{ $agent.securityContext.runAsGroup | default 20000 }}:rx,g:{{ $agent.securityContext.runAsGroup | default 20000 }}:rx /var/log/crio/pods; + setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx,g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx /var/log/crio/pods; fi; if [ -d "/var/log/pods" ]; then - setfacl -n -Rm d:g:{{ $agent.securityContext.runAsGroup | default 20000 }}:rx,g:{{ $agent.securityContext.runAsGroup | default 20000 }}:rx /var/log/pods; + setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx,g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx /var/log/pods; + fi; + {{- end }} + {{- if .Values.logsCollection.journald.enabled }} + if [ -d "{{ .Values.logsCollection.journald.directory }}" ]; + then + setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx,g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx {{ .Values.logsCollection.journald.directory }}; fi; {{- end }}'] securityContext: @@ -156,13 +192,20 @@ spec: volumeMounts: - name: checkpoint mountPath: {{ .Values.logsCollection.checkpointPath }} + {{- if ne .Values.fluentd.config.posFilePrefix "/var/log/splunk-fluentd" }} + - name: fluentd-checkpoint-dir + mountPath: {{ dir .Values.fluentd.config.posFilePrefix }} + {{- end }} {{- if .Values.logsCollection.containers.enabled }} - name: varlog mountPath: /var/log - name: varlibdockercontainers mountPath: /var/lib/docker/containers {{- end }} - {{- end }} + {{- if .Values.logsCollection.journald.enabled }} + - name: journaldlogs + mountPath: {{.Values.logsCollection.journald.directory}} + {{- end }} {{- end }} {{- end }} containers: @@ -195,6 +238,7 @@ spec: volumeMounts: - name: varlog mountPath: {{ .Values.fluentd.config.containers.path }} + readOnly: true - name: varlogdest mountPath: {{ .Values.fluentd.config.containers.pathDest }} readOnly: true @@ -203,6 +247,10 @@ spec: readOnly: true - name: fluentd-config mountPath: /fluentd/etc + {{- if ne .Values.fluentd.config.posFilePrefix "/var/log/splunk-fluentd" }} + - name: fluentd-checkpoint-dir + mountPath: {{ dir .Values.fluentd.config.posFilePrefix }} + {{- end }} - name: tmp mountPath: /tmp {{- end }} @@ -396,7 +444,24 @@ spec: terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} volumes: {{- if (eq (include "splunk-otel-collector.logsEnabled" .) "true") }} + {{- if not .Values.isWindows }} + - name: checkpoint + hostPath: + path: {{ .Values.logsCollection.checkpointPath }} + type: DirectoryOrCreate + {{- if ne .Values.fluentd.config.posFilePrefix "/var/log/splunk-fluentd" }} + - name: fluentd-checkpoint-dir + hostPath: + path: {{ dir .Values.fluentd.config.posFilePrefix }} + type: DirectoryOrCreate + {{- end }} + {{- end }} {{- if eq .Values.logsEngine "fluentd" }} + {{- if or (and (.Values.fluentd.securityContext.runAsUser) (.Values.fluentd.securityContext.runAsGroup)) (and ($agent.securityContext.runAsUser) ($agent.securityContext.runAsGroup)) }} + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + {{- end }} - name: varlog hostPath: path: {{ .Values.fluentd.config.containers.path }} @@ -438,10 +503,6 @@ spec: hostPath: path: /var/lib/docker/containers {{- end }} - - name: checkpoint - hostPath: - path: {{ .Values.logsCollection.checkpointPath }} - type: DirectoryOrCreate {{- if .Values.logsCollection.journald.enabled}} - name: journaldlogs hostPath: diff --git a/helm-charts/splunk-otel-collector/templates/revert-patch-log-dirs-hook.yaml b/helm-charts/splunk-otel-collector/templates/revert-patch-log-dirs-hook.yaml new file mode 100644 index 0000000000..6fcb3907f2 --- /dev/null +++ b/helm-charts/splunk-otel-collector/templates/revert-patch-log-dirs-hook.yaml @@ -0,0 +1,82 @@ +{{- if or (and (.Values.fluentd.securityContext.runAsUser) (.Values.fluentd.securityContext.runAsGroup)) (and (.Values.agent.securityContext.runAsUser) (.Values.agent.securityContext.runAsGroup)) }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ template "splunk-otel-collector.fullname" . }}-revert-patch-log-dir + labels: + {{- include "splunk-otel-collector.commonLabels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: revert-patch-log-dirs + image: {{ template "splunk-otel-collector.image.initPatchLogDirs" . }} + imagePullPolicy: {{ .Values.image.initPatchLogDirs.pullPolicy }} + securityContext: + runAsUser: 0 + command: ['sh', '-c', ' + setfacl --recursive --remove-all {{ .Values.logsCollection.checkpointPath }}; + {{- if ne .Values.fluentd.config.posFilePrefix "/var/log/splunk-fluentd" }} + setfacl --recursive --remove-all {{ dir .Values.fluentd.config.posFilePrefix }}; + {{- end }} + {{ if .Values.logsCollection.containers.enabled -}} + if [ -d "/var/lib/docker/containers" ]; + then + setfacl --recursive --remove-all /var/lib/docker/containers; + fi; + if [ -d "/var/log/crio/pods" ]; + then + setfacl --recursive --remove-all /var/log/crio/pods; + fi; + if [ -d "/var/log/pods" ]; + then + setfacl --recursive --remove-all /var/log/pods; + fi; + {{- end }} + {{- if .Values.logsCollection.journald.enabled }} + if [ -d "{{ .Values.logsCollection.journald.directory }}" ]; + then + setfacl --recursive --remove-all d:m::rx,m::rx,d:g:{{ .Values.agent.securityContext.runAsGroup | default 999 }}:rx,g:{{ .Values.agent.securityContext.runAsGroup | default 999 }}:rx {{ .Values.logsCollection.journald.directory }}; + fi; + {{- end }}'] + volumeMounts: + - name: checkpoint + mountPath: {{ .Values.logsCollection.checkpointPath }} + {{- if ne .Values.fluentd.config.posFilePrefix "/var/log/splunk-fluentd" }} + - name: fluentd-checkpoint-dir + mountPath: {{ dir .Values.fluentd.config.posFilePrefix }} + {{- end }} + {{- if .Values.logsCollection.containers.enabled }} + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + {{- end }} + {{- if .Values.logsCollection.journald.enabled }} + - name: journaldlogs + mountPath: {{.Values.logsCollection.journald.directory}} + {{- end }} + volumes: + - name: checkpoint + hostPath: + path: {{ .Values.logsCollection.checkpointPath }} + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + {{- if ne .Values.fluentd.config.posFilePrefix "/var/log/splunk-fluentd" }} + - name: fluentd-checkpoint-dir + hostPath: + path: {{ dir .Values.fluentd.config.posFilePrefix }} + type: DirectoryOrCreate + {{- end }} + {{- if .Values.logsCollection.journald.enabled}} + - name: journaldlogs + hostPath: + path: {{.Values.logsCollection.journald.directory}} + {{- end}} +{{- end }} diff --git a/helm-charts/splunk-otel-collector/templates/securityContextConstraints.yaml b/helm-charts/splunk-otel-collector/templates/securityContextConstraints.yaml index 150552f83e..97eb54711b 100644 --- a/helm-charts/splunk-otel-collector/templates/securityContextConstraints.yaml +++ b/helm-charts/splunk-otel-collector/templates/securityContextConstraints.yaml @@ -42,6 +42,8 @@ runAsUser: type: RunAsAny supplementalGroups: type: RunAsAny +seccompProfiles: +- runtime/default requiredDropCapabilities: - ALL {{- end }} diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index b37506e905..cafc55114e 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -350,13 +350,19 @@ agent: memory: 500Mi # To collect container logs and journald logs, it will run the agent as a root user. - # To run it as non root user, uncomment below `securityContext` options. + # To run it as non root user, change runAsUser and runAsGroup to non-zero value (e.g. 999) and change runAsNonRoot to true. # Setting runAsUser and runAsGroup to a non root user enables an init container that patches group # permissions of container logs directories on the host filesystem to make logs readable by this non root user. - - securityContext: {} - # runAsUser: 20000 - # runAsGroup: 20000 + # NOTE: Running this container as a non-root user doesn't work with openshift cluser or cri-o runntime. + # See https://github.com/cri-o/cri-o/issues/6519 for more details. + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false # Specifies DaemonSet update strategy. # Possible values: "OnDelete" and "RollingUpdate". @@ -606,8 +612,23 @@ fluentd: cpu: 100m memory: 200Mi + # To collect container logs and journald logs, it will run the fluentd sidecar as a root user. + # To run it as non root user, change runAsUser and runAsGroup to non-zero value (e.g. 999) and change runAsNonRoot to true. + # Setting runAsUser and runAsGroup to a non root user enables an init container that patches group + # permissions of container logs directories on the host filesystem to make logs readable by this non root user. + # NOTE: Running this container as a non-root user doesn't work with openshift cluser or cri-o runntime. + # See https://github.com/cri-o/cri-o/issues/6519 for more details. + securityContext: runAsUser: 0 + # securityContext: + # readOnlyRootFilesystem: true + # allowPrivilegeEscalation: false + # seccompProfile: + # type: RuntimeDefault + # runAsUser: 999 + # runAsGroup: 999 + # runAsNonRoot: true # Extra enviroment variables to be set in the FluentD container extraEnvs: [] @@ -662,7 +683,7 @@ fluentd: # Prefix for pos_file tail source parameter # Can be used if you want to run multiple instances of fluentd on the same host # https://docs.fluentd.org/input/tail#pos_file-highly-recommended - posFilePrefix: /var/log/splunk-fluentd + posFilePrefix: /var/addon/splunk/fluent_pos/splunk-fluentd # `customFilters` defines the custom filters to be used. # This section can be used to define custom filters using plugins like https://github.com/splunk/fluent-plugin-jq diff --git a/rendered/manifests/agent-only/daemonset.yaml b/rendered/manifests/agent-only/daemonset.yaml index 9f06ac80e3..d2a3b7c38f 100644 --- a/rendered/manifests/agent-only/daemonset.yaml +++ b/rendered/manifests/agent-only/daemonset.yaml @@ -79,6 +79,14 @@ spec: protocol: TCP image: quay.io/signalfx/splunk-otel-collector:0.67.0 imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault env: - name: SPLUNK_MEMORY_TOTAL_MIB value: "500" diff --git a/rendered/manifests/cluster-receiver-objects/daemonset.yaml b/rendered/manifests/cluster-receiver-objects/daemonset.yaml index eb8ebd5fea..306d809186 100644 --- a/rendered/manifests/cluster-receiver-objects/daemonset.yaml +++ b/rendered/manifests/cluster-receiver-objects/daemonset.yaml @@ -42,6 +42,27 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/master initContainers: + # Previously, fluentd checkpoints were written to /var/log directory. + # So, /var/log directory could not be mounted as read-only. + # Now, default fluentd checkpoint is moved to /var/addon/splunk/fluent_pos directory + # move-fluent-checkpoint will handle 2 scenarios + # - When upgrading from fluentd to fluentd, it will move checkpoint, so fluentd can continue from where it left + # - When migrating from fluentd to otel, it will move checkpoint, and then migrate to otel checkpoint. So, it + # covers both previous and current fluentd checkpoint directory. + - name: move-fluent-checkpoint + image: registry.access.redhat.com/ubi9/ubi + imagePullPolicy: IfNotPresent + command: ['sh', '-c', ' + mkdir -p /var/addon/splunk/fluent_pos; + find /var/log -maxdepth 1 -name *.json | xargs -I{} mv {} -t /var/addon/splunk/fluent_pos; + find /var/log -maxdepth 1 -name *.pos | xargs -I{} mv {} -t /var/addon/splunk/fluent_pos;'] + securityContext: + runAsUser: 0 + volumeMounts: + - name: fluentd-checkpoint-dir + mountPath: /var/addon/splunk/fluent_pos + - name: varlog + mountPath: /var/log - name: migrate-checkpoint image: quay.io/signalfx/splunk-otel-collector:0.67.0 imagePullPolicy: IfNotPresent @@ -50,15 +71,15 @@ spec: runAsUser: 0 env: - name: CONTAINER_LOG_PATH_FLUENTD - value: "/var/log/splunk-fluentd-containers.log.pos" + value: "/var/addon/splunk/fluent_pos/splunk-fluentd-containers.log.pos" - name: CONTAINER_LOG_PATH_OTEL value: "/var/addon/splunk/otel_pos/receiver_filelog_" - name: CUSTOM_LOG_PATH_FLUENTD - value: "/var/log/splunk-fluentd-*.pos" + value: "/var/addon/splunk/fluent_pos/splunk-fluentd-*.pos" - name: CUSTOM_LOG_PATH_OTEL value: "/var/addon/splunk/otel_pos/receiver_filelog_" - name: CUSTOM_LOG_CAPTURE_REGEX - value: '\/var\/log\/splunk\-fluentd\-(?P[\w0-9-_]+)\.pos' + value: '\/var\/addon\/splunk\/fluent_pos\/splunk\-fluentd\-(?P[\w0-9-_]+)\.pos' - name: JOURNALD_LOG_PATH_FLUENTD value: "/var/log/splunkd-fluentd-journald-*.pos.json" - name: JOURNALD_LOG_PATH_OTEL @@ -72,6 +93,8 @@ spec: mountPath: /var/log - name: varlibdockercontainers mountPath: /var/lib/docker/containers + - name: fluentd-checkpoint-dir + mountPath: /var/addon/splunk/fluent_pos containers: - name: otel-collector command: @@ -115,7 +138,13 @@ spec: image: quay.io/signalfx/splunk-otel-collector:0.67.0 imagePullPolicy: IfNotPresent securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsNonRoot: false runAsUser: 0 + seccompProfile: + type: RuntimeDefault env: - name: SPLUNK_MEMORY_TOTAL_MIB value: "500" @@ -212,16 +241,20 @@ spec: mountPath: /var/addon/splunk/otel_pos terminationGracePeriodSeconds: 600 volumes: + - name: checkpoint + hostPath: + path: /var/addon/splunk/otel_pos + type: DirectoryOrCreate + - name: fluentd-checkpoint-dir + hostPath: + path: /var/addon/splunk/fluent_pos + type: DirectoryOrCreate - name: varlog hostPath: path: /var/log - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - - name: checkpoint - hostPath: - path: /var/addon/splunk/otel_pos - type: DirectoryOrCreate - name: host-dev hostPath: path: /dev diff --git a/rendered/manifests/logs-only/configmap-fluentd.yaml b/rendered/manifests/logs-only/configmap-fluentd.yaml index d2e8b6c3ea..16017bdd9c 100644 --- a/rendered/manifests/logs-only/configmap-fluentd.yaml +++ b/rendered/manifests/logs-only/configmap-fluentd.yaml @@ -70,7 +70,7 @@ data: @label @CONCAT tag tail.containers.* path /var/log/containers/*.log - pos_file /var/log/splunk-fluentd-containers.log.pos + pos_file /var/addon/splunk/fluent_pos/splunk-fluentd-containers.log.pos path_key source read_from_head true @@ -89,7 +89,7 @@ data: @label @CONCAT tag tail.file.kube:apiserver-audit path /var/log/kube-apiserver-audit.log - pos_file /var/log/splunk-fluentd-kube-audit.pos + pos_file /var/addon/splunk/fluent_pos/splunk-fluentd-kube-audit.pos read_from_head true path_key source @@ -114,7 +114,7 @@ data: @type local persistent true - path /var/log/splunkd-fluentd-journald-docker.pos.json + path /var/addon/splunk/fluent_pos/splunk-fluentd-journald-docker.pos.json field_map {"MESSAGE": "log", "_SYSTEMD_UNIT": "source"} @@ -132,7 +132,7 @@ data: @type local persistent true - path /var/log/splunkd-fluentd-journald-kubelet.pos.json + path /var/addon/splunk/fluent_pos/splunk-fluentd-journald-kubelet.pos.json field_map {"MESSAGE": "log", "_SYSTEMD_UNIT": "source"} diff --git a/rendered/manifests/logs-only/daemonset.yaml b/rendered/manifests/logs-only/daemonset.yaml index 0a7792b315..88b3bc35ee 100644 --- a/rendered/manifests/logs-only/daemonset.yaml +++ b/rendered/manifests/logs-only/daemonset.yaml @@ -30,7 +30,7 @@ spec: app: splunk-otel-collector release: default annotations: - checksum/config: 3f26384bd74849568f07d76e43867944f957a4681c542480b0a974b79fcd9473 + checksum/config: f374556b2956b17609750df53e835c2d4022a21fcb30ab2c2f5115a13cc210bb kubectl.kubernetes.io/default-container: otel-collector spec: hostNetwork: true @@ -43,6 +43,27 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/master initContainers: + # Previously, fluentd checkpoints were written to /var/log directory. + # So, /var/log directory could not be mounted as read-only. + # Now, default fluentd checkpoint is moved to /var/addon/splunk/fluent_pos directory + # move-fluent-checkpoint will handle 2 scenarios + # - When upgrading from fluentd to fluentd, it will move checkpoint, so fluentd can continue from where it left + # - When migrating from fluentd to otel, it will move checkpoint, and then migrate to otel checkpoint. So, it + # covers both previous and current fluentd checkpoint directory. + - name: move-fluent-checkpoint + image: registry.access.redhat.com/ubi9/ubi + imagePullPolicy: IfNotPresent + command: ['sh', '-c', ' + mkdir -p /var/addon/splunk/fluent_pos; + find /var/log -maxdepth 1 -name *.json | xargs -I{} mv {} -t /var/addon/splunk/fluent_pos; + find /var/log -maxdepth 1 -name *.pos | xargs -I{} mv {} -t /var/addon/splunk/fluent_pos;'] + securityContext: + runAsUser: 0 + volumeMounts: + - name: fluentd-checkpoint-dir + mountPath: /var/addon/splunk/fluent_pos + - name: varlog + mountPath: /var/log - name: prepare-fluentd-config image: splunk/fluentd-hec:1.2.8 imagePullPolicy: IfNotPresent @@ -97,6 +118,7 @@ spec: volumeMounts: - name: varlog mountPath: /var/log + readOnly: true - name: varlogdest mountPath: /var/lib/docker/containers readOnly: true @@ -105,6 +127,8 @@ spec: readOnly: true - name: fluentd-config mountPath: /fluentd/etc + - name: fluentd-checkpoint-dir + mountPath: /var/addon/splunk/fluent_pos - name: tmp mountPath: /tmp - name: otel-collector @@ -128,6 +152,14 @@ spec: protocol: TCP image: quay.io/signalfx/splunk-otel-collector:0.67.0 imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault env: - name: SPLUNK_MEMORY_TOTAL_MIB value: "500" @@ -180,6 +212,14 @@ spec: name: otel-configmap terminationGracePeriodSeconds: 600 volumes: + - name: checkpoint + hostPath: + path: /var/addon/splunk/otel_pos + type: DirectoryOrCreate + - name: fluentd-checkpoint-dir + hostPath: + path: /var/addon/splunk/fluent_pos + type: DirectoryOrCreate - name: varlog hostPath: path: /var/log diff --git a/rendered/manifests/metrics-only/daemonset.yaml b/rendered/manifests/metrics-only/daemonset.yaml index 2c205705e0..be1672054c 100644 --- a/rendered/manifests/metrics-only/daemonset.yaml +++ b/rendered/manifests/metrics-only/daemonset.yaml @@ -63,6 +63,14 @@ spec: protocol: TCP image: quay.io/signalfx/splunk-otel-collector:0.67.0 imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault env: - name: SPLUNK_MEMORY_TOTAL_MIB value: "500" diff --git a/rendered/manifests/otel-logs/daemonset.yaml b/rendered/manifests/otel-logs/daemonset.yaml index eb8ebd5fea..306d809186 100644 --- a/rendered/manifests/otel-logs/daemonset.yaml +++ b/rendered/manifests/otel-logs/daemonset.yaml @@ -42,6 +42,27 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/master initContainers: + # Previously, fluentd checkpoints were written to /var/log directory. + # So, /var/log directory could not be mounted as read-only. + # Now, default fluentd checkpoint is moved to /var/addon/splunk/fluent_pos directory + # move-fluent-checkpoint will handle 2 scenarios + # - When upgrading from fluentd to fluentd, it will move checkpoint, so fluentd can continue from where it left + # - When migrating from fluentd to otel, it will move checkpoint, and then migrate to otel checkpoint. So, it + # covers both previous and current fluentd checkpoint directory. + - name: move-fluent-checkpoint + image: registry.access.redhat.com/ubi9/ubi + imagePullPolicy: IfNotPresent + command: ['sh', '-c', ' + mkdir -p /var/addon/splunk/fluent_pos; + find /var/log -maxdepth 1 -name *.json | xargs -I{} mv {} -t /var/addon/splunk/fluent_pos; + find /var/log -maxdepth 1 -name *.pos | xargs -I{} mv {} -t /var/addon/splunk/fluent_pos;'] + securityContext: + runAsUser: 0 + volumeMounts: + - name: fluentd-checkpoint-dir + mountPath: /var/addon/splunk/fluent_pos + - name: varlog + mountPath: /var/log - name: migrate-checkpoint image: quay.io/signalfx/splunk-otel-collector:0.67.0 imagePullPolicy: IfNotPresent @@ -50,15 +71,15 @@ spec: runAsUser: 0 env: - name: CONTAINER_LOG_PATH_FLUENTD - value: "/var/log/splunk-fluentd-containers.log.pos" + value: "/var/addon/splunk/fluent_pos/splunk-fluentd-containers.log.pos" - name: CONTAINER_LOG_PATH_OTEL value: "/var/addon/splunk/otel_pos/receiver_filelog_" - name: CUSTOM_LOG_PATH_FLUENTD - value: "/var/log/splunk-fluentd-*.pos" + value: "/var/addon/splunk/fluent_pos/splunk-fluentd-*.pos" - name: CUSTOM_LOG_PATH_OTEL value: "/var/addon/splunk/otel_pos/receiver_filelog_" - name: CUSTOM_LOG_CAPTURE_REGEX - value: '\/var\/log\/splunk\-fluentd\-(?P[\w0-9-_]+)\.pos' + value: '\/var\/addon\/splunk\/fluent_pos\/splunk\-fluentd\-(?P[\w0-9-_]+)\.pos' - name: JOURNALD_LOG_PATH_FLUENTD value: "/var/log/splunkd-fluentd-journald-*.pos.json" - name: JOURNALD_LOG_PATH_OTEL @@ -72,6 +93,8 @@ spec: mountPath: /var/log - name: varlibdockercontainers mountPath: /var/lib/docker/containers + - name: fluentd-checkpoint-dir + mountPath: /var/addon/splunk/fluent_pos containers: - name: otel-collector command: @@ -115,7 +138,13 @@ spec: image: quay.io/signalfx/splunk-otel-collector:0.67.0 imagePullPolicy: IfNotPresent securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsNonRoot: false runAsUser: 0 + seccompProfile: + type: RuntimeDefault env: - name: SPLUNK_MEMORY_TOTAL_MIB value: "500" @@ -212,16 +241,20 @@ spec: mountPath: /var/addon/splunk/otel_pos terminationGracePeriodSeconds: 600 volumes: + - name: checkpoint + hostPath: + path: /var/addon/splunk/otel_pos + type: DirectoryOrCreate + - name: fluentd-checkpoint-dir + hostPath: + path: /var/addon/splunk/fluent_pos + type: DirectoryOrCreate - name: varlog hostPath: path: /var/log - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - - name: checkpoint - hostPath: - path: /var/addon/splunk/otel_pos - type: DirectoryOrCreate - name: host-dev hostPath: path: /dev diff --git a/rendered/manifests/traces-only/daemonset.yaml b/rendered/manifests/traces-only/daemonset.yaml index 8d81031919..8227e17f63 100644 --- a/rendered/manifests/traces-only/daemonset.yaml +++ b/rendered/manifests/traces-only/daemonset.yaml @@ -75,6 +75,14 @@ spec: protocol: TCP image: quay.io/signalfx/splunk-otel-collector:0.67.0 imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault env: - name: SPLUNK_MEMORY_TOTAL_MIB value: "500"