Attaching a logGroomerSidecar to dag processor

chart/templates/dag-processor/dag-processor-deployment.yaml

@@ -176,6 +176,35 @@ spec:
         {{- if .Values.dagProcessor.extraContainers }}
           {{- toYaml .Values.dagProcessor.extraContainers | nindent 8 }}
         {{- end }}
+        {{- if .Values.dagProcessor.logGroomerSidecar.enabled }}
+        - name: dag-processor-log-groomer
+          resources: {{- toYaml .Values.dagProcessor.logGroomerSidecar.resources | nindent 12 }}
+          image: {{ template "airflow_image" . }}
+          imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
+          {{- if .Values.dagProcessor.logGroomerSidecar.command }}
+          command: {{ tpl (toYaml .Values.dagProcessor.logGroomerSidecar.command) . | nindent 12 }}
+          {{- end }}
+          {{- if .Values.dagProcessor.logGroomerSidecar.args }}
+          args: {{- tpl (toYaml .Values.dagProcessor.logGroomerSidecar.args) . | nindent 12 }}
+          {{- end }}
+          {{- if .Values.dagProcessor.logGroomerSidecar.retentionDays }}
+          env:
+            - name: AIRFLOW__LOG_RETENTION_DAYS
+              value: "{{ .Values.dagProcessor.logGroomerSidecar.retentionDays }}"
+          {{- end }}
+          volumeMounts:
+            - name: logs
+              mountPath: {{ template "airflow_logs" . }}
+            {{- if .Values.volumeMounts }}
+              {{- toYaml .Values.volumeMounts | nindent 12 }}
+            {{- end }}
+            {{- if .Values.dagProcessor.extraVolumeMounts }}
+              {{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
+            {{- end }}
+            {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
+              {{- include "airflow_webserver_config_mount" . | nindent 12 }}
+            {{- end }}
+          {{- end }}
       volumes:
         - name: config
           configMap:

chart/values.schema.json

@@ -2697,6 +2697,10 @@
                         }
                     ]
                 },
+                "logGroomerSidecar": {
+                    "$ref": "#/definitions/logGroomerConfigType",
+                    "description": "Configuration for log groomer sidecar"
+                },
                 "waitForMigrations": {
                     "description": "wait-for-airflow-migrations init container.",
                     "type": "object",

chart/values.yaml

@@ -1357,6 +1357,23 @@ dagProcessor:
 
   podAnnotations: {}
 
+  logGroomerSidecar:
+    # Whether to deploy the Airflow dag processor log groomer sidecar.
+    enabled: true
+    # Command to use when running the Airflow dag processor log groomer sidecar (templated).
+    command: ~
+    # Args to use when running the Airflow dag processor log groomer sidecar (templated).
+    args: ["bash", "/clean-logs"]
+    # Number of days to retain logs
+    retentionDays: 15
+    resources: {}
+    #  limits:
+    #   cpu: 100m
+    #   memory: 128Mi
+    #  requests:
+    #   cpu: 100m
+    #   memory: 128Mi
+
   waitForMigrations:
     # Whether to create init container to wait for db migrations
     enabled: true

tests/charts/test_dag_processor.py

@@ -20,6 +20,7 @@
 import pytest
 
 from tests.charts.helm_template_generator import render_chart
+from tests.charts.log_groomer import LogGroomerTestBase
 
 
 class TestDagProcessor:
@@ -549,3 +550,8 @@ def test_should_add_component_specific_annotations(self):
         )
         assert "annotations" in jmespath.search("metadata", docs[0])
         assert jmespath.search("metadata.annotations", docs[0])["test_annotation"] == "test_annotation_value"
+
+
+class TestDagProcessorLogGroomer(LogGroomerTestBase):
+    obj_name = "dag-processor"
+    folder = "dag-processor"