From e79cfbd4d02e8d15d4ee0f056894abd256d18d46 Mon Sep 17 00:00:00 2001 From: Demethyl Date: Fri, 31 Dec 2021 16:25:00 +0100 Subject: [PATCH 1/7] AIRFLOW-20533: add airflow.cfg as volumeMounts on initContainer wait-for-airflow-migrations This allows the init containers to work with secretbackend externalized sql alchemy urls for instance. --- chart/templates/scheduler/scheduler-deployment.yaml | 5 +++++ chart/templates/triggerer/triggerer-deployment.yaml | 5 +++++ chart/templates/webserver/webserver-deployment.yaml | 5 +++++ chart/templates/workers/worker-deployment.yaml | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index d7a5e1a0ba509..5f3e01d3bbb39 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -110,6 +110,11 @@ spec: {{ toYaml .Values.scheduler.resources | indent 12 }} image: {{ template "airflow_image_for_migrations" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + volumeMounts: + - name: config + mountPath: {{ template "airflow_config_path" . }} + subPath: airflow.cfg + readOnly: true args: {{- include "wait-for-migrations-command" . | indent 10 }} envFrom: diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index 11ccdc6f8d65c..d7c93552e8876 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -93,6 +93,11 @@ spec: {{- toYaml .Values.triggerer.resources | nindent 12 }} image: {{ template "airflow_image_for_migrations" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + volumeMounts: + - name: config + mountPath: { { template "airflow_config_path" . } } + subPath: airflow.cfg + readOnly: true args: {{- include "wait-for-migrations-command" . | nindent 10 }} envFrom: diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index 8aeb91bb04c54..129117a80d0f8 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -105,6 +105,11 @@ spec: {{ toYaml .Values.webserver.resources | indent 12 }} image: {{ template "airflow_image_for_migrations" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + volumeMounts: + - name: config + mountPath: {{ template "airflow_config_path" . }} + subPath: airflow.cfg + readOnly: true args: {{- include "wait-for-migrations-command" . | indent 10 }} envFrom: diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 38c50ceede959..6610c65fb8626 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -124,6 +124,11 @@ spec: {{ toYaml .Values.workers.resources | indent 12 }} image: {{ template "airflow_image_for_migrations" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + volumeMounts: + - name: config + mountPath: {{ template "airflow_config_path" . }} + subPath: airflow.cfg + readOnly: true args: {{- include "wait-for-migrations-command" . | indent 10 }} envFrom: From 73ad0595ffd7d13425271ba0c83b6fd61f02c9f5 Mon Sep 17 00:00:00 2001 From: demetthyl Date: Fri, 31 Dec 2021 17:32:14 +0100 Subject: [PATCH 2/7] delete useless spaces breaking the templating Co-authored-by: Kaxil Naik --- chart/templates/triggerer/triggerer-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index d7c93552e8876..dcc03c8e21cc9 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -95,7 +95,7 @@ spec: imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} volumeMounts: - name: config - mountPath: { { template "airflow_config_path" . } } + mountPath: {{ template "airflow_config_path" . }} subPath: airflow.cfg readOnly: true args: From ee85010bec52eb1b75c16fc0bc1bc96c53b6d135 Mon Sep 17 00:00:00 2001 From: Demethyl Date: Mon, 17 Jan 2022 13:33:14 +0100 Subject: [PATCH 3/7] AIRFLOW-20533: add test checking volumeMount on initcontainer --- chart/tests/test_airflow_common.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/chart/tests/test_airflow_common.py b/chart/tests/test_airflow_common.py index a54cf082a7840..e847dc501b861 100644 --- a/chart/tests/test_airflow_common.py +++ b/chart/tests/test_airflow_common.py @@ -188,3 +188,22 @@ def test_have_all_variables(self): assert variables == jmespath.search( "spec.template.spec.containers[0].env[*].name", doc ), f"Wrong vars in {component}" + + def test_have_all_config_mounts_on_init_containers(self, expected_read_only): + docs = render_chart( + values={}, + show_only=[ + "templates/scheduler/scheduler-deployment.yaml", + "templates/workers/worker-deployment.yaml", + "templates/webserver/webserver-deployment.yaml", + "templates/triggerer/triggerer-deployment.yaml", + ], + ) + assert 4 == len(docs) + for doc in docs: + expected_mount = { + "subPath": "airflow.cfg", + "name": "config", + "readOnly": expected_read_only, + } + assert expected_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", doc) From 7ee08571de6d503b1af545c6b87de88a4103b3e3 Mon Sep 17 00:00:00 2001 From: Demethyl Date: Mon, 17 Jan 2022 18:38:09 +0100 Subject: [PATCH 4/7] AIRFLOW-20533: remove method parameter, add mountPath --- chart/tests/test_airflow_common.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chart/tests/test_airflow_common.py b/chart/tests/test_airflow_common.py index c1f8afe1736c9..b6ee6fa47b8ee 100644 --- a/chart/tests/test_airflow_common.py +++ b/chart/tests/test_airflow_common.py @@ -249,7 +249,7 @@ def test_have_all_variables(self): "spec.template.spec.containers[0].env[*].name", doc ), f"Wrong vars in {component}" - def test_have_all_config_mounts_on_init_containers(self, expected_read_only): + def test_have_all_config_mounts_on_init_containers(self): docs = render_chart( values={}, show_only=[ @@ -264,6 +264,7 @@ def test_have_all_config_mounts_on_init_containers(self, expected_read_only): expected_mount = { "subPath": "airflow.cfg", "name": "config", - "readOnly": expected_read_only, + "readOnly": True, + "mountPath": "/opt/airflow" } - assert expected_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", doc) + assert expected_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", dqoc) From 9e4180b2d34b36aff11a8b64c3b37385d27472f1 Mon Sep 17 00:00:00 2001 From: demetthyl Date: Tue, 18 Jan 2022 22:36:11 +0100 Subject: [PATCH 5/7] Update chart/tests/test_airflow_common.py Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- chart/tests/test_airflow_common.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/chart/tests/test_airflow_common.py b/chart/tests/test_airflow_common.py index b6ee6fa47b8ee..edc7380394dcb 100644 --- a/chart/tests/test_airflow_common.py +++ b/chart/tests/test_airflow_common.py @@ -260,11 +260,11 @@ def test_have_all_config_mounts_on_init_containers(self): ], ) assert 4 == len(docs) + expected_mount = { + "subPath": "airflow.cfg", + "name": "config", + "readOnly": True, + "mountPath": "/opt/airflow" + } for doc in docs: - expected_mount = { - "subPath": "airflow.cfg", - "name": "config", - "readOnly": True, - "mountPath": "/opt/airflow" - } assert expected_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", dqoc) From 93df6e96893b45f49130ad773441d485155c4b08 Mon Sep 17 00:00:00 2001 From: demetthyl Date: Tue, 18 Jan 2022 22:36:46 +0100 Subject: [PATCH 6/7] Update chart/tests/test_airflow_common.py Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- chart/tests/test_airflow_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/tests/test_airflow_common.py b/chart/tests/test_airflow_common.py index edc7380394dcb..9fce39a04a994 100644 --- a/chart/tests/test_airflow_common.py +++ b/chart/tests/test_airflow_common.py @@ -267,4 +267,4 @@ def test_have_all_config_mounts_on_init_containers(self): "mountPath": "/opt/airflow" } for doc in docs: - assert expected_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", dqoc) + assert expected_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", doc) From 410de283852111991c1862ca55b7ca38741268d2 Mon Sep 17 00:00:00 2001 From: demetthyl Date: Wed, 19 Jan 2022 08:49:50 +0100 Subject: [PATCH 7/7] as suggested, adding filePath into whole path Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- chart/tests/test_airflow_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/tests/test_airflow_common.py b/chart/tests/test_airflow_common.py index 9fce39a04a994..de8fb896440f3 100644 --- a/chart/tests/test_airflow_common.py +++ b/chart/tests/test_airflow_common.py @@ -264,7 +264,7 @@ def test_have_all_config_mounts_on_init_containers(self): "subPath": "airflow.cfg", "name": "config", "readOnly": True, - "mountPath": "/opt/airflow" + "mountPath": "/opt/airflow/airflow.cfg", } for doc in docs: assert expected_mount in jmespath.search("spec.template.spec.initContainers[0].volumeMounts", doc)