Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix chart DAG gitsync and persistence with KubernetesExecutor #15657

Merged
merged 1 commit into from
May 4, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 2 additions & 8 deletions chart/files/pod-template-file.kubernetes-helm-yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ kind: Pod
metadata:
name: dummy-name
spec:
{{- if .Values.dags.gitSync.enabled }}
{{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }}
initContainers:
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true") | indent 4 }}
{{- end }}
Expand Down Expand Up @@ -61,17 +61,11 @@ spec:
name: git-sync-ssh-key
subPath: ssh
{{- end }}
{{- if .Values.dags.persistence.enabled }}
{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}
- mountPath: {{ include "airflow_dags_mount_path" . }}
name: dags
readOnly: true
{{- end }}
{{- if .Values.dags.gitSync.enabled }}
- mountPath: {{ include "airflow_dags" . }}
name: dags
readOnly: true
subPath: {{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subPath }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{ toYaml .Values.workers.extraVolumeMounts | indent 8 }}
{{- end }}
Expand Down
2 changes: 1 addition & 1 deletion chart/templates/workers/worker-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ spec:
subPath: airflow_local_settings.py
readOnly: true
{{- end }}
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
- name: dags
mountPath: {{ template "airflow_dags_mount_path" . }}
{{- end }}
Expand Down
61 changes: 60 additions & 1 deletion chart/tests/test_pod_template_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from shutil import copyfile

import jmespath
from parameterized import parameterized

from tests.helm_template_generator import render_chart

Expand Down Expand Up @@ -101,6 +102,56 @@ def test_should_add_an_init_container_if_git_sync_is_true(self):
"volumeMounts": [{"mountPath": "/git-root", "name": "dags"}],
} == jmespath.search("spec.initContainers[0]", docs[0])

def test_should_not_add_init_container_if_dag_persistence_is_true(self):
docs = render_chart(
values={
"dags": {
"persistence": {"enabled": True},
"gitSync": {"enabled": True},
}
},
show_only=["templates/pod-template-file.yaml"],
)

assert jmespath.search("spec.initContainers", docs[0]) is None

@parameterized.expand(
[
({"gitSync": {"enabled": True}},),
({"persistence": {"enabled": True}},),
(
{
"gitSync": {"enabled": True},
"persistence": {"enabled": True},
},
),
]
)
def test_dags_mount(self, dag_values):
docs = render_chart(
values={"dags": dag_values},
show_only=["templates/pod-template-file.yaml"],
)

assert {"mountPath": "/opt/airflow/dags", "name": "dags", "readOnly": True} in jmespath.search(
"spec.containers[0].volumeMounts", docs[0]
)

def test_dags_mount_with_gitsync_and_persistence(self):
docs = render_chart(
values={
"dags": {
"gitSync": {"enabled": True},
"persistence": {"enabled": True},
}
},
show_only=["templates/pod-template-file.yaml"],
)

assert {"mountPath": "/opt/airflow/dags", "name": "dags", "readOnly": True} in jmespath.search(
"spec.containers[0].volumeMounts", docs[0]
)

def test_validate_if_ssh_params_are_added(self):
docs = render_chart(
values={
Expand Down Expand Up @@ -182,7 +233,7 @@ def test_should_set_username_and_pass_env_variables(self):
"valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GIT_SYNC_PASSWORD"}},
} in jmespath.search("spec.initContainers[0].env", docs[0])

def test_should_set_the_volume_claim_correctly_when_using_an_existing_claim(self):
def test_should_set_the_dags_volume_claim_correctly_when_using_an_existing_claim(self):
docs = render_chart(
values={"dags": {"persistence": {"enabled": True, "existingClaim": "test-claim"}}},
show_only=["templates/pod-template-file.yaml"],
Expand All @@ -192,6 +243,14 @@ def test_should_set_the_volume_claim_correctly_when_using_an_existing_claim(self
"spec.volumes", docs[0]
)

def test_should_use_empty_dir_for_gitsync_without_persistence(self):
docs = render_chart(
values={"dags": {"gitSync": {"enabled": True}}},
show_only=["templates/pod-template-file.yaml"],
)

assert {"name": "dags", "emptyDir": {}} in jmespath.search("spec.volumes", docs[0])

def test_should_set_a_custom_image_in_pod_template(self):
docs = render_chart(
values={"images": {"pod_template": {"repository": "dummy_image", "tag": "latest"}}},
Expand Down