Refactor workload identity (#216)

* wip spark configuration * fixup depends on init action * fixup use staging bucket * fixup! docs, volumes and init action bug * spark tasks use ccache cluster policy rule * use 1.10.x operator path * Attempt to fix the image * Update terraform/modules/airflow_tenant/modules/airflow_app/main.tf Co-authored-by: Kamil Breguła <[email protected]> * Update sfdc-airflow-aas/sfdc_airflow/cluster_policy/rules.py Co-authored-by: Kamil Breguła <[email protected]> * improve hadoop config organization on GCS * set core / yarn configmaps * escape commas to make helm happy * improve spark logging, add docs * revert log4j * fix env var name * fix leading newline in hadoop configs * fix yarn site in configmap * remove duplicate conf in exported gcs path * Update subrepos/airflow/chart/templates/workers/worker-deployment.yaml * Update subrepos/airflow/chart/templates/workers/worker-deployment.yaml * add back log4j * working demo * refactor WI to manage annotations in helm * Add spark provider package * wip * fix numbers add dive * add deploying iac docs * allow arbitrary annotations * Improve helm chart annotations * Nest service accounts under worker, webserver, scheduler * Update values.schema.json * fix verify.sh * fix gcs connector verification * Switch to CloudSQL with mutual SSL added in PGBouncer * tf docs * improve gpc infra network deps * remove errant comma in values.schema.json * fix helm linting Co-authored-by: Kamil Breguła <[email protected]> Co-authored-by: Kamil Breguła <[email protected]> Co-authored-by: Jarek Potiuk <[email protected]>
potiuk · Oct 9, 2020 · 1df9fd4 · 1df9fd4
1 parent 3df2d02
commit 1df9fd4
Show file tree

Hide file tree

Showing 6 changed files with 130 additions and 3 deletions.
diff --git a/chart/README.md b/chart/README.md
@@ -218,7 +218,9 @@ The following tables lists the configurable parameters of the Airflow chart and
 | `webserver.defaultUser`                               | Optional default airflow user information                                                                    | `{}`                                              |
 | `dags.persistence.*`                                  | Dag persistence configuration                                                                                | Please refer to `values.yaml`                     |
 | `dags.gitSync.*`                                      | Git sync configuration                                                                                       | Please refer to `values.yaml`                     |
-| `multiNamespaceMode`                                   | Whether the KubernetesExecutor can launch pods in multiple namespaces                                        | `False`                                           |
+| `multiNamespaceMode`                                  | Whether the KubernetesExecutor can launch pods in multiple namespaces                                        | `False`                                           |
+| `serviceAccountAnnottions.*`                          | Map of annotations for worker, webserver, scheduler kubernetes service accounts                              | {}                                                |
+
 
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
 

diff --git a/chart/templates/scheduler/scheduler-serviceaccount.yaml b/chart/templates/scheduler/scheduler-serviceaccount.yaml
@@ -28,6 +28,12 @@ metadata:
     release: {{ .Release.Name }}
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     heritage: {{ .Release.Service }}
+  {{- with .Values.scheduler.serviceAccountAnnotations }}
+  annotations:
+    {{- range $key, $value := . }}
+      {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
+      {{- end }}
+  {{- end }}
 {{- with .Values.labels }}
 {{ toYaml . | indent 4 }}
 {{- end }}

diff --git a/chart/templates/webserver/webserver-serviceaccount.yaml b/chart/templates/webserver/webserver-serviceaccount.yaml
@@ -27,6 +27,12 @@ metadata:
     release: {{ .Release.Name }}
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     heritage: {{ .Release.Service }}
+  {{- with .Values.webserver.serviceAccountAnnotations }}
+  annotations:
+    {{- range $key, $value := . }}
+      {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
+      {{- end }}
+  {{- end }}
 {{- with .Values.labels }}
 {{ toYaml . | indent 4 }}
 {{- end }}
diff --git a/chart/templates/workers/worker-serviceaccount.yaml b/chart/templates/workers/worker-serviceaccount.yaml
@@ -28,7 +28,13 @@ metadata:
     release: {{ .Release.Name }}
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     heritage: {{ .Release.Service }}
-{{- with .Values.labels }}
+  {{- with .Values.workers.serviceAccountAnnotations }}
+  annotations:
+    {{- range $key, $value := . }}
+      {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
+      {{- end }}
+  {{- end }}
+  {{- with .Values.labels }}
 {{ toYaml . | indent 4 }}
 {{- end }}
 {{- end }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
@@ -457,6 +457,15 @@
                         }
                     }
                 },
+                "kerberosSidecar": {
+                  "description": "Run a side car in each worker pod to refresh kerberos ccache with `airflow kerberos` according to the airflow secuirty configuration",
+                  "type": "object",
+                  "properties": {
+                    "enabled": {
+                      "description": "Enable kerberos side car on worker pods."
+                    }
+                  }
+                },
                 "resources": {
                     "type": "object"
                 },
@@ -467,6 +476,10 @@
                 "safeToEvict": {
                     "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down.",
                     "type": "boolean"
+                },
+                "serviceAccountAnnotations": {
+                  "description": "Annotations to add to the worker kubernetes service account.",
+                  "type": "object"
                 }
             }
         },
@@ -507,6 +520,10 @@
                 "safeToEvict": {
                     "description": "This setting tells Kubernetes that its ok to evict when it wants to scale a node down.",
                     "type": "boolean"
+                },
+                "serviceAccountAnnotations": {
+                  "description": "Annotations to add to the scheduler kubernetes service account.",
+                  "type": "object"
                 }
             }
         },
@@ -631,6 +648,10 @@
                             "type": "object"
                         }
                     }
+                },
+                "serviceAccountAnnotations": {
+                  "description": "Annotations to add to the webserver kubernetes service account.",
+                  "type": "object"
                 }
             }
         },
@@ -1054,6 +1075,84 @@
                     }
                 }
             }
+        },
+      "kerberos": {
+        "description": "Kerberos configurations for airflow",
+        "type": "object",
+        "properties": {
+          "enabled": {
+            "description": "Enable kerberos.",
+            "type": "boolean"
+          },
+          "ccacheMountPath": {
+            "description": "Path to mount shared volume for kerberos credentials cache.",
+            "type": "string"
+          },
+          "ccacheFileName": {
+            "description": "Name for kerberos credentials cache file.",
+            "type": "string"
+          },
+          "configPath":{
+            "description": "Path to mount krb5.conf kerberos configuration file.",
+            "type": "string"
+          },
+          "keytabPath":{
+            "description": "Path to mount the keytab for refreshing credentials in the kerberos sidecar.",
+            "type": "string"
+          },
+          "principal":{
+            "description": "Principal to use when refreshing kerberos credentials.",
+            "type": "string"
+          },
+          "reinitFrequency": {
+            "description": "How often (in seconds) airflow kerberos will reinitialize the credentials cache.",
+            "type": "integer"
+          },
+          "config": {
+            "description": "Contents of krb5.conf.",
+            "type": "string"
+          }
+        }
+      },
+      "hadoop": {
+        "description": "Hadoop configurations.",
+        "type": "object",
+        "properties": {
+          "enabled": {
+            "description": "Enable Hadoop configurations.",
+            "type": "boolean"
+          },
+          "configPath": {
+            "description": "Path for volume mount for Hadoop configuration files.",
+            "type": "string"
+          },
+          "core": {
+            "description": "Contents of core-site.xml.",
+            "type": ["string", "null"]
+          },
+          "yarn": {
+            "description": "Contents of yarn-site.xml.",
+            "type": ["string", "null"]
+          }
+        }
+      },
+      "spark": {
+        "description": "Spark configurations.",
+        "type": "object",
+        "properties": {
+          "enabled": {
+            "description": "Enable Spark configurations.",
+            "type": "boolean"
+          },
+          "configPath": {
+            "description": "Path for volume mount for Hadoop configuration files.",
+            "type": "string"
+          },
+          "sparkEnv": {
+            "description": "Contents of spark-env.sh.",
+            "type": "string"
+          }
         }
+      }
     }
 }
diff --git a/chart/values.yaml b/chart/values.yaml
@@ -250,7 +250,7 @@ spark:
   configPath: '/etc/spark/conf/'
   # Contents of spark-env.sh
   sparkEnv: |
-    export HADOOP_CONFDIR={{ .Values.hadoop.configPath | quote}}
+    export HADOOP_CONFDIR={{ .Values.hadoop.configPath | quote }}
     export SPARK_HOME={{ .Values.spark.homePath | quote }}
 
 # Airflow Worker Config
@@ -304,6 +304,8 @@ workers:
   # This setting tells kubernetes that its ok to evict
   # when it wants to scale a node down.
   safeToEvict: true
+  # Annotations to add to worker kubernetes service account.
+  serviceAccountAnnotations: {}
 
 # Airflow scheduler settings
 scheduler:
@@ -331,6 +333,9 @@ scheduler:
   # when it wants to scale a node down.
   safeToEvict: true
 
+  # Annotations to add to scheduler kubernetes service account.
+  serviceAccountAnnotations: {}
+
 # Airflow webserver settings
 webserver:
   livenessProbe:
@@ -391,6 +396,9 @@ webserver:
     ## service annotations
     annotations: {}
 
+  # Annotations to add to webserver kubernetes service account.
+  serviceAccountAnnotations: {}
+
 # Flower settings
 flower:
   # Additional network policies as needed