From 232821de96792e6c6fd2f4ec9e3a3b59f648fea8 Mon Sep 17 00:00:00 2001
From: Khor Shu Heng <32997938+khorshuheng@users.noreply.github.com>
Date: Tue, 26 Jan 2021 12:56:48 +0800
Subject: [PATCH] Enable user to provide spark job template as input for
 jobservice deployment (#1285)

* Enable user to provide spark job template as input for jobservice deployment

Signed-off-by: Khor Shu Heng <khor.heng@gojek.com>

* Peg mypy version for CI

Signed-off-by: Khor Shu Heng <khor.heng@gojek.com>

Co-authored-by: Khor Shu Heng <khor.heng@gojek.com>
---
 .../feast-jobservice/templates/configmap.yaml   | 16 ++++++++++++++++
 .../feast-jobservice/templates/deployment.yaml  | 17 +++++++++++++++--
 .../feast/charts/feast-jobservice/values.yaml   |  9 +++++++++
 sdk/python/requirements-ci.txt                  |  2 +-
 4 files changed, 41 insertions(+), 3 deletions(-)
 create mode 100644 infra/charts/feast/charts/feast-jobservice/templates/configmap.yaml

diff --git a/infra/charts/feast/charts/feast-jobservice/templates/configmap.yaml b/infra/charts/feast/charts/feast-jobservice/templates/configmap.yaml
new file mode 100644
index 0000000000..356358fd3d
--- /dev/null
+++ b/infra/charts/feast/charts/feast-jobservice/templates/configmap.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.sparkOperator.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "feast-jobservice.fullname" . }}-spark-template
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "feast-jobservice.name" . }}
+    component: jobservice
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  jobTemplate.yaml: |
+{{- toYaml .Values.sparkOperator.jobTemplate | nindent 4 }}
+{{- end }}
\ No newline at end of file
diff --git a/infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml b/infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml
index 00667ccbb0..506d8a5cfd 100644
--- a/infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml
+++ b/infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml
@@ -37,13 +37,18 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
 
-      {{- if .Values.secrets }}
+      {{- if or .Values.secrets .Values.sparkOperator.enabled }}
       volumes:
+      {{- end }}
       {{- range $secret := .Values.secrets }}
       - name: {{ $secret }}
         secret:
           secretName: {{ $secret }}
       {{- end }}
+      {{- if .Values.sparkOperator.enabled }}
+      - name: {{ template "feast-jobservice.fullname" . }}-spark-template
+        configMap:
+          name: {{ template "feast-jobservice.fullname" . }}-spark-template
       {{- end }}
 
       containers:
@@ -51,16 +56,24 @@ spec:
         image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
 
-        {{- if .Values.secrets }}
+        {{- if or .Values.secrets .Values.sparkOperator.enabled }}
         volumeMounts:
+        {{- end }}
         {{- range $secret := .Values.secrets }}
         - name: {{ $secret }}
           mountPath: "/etc/secrets/{{ $secret }}"
           readOnly: true
         {{- end }}
+        {{- if .Values.sparkOperator.enabled }}
+        - name: {{ template "feast-jobservice.fullname" . }}-spark-template
+          mountPath: "/etc/configs"
         {{- end }}
 
         env:
+        {{- if .Values.sparkOperator.enabled }}
+        - name: FEAST_SPARK_K8S_JOB_TEMPLATE_PATH
+          value: /etc/configs/jobTemplate.yaml
+        {{- end }}
         {{- range $key, $value := .Values.envOverrides }}
         - name: {{ printf "%s" $key | replace "." "_" | upper | quote }}
           {{- if eq (kindOf $value) "map" }}
diff --git a/infra/charts/feast/charts/feast-jobservice/values.yaml b/infra/charts/feast/charts/feast-jobservice/values.yaml
index 03b8296a26..c069211a76 100644
--- a/infra/charts/feast/charts/feast-jobservice/values.yaml
+++ b/infra/charts/feast/charts/feast-jobservice/values.yaml
@@ -21,6 +21,12 @@ gcpServiceAccount:
 # gcpProjectId -- Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow
 gcpProjectId: ""
 
+sparkOperator:
+  # sparkOperator.enabled -- Flag to create and mount custom job template on the jobservice deployment as configmap
+  enabled: false
+  # sparkOperator.jobTemplate -- Content of the job template, in yaml format
+  jobTemplate: {}
+
 prometheus:
   # prometheus.enabled -- Flag to enable scraping of metrics
   enabled: true
@@ -124,3 +130,6 @@ envOverrides: {}
 
 # podLabels -- Labels to be added to Feast Job Service pods
 podLabels: {}
+
+# secrets -- Arbitrary secrets to mount on the job service pod, on /etc/secrets/<secret name>
+secrets: []
diff --git a/sdk/python/requirements-ci.txt b/sdk/python/requirements-ci.txt
index 8897021099..e3a7fd2d95 100644
--- a/sdk/python/requirements-ci.txt
+++ b/sdk/python/requirements-ci.txt
@@ -9,7 +9,7 @@ pandas~=1.0.0
 mock==2.0.0
 pandavro==1.5.*
 moto
-mypy
+mypy==0.790
 mypy-protobuf
 avro==1.10.0
 gcsfs