Fix webhook rbac and add istio tests

SeldonIO · Dec 16, 2019 · e12524f · e12524f
1 parent 444fc04
commit e12524f
Show file tree

Hide file tree

Showing 16 changed files with 25,086 additions and 107 deletions.
diff --git a/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml b/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml
@@ -69,6 +69,8 @@ spec:
           value: '{{ .Values.istio.enabled }}'
         - name: ISTIO_GATEWAY
           value: '{{ .Values.istio.gateway }}'
+        - name: ISTIO_TLS_MODE
+          value: '{{ .Values.istio.tlsMode }}'
         image: '{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}'
         imagePullPolicy: '{{ .Values.image.pullPolicy }}'
         name: manager

diff --git a/helm-charts/seldon-core-operator/templates/webhook.yaml b/helm-charts/seldon-core-operator/templates/webhook.yaml
@@ -61,13 +61,39 @@ webhooks:
     resources:
     - seldondeployments
 - clientConfig:
-    caBundle: Cg==
+    caBundle: '{{ $ca.Cert | b64enc }}'
     service:
       name: seldon-webhook-service
-      namespace: seldon-system
+      namespace: '{{ .Release.Namespace }}'
       path: /validate-machinelearning-seldon-io-v1alpha2-seldondeployment
   failurePolicy: Fail
   name: vseldondeployment.kb.io
+{{- if semverCompare ">=1.15.0" .Capabilities.KubeVersion.Version }}
+{{- if not .Values.singleNamespace }}
+  namespaceSelector:
+    matchExpressions:
+    - key: seldon.io/controller-id
+      operator: DoesNotExist
+{{- end }}
+{{- end }}
+{{- if .Values.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      seldon.io/controller-id: {{ .Release.Namespace }}
+{{- end }}
+{{- if semverCompare ">=1.15.0" .Capabilities.KubeVersion.Version }}
+{{- if not .Values.controllerId }}
+  objectSelector:
+    matchExpressions:
+    - key: seldon.io/controller-id
+      operator: DoesNotExist
+{{- end }}
+{{- end }}
+{{- if .Values.controllerId }}
+  objectSelector:
+    matchLabels:
+      seldon.io/controller-id: {{ .Values.controllerId }}
+{{- end }}
   rules:
   - apiGroups:
     - machinelearning.seldon.io
@@ -79,13 +105,39 @@ webhooks:
     resources:
     - seldondeployments
 - clientConfig:
-    caBundle: Cg==
+    caBundle: '{{ $ca.Cert | b64enc }}'
     service:
       name: seldon-webhook-service
-      namespace: seldon-system
+      namespace: '{{ .Release.Namespace }}'
       path: /validate-machinelearning-seldon-io-v1alpha3-seldondeployment
   failurePolicy: Fail
   name: vseldondeployment.kb.io
+{{- if semverCompare ">=1.15.0" .Capabilities.KubeVersion.Version }}
+{{- if not .Values.singleNamespace }}
+  namespaceSelector:
+    matchExpressions:
+    - key: seldon.io/controller-id
+      operator: DoesNotExist
+{{- end }}
+{{- end }}
+{{- if .Values.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      seldon.io/controller-id: {{ .Release.Namespace }}
+{{- end }}
+{{- if semverCompare ">=1.15.0" .Capabilities.KubeVersion.Version }}
+{{- if not .Values.controllerId }}
+  objectSelector:
+    matchExpressions:
+    - key: seldon.io/controller-id
+      operator: DoesNotExist
+{{- end }}
+{{- end }}
+{{- if .Values.controllerId }}
+  objectSelector:
+    matchLabels:
+      seldon.io/controller-id: {{ .Values.controllerId }}
+{{- end }}
   rules:
   - apiGroups:
     - machinelearning.seldon.io
@@ -98,20 +150,6 @@ webhooks:
     - seldondeployments
 ---
 
-{{- if not .Values.certManager.enabled -}}
-apiVersion: v1
-data:
-  ca.crt: '{{ $ca.Cert | b64enc }}'
-  tls.crt: '{{ $cert.Cert | b64enc }}'
-  tls.key: '{{ $cert.Key | b64enc }}'
-kind: Secret
-metadata:
-  name: seldon-webhook-server-cert
-  namespace: '{{ .Release.Namespace }}'
-type: kubernetes.io/tls
-{{- end }}
----
-
 apiVersion: admissionregistration.k8s.io/v1beta1
 kind: MutatingWebhookConfiguration
 metadata:
@@ -170,13 +208,39 @@ webhooks:
     resources:
     - seldondeployments
 - clientConfig:
-    caBundle: Cg==
+    caBundle: '{{ $ca.Cert | b64enc }}'
     service:
       name: seldon-webhook-service
-      namespace: seldon-system
+      namespace: '{{ .Release.Namespace }}'
       path: /mutate-machinelearning-seldon-io-v1alpha2-seldondeployment
   failurePolicy: Fail
   name: mseldondeployment.kb.io
+{{- if semverCompare ">=1.15.0" .Capabilities.KubeVersion.Version }}
+{{- if not .Values.singleNamespace }}
+  namespaceSelector:
+    matchExpressions:
+    - key: seldon.io/controller-id
+      operator: DoesNotExist
+{{- end }}
+{{- end }}
+{{- if .Values.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      seldon.io/controller-id: {{ .Release.Namespace }}
+{{- end }}
+{{- if semverCompare ">=1.15.0" .Capabilities.KubeVersion.Version }}
+{{- if not .Values.controllerId }}
+  objectSelector:
+    matchExpressions:
+    - key: seldon.io/controller-id
+      operator: DoesNotExist
+{{- end }}
+{{- end }}
+{{- if .Values.controllerId }}
+  objectSelector:
+    matchLabels:
+      seldon.io/controller-id: {{ .Values.controllerId }}
+{{- end }}
   rules:
   - apiGroups:
     - machinelearning.seldon.io
@@ -188,13 +252,39 @@ webhooks:
     resources:
     - seldondeployments
 - clientConfig:
-    caBundle: Cg==
+    caBundle: '{{ $ca.Cert | b64enc }}'
     service:
       name: seldon-webhook-service
-      namespace: seldon-system
+      namespace: '{{ .Release.Namespace }}'
       path: /mutate-machinelearning-seldon-io-v1alpha3-seldondeployment
   failurePolicy: Fail
   name: mseldondeployment.kb.io
+{{- if semverCompare ">=1.15.0" .Capabilities.KubeVersion.Version }}
+{{- if not .Values.singleNamespace }}
+  namespaceSelector:
+    matchExpressions:
+    - key: seldon.io/controller-id
+      operator: DoesNotExist
+{{- end }}
+{{- end }}
+{{- if .Values.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      seldon.io/controller-id: {{ .Release.Namespace }}
+{{- end }}
+{{- if semverCompare ">=1.15.0" .Capabilities.KubeVersion.Version }}
+{{- if not .Values.controllerId }}
+  objectSelector:
+    matchExpressions:
+    - key: seldon.io/controller-id
+      operator: DoesNotExist
+{{- end }}
+{{- end }}
+{{- if .Values.controllerId }}
+  objectSelector:
+    matchLabels:
+      seldon.io/controller-id: {{ .Values.controllerId }}
+{{- end }}
   rules:
   - apiGroups:
     - machinelearning.seldon.io
@@ -205,3 +295,17 @@ webhooks:
     - UPDATE
     resources:
     - seldondeployments
+---
+
+{{- if not .Values.certManager.enabled -}}
+apiVersion: v1
+data:
+  ca.crt: '{{ $ca.Cert | b64enc }}'
+  tls.crt: '{{ $cert.Cert | b64enc }}'
+  tls.key: '{{ $cert.Key | b64enc }}'
+kind: Secret
+metadata:
+  name: seldon-webhook-server-cert
+  namespace: '{{ .Release.Namespace }}'
+type: kubernetes.io/tls
+{{- end }}
diff --git a/helm-charts/seldon-core-operator/values.yaml b/helm-charts/seldon-core-operator/values.yaml
@@ -21,13 +21,14 @@ engine:
     name: default
   user: 8888
 image:
-  pullPolicy: Always
+  pullPolicy: IfNotPresent
   registry: docker.io
   repository: seldonio/seldon-core-operator
   tag: 0.5.2-SNAPSHOT
 istio:
   enabled: false
   gateway: seldon-gateway
+  tlsMode: ''
 predictiveUnit:
   port: 9000
 rbac:

diff --git a/operator/config/manager/manager.yaml b/operator/config/manager/manager.yaml
@@ -62,6 +62,8 @@ spec:
           value: "false"
         - name: ISTIO_GATEWAY
           value: seldon-gateway
+        - name: ISTIO_TLS_MODE
+          value: ""
         image: controller:latest
         name: manager
         resources:

diff --git a/operator/config/webhook/kustomization.yaml b/operator/config/webhook/kustomization.yaml
@@ -6,5 +6,16 @@ configurations:
 - kustomizeconfig.yaml
 
 # Comment this if you have a k8s cluster < 1.15 and want to use namespaced or labelled operators
-patchesStrategicMerge:
-- patch_object_selector.yaml
+patchesJson6902:
+- target:
+    group: admissionregistration.k8s.io
+    version: v1beta1
+    kind: MutatingWebhookConfiguration
+    name: mutating-webhook-configuration
+  path: patch_webhook.yaml
+- target:
+    group: admissionregistration.k8s.io
+    version: v1beta1
+    kind: ValidatingWebhookConfiguration
+    name: validating-webhook-configuration
+  path: patch_webhook.yaml
diff --git a/operator/config/webhook/patch_object_selector.yaml b/operator/config/webhook/patch_object_selector.yaml
diff --git a/operator/controllers/istio.go b/operator/controllers/istio.go
@@ -3,5 +3,6 @@ package controllers
 const (
 	ENV_ISTIO_ENABLED        = "ISTIO_ENABLED"
 	ENV_ISTIO_GATEWAY        = "ISTIO_GATEWAY"
+	ENV_ISTIO_TLS_MODE       = "ISTIO_TLS_MODE"
 	ANNOTATION_ISTIO_GATEWAY = "seldon.io/istio-gateway"
 )
diff --git a/operator/controllers/seldondeployment_controller.go b/operator/controllers/seldondeployment_controller.go
@@ -126,6 +126,7 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment,
 	grpcAllowed bool) ([]*istio.VirtualService, []*istio.DestinationRule) {
 
 	istio_gateway := GetEnv(ENV_ISTIO_GATEWAY, "seldon-gateway")
+	istioTLSMode := GetEnv(ENV_ISTIO_TLS_MODE, "")
 	httpVsvc := &istio.VirtualService{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      seldonId + "-http",
@@ -197,11 +198,6 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment,
 			},
 			Spec: istio.DestinationRuleSpec{
 				Host: pSvcName,
-				TrafficPolicy: &istio.TrafficPolicy{
-					TLS: &istio.TLSSettings{
-						Mode: istio.TLSmodeIstioMutual,
-					},
-				},
 				Subsets: []istio.Subset{
 					{
 						Name: p.Name,
@@ -213,6 +209,13 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment,
 			},
 		}
 
+		if istioTLSMode != "" {
+			drule.Spec.TrafficPolicy = &istio.TrafficPolicy{
+				TLS: &istio.TLSSettings{
+					Mode: istio.TLSmode(istioTLSMode),
+				},
+			}
+		}
 		drules[i] = drule
 
 		if p.Shadow == true {
@@ -1353,17 +1356,17 @@ func (r *SeldonDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 		return ctrl.Result{}, err
 	}
 
-	virtualServicesReady, err := createIstioServices(r, components, instance, log)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
+	//virtualServicesReady, err := createIstioServices(r, components, instance, log)
+	//if err != nil {
+	//	return ctrl.Result{}, err
+	//}
 
 	hpasReady, err := createHpas(r, components, instance, log)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
 
-	if deploymentsReady && servicesReady && hpasReady && virtualServicesReady {
+	if deploymentsReady && servicesReady && hpasReady {
 		instance.Status.State = "Available"
 	} else {
 		instance.Status.State = "Creating"

diff --git a/operator/helm/split_resources.py b/operator/helm/split_resources.py
@@ -35,6 +35,7 @@
     "ENGINE_CONTAINER_SERVICE_ACCOUNT_NAME": "engine.serviceAccount.name",
     "ISTIO_ENABLED":"istio.enabled",
     "ISTIO_GATEWAY":"istio.gateway",
+    "ISTIO_TLS_MODE":"istio.tlsMode",
     "PREDICTIVE_UNIT_SERVICE_PORT":"predictiveUnit.port"
 
 }
@@ -135,6 +136,10 @@ def helm_release(value: str):
                 res["metadata"]["name"] = res["metadata"]["name"] + "-" + helm_release("Namespace")
                 res["webhooks"][0]["clientConfig"]["caBundle"] = "{{ $ca.Cert | b64enc }}"
                 res["webhooks"][0]["clientConfig"]["service"]["namespace"] = helm_release("Namespace")
+                res["webhooks"][1]["clientConfig"]["caBundle"] = "{{ $ca.Cert | b64enc }}"
+                res["webhooks"][1]["clientConfig"]["service"]["namespace"] = helm_release("Namespace")
+                res["webhooks"][2]["clientConfig"]["caBundle"] = "{{ $ca.Cert | b64enc }}"
+                res["webhooks"][2]["clientConfig"]["service"]["namespace"] = helm_release("Namespace")
                 if "certmanager.k8s.io/inject-ca-from" in res["metadata"]["annotations"]:
                     res["metadata"]["annotations"]["certmanager.k8s.io/inject-ca-from"] = helm_release("Namespace") + "/seldon-serving-cert"
 

diff --git a/testing/resources/seldon-gateway.yaml b/testing/resources/seldon-gateway.yaml
@@ -0,0 +1,14 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: seldon-gateway
+spec:
+  selector:
+    istio: ingressgateway # use istio default controller
+  servers:
+  - port:
+      number: 80
+      name: http
+      protocol: HTTP
+    hosts:
+    - "*"