combine virtualservices into one (SeldonIO#3609)

* combine virtualservices into one * fix return vsvcs * rename vsvc * update istio cleaner to only delete with matching uid * add istio mesh docs * add images * remove azure metrics file * remove azure metrics file
stephen37 · Oct 25, 2021 · 8a5f2f8 · 8a5f2f8
1 parent 5b3db17
commit 8a5f2f8
Show file tree

Hide file tree

Showing 3 changed files with 87 additions and 25 deletions.
diff --git a/doc/source/ingress/istio.md b/doc/source/ingress/istio.md
@@ -144,3 +144,78 @@ You can fix this by changing `defaultUserID=0` in your helm chart, or add the fo
 securityContext:
   runAsUser: 0
 ```
+
+
+# Using the Istio Service Mesh
+Istio can also be used to direct traffic internal to the cluster, rather than using it as an ingress (traffic from outside the cluster). 
+
+To do this, the Virutal Services Seldon will create need to be attached to the "special" Gateway named `mesh`. This applies the routing rules to traffic inside the mesh without needing to route through a Gateway.
+
+Due to limitations in Istio (as of v1.11.3), virtual services in the local mesh can only apply to one Host. (see their docs [here](https://istio.io/latest/docs/ops/best-practices/traffic-management/#split-virtual-services)). Therefor, a unique service is required for each Graph, which can be achieved by setting the `seldon.io/svc-name` annotation in the main predictor. 
+
+Here's an example `SeldonDeployment` that will utilize the internal mesh networking to split traffic between two predictors, 75% to the first, 25% to the second: 
+``` yaml
+apiVersion: machinelearning.seldon.io/v1
+kind: SeldonDeployment
+metadata:
+  labels:
+    app: seldon
+  name: canary-example-1
+  namespace: my-ns
+spec:
+  annotations:
+    seldon.io/istio-gateway: mesh # NOTE
+    seldon.io/istio-host: canary-example-1 # NOTE
+  name: canary-example-1
+  predictors:
+  - annotations:
+      seldon.io/svc-name: canary-example-1   # NOTE
+    componentSpecs:
+    - spec:
+        containers:
+        - image: seldonio/mock_classifier:1.11.0
+          imagePullPolicy: IfNotPresent
+          name: classifier
+          securityContext:
+            readOnlyRootFilesystem: false
+        terminationGracePeriodSeconds: 1
+    graph:
+      endpoint:
+        type: REST
+      name: classifier
+      type: MODEL
+    labels:
+      sidecar.istio.io/inject: "true"
+    name: main
+    replicas: 1
+    traffic: 75
+  - componentSpecs:
+    - spec:
+        containers:
+        - image: seldonio/mock_classifier:1.11.0
+          imagePullPolicy: IfNotPresent
+          name: classifier
+        terminationGracePeriodSeconds: 1
+    graph:
+      endpoint:
+        type: REST
+      name: classifier
+      type: MODEL
+    labels:
+      sidecar.istio.io/inject: "true"
+    name: canary
+    replicas: 1
+    traffic: 25
+```
+
+A few key things to point out:  
+1. A unique service is created for the main (first) predictor named `canary-example-1`. This service cannot collide with any other services in the namespace. This service could be a service _not_ created via the SeldonDeployment, but also must match the necessary Istio routing rules. 
+2. The above service is referenced in the annotations in `spec` by specify ing the host as follows:  `seldon.io/istio-host: canary-example-1`. This will set the host in the Istio Virutal Service to be the newly created service. 
+3. The gateway is specified as `seldon.io/istio-gateway: mesh` to utilize this routing in the Istio Mesh. NOTE: In order to call this service, and have the appropriate routing take place, the Client _must_ also be _inside_ the mesh. This is accomplished by injecting the Istio Sidecar into the pod of the client. 
+
+From within the cluster, and inside a pod that is inside the mesh, a call like the following will work, as well as split traffic between the two predictors:
+``` shell
+curl -X POST -H 'Content-Type: application/json' \
+  -d '{"data": { "names": ["a", "b"], "ndarray": [[1,2]]}}' \
+  http://mysvcname:8000/seldon/batest/canary-example-1/api/v1.0/predictions
+```
diff --git a/operator/controllers/cleaners.go b/operator/controllers/cleaners.go
@@ -23,7 +23,7 @@ func (r *ResourceCleaner) cleanUnusedVirtualServices() ([]*istio.VirtualService,
 	err := r.client.List(context.Background(), vlist, &client.ListOptions{Namespace: r.instance.Namespace})
 	for _, vsvc := range vlist.Items {
 		for _, ownerRef := range vsvc.OwnerReferences {
-			if ownerRef.Name == r.instance.Name {
+			if ownerRef.UID == r.instance.GetUID() {
 				found := false
 				for _, expectedVsvc := range r.virtualServices {
 					if expectedVsvc.Name == vsvc.Name {

diff --git a/operator/controllers/seldondeployment_controller.go b/operator/controllers/seldondeployment_controller.go
@@ -221,9 +221,9 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment,
 			return nil, nil, err
 		}
 	}
-	httpVsvc := &istio.VirtualService{
+	vsvc := &istio.VirtualService{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      seldonId + "-http",
+			Name:      seldonId,
 			Namespace: namespace,
 		},
 		Spec: istio_networking.VirtualService{
@@ -238,19 +238,6 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment,
 					},
 					Rewrite: &istio_networking.HTTPRewrite{Uri: "/"},
 				},
-			},
-		},
-	}
-
-	grpcVsvc := &istio.VirtualService{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      seldonId + "-grpc",
-			Namespace: namespace,
-		},
-		Spec: istio_networking.VirtualService{
-			Hosts:    []string{getAnnotation(mlDep, ANNOTATION_ISTIO_HOST, "*")},
-			Gateways: []string{getAnnotation(mlDep, ANNOTATION_ISTIO_GATEWAY, istio_gateway)},
-			Http: []*istio_networking.HTTPRoute{
 				{
 					Match: []*istio_networking.HTTPMatchRequest{
 						{
@@ -265,10 +252,11 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment,
 			},
 		},
 	}
+
 	// Add retries
 	if istioRetries > 0 {
-		httpVsvc.Spec.Http[0].Retries = &istio_networking.HTTPRetry{Attempts: int32(istioRetries), PerTryTimeout: &types2.Duration{Seconds: int64(istioRetriesTimeout)}, RetryOn: "gateway-error,connect-failure,refused-stream"}
-		grpcVsvc.Spec.Http[0].Retries = &istio_networking.HTTPRetry{Attempts: int32(istioRetries), PerTryTimeout: &types2.Duration{Seconds: int64(istioRetriesTimeout)}, RetryOn: "gateway-error,connect-failure,refused-stream"}
+		vsvc.Spec.Http[0].Retries = &istio_networking.HTTPRetry{Attempts: int32(istioRetries), PerTryTimeout: &types2.Duration{Seconds: int64(istioRetriesTimeout)}, RetryOn: "gateway-error,connect-failure,refused-stream"}
+		vsvc.Spec.Http[1].Retries = &istio_networking.HTTPRetry{Attempts: int32(istioRetries), PerTryTimeout: &types2.Duration{Seconds: int64(istioRetriesTimeout)}, RetryOn: "gateway-error,connect-failure,refused-stream"}
 	}
 
 	// shadows don't get destinations in the vs as a shadow is a mirror instead
@@ -322,15 +310,15 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment,
 		if p.Shadow == true {
 			//if there's a shadow then add a mirror section to the VirtualService
 
-			httpVsvc.Spec.Http[0].Mirror = &istio_networking.Destination{
+			vsvc.Spec.Http[0].Mirror = &istio_networking.Destination{
 				Host:   pSvcName,
 				Subset: p.Name,
 				Port: &istio_networking.PortSelector{
 					Number: uint32(ports[i].httpPort),
 				},
 			}
 
-			grpcVsvc.Spec.Http[0].Mirror = &istio_networking.Destination{
+			vsvc.Spec.Http[1].Mirror = &istio_networking.Destination{
 				Host:   pSvcName,
 				Subset: p.Name,
 				Port: &istio_networking.PortSelector{
@@ -366,12 +354,11 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment,
 		routesIdx += 1
 
 	}
-	httpVsvc.Spec.Http[0].Route = routesHttp
-	grpcVsvc.Spec.Http[0].Route = routesGrpc
+	vsvc.Spec.Http[0].Route = routesHttp
+	vsvc.Spec.Http[1].Route = routesGrpc
 
-	vscs := make([]*istio.VirtualService, 2)
-	vscs[0] = httpVsvc
-	vscs[1] = grpcVsvc
+	vscs := make([]*istio.VirtualService, 1)
+	vscs[0] = vsvc
 	return vscs, drules, nil
 
 }