diff --git a/doc/source/ingress/istio.md b/doc/source/ingress/istio.md index d37f3ad7cb..eb6ebdfef0 100644 --- a/doc/source/ingress/istio.md +++ b/doc/source/ingress/istio.md @@ -144,3 +144,78 @@ You can fix this by changing `defaultUserID=0` in your helm chart, or add the fo securityContext: runAsUser: 0 ``` + + +# Using the Istio Service Mesh +Istio can also be used to direct traffic internal to the cluster, rather than using it as an ingress (traffic from outside the cluster). + +To do this, the Virutal Services Seldon will create need to be attached to the "special" Gateway named `mesh`. This applies the routing rules to traffic inside the mesh without needing to route through a Gateway. + +Due to limitations in Istio (as of v1.11.3), virtual services in the local mesh can only apply to one Host. (see their docs [here](https://istio.io/latest/docs/ops/best-practices/traffic-management/#split-virtual-services)). Therefor, a unique service is required for each Graph, which can be achieved by setting the `seldon.io/svc-name` annotation in the main predictor. + +Here's an example `SeldonDeployment` that will utilize the internal mesh networking to split traffic between two predictors, 75% to the first, 25% to the second: +``` yaml +apiVersion: machinelearning.seldon.io/v1 +kind: SeldonDeployment +metadata: + labels: + app: seldon + name: canary-example-1 + namespace: my-ns +spec: + annotations: + seldon.io/istio-gateway: mesh # NOTE + seldon.io/istio-host: canary-example-1 # NOTE + name: canary-example-1 + predictors: + - annotations: + seldon.io/svc-name: canary-example-1 # NOTE + componentSpecs: + - spec: + containers: + - image: seldonio/mock_classifier:1.11.0 + imagePullPolicy: IfNotPresent + name: classifier + securityContext: + readOnlyRootFilesystem: false + terminationGracePeriodSeconds: 1 + graph: + endpoint: + type: REST + name: classifier + type: MODEL + labels: + sidecar.istio.io/inject: "true" + name: main + replicas: 1 + traffic: 75 + - componentSpecs: + - spec: + containers: + - image: seldonio/mock_classifier:1.11.0 + imagePullPolicy: IfNotPresent + name: classifier + terminationGracePeriodSeconds: 1 + graph: + endpoint: + type: REST + name: classifier + type: MODEL + labels: + sidecar.istio.io/inject: "true" + name: canary + replicas: 1 + traffic: 25 +``` + +A few key things to point out: +1. A unique service is created for the main (first) predictor named `canary-example-1`. This service cannot collide with any other services in the namespace. This service could be a service _not_ created via the SeldonDeployment, but also must match the necessary Istio routing rules. +2. The above service is referenced in the annotations in `spec` by specify ing the host as follows: `seldon.io/istio-host: canary-example-1`. This will set the host in the Istio Virutal Service to be the newly created service. +3. The gateway is specified as `seldon.io/istio-gateway: mesh` to utilize this routing in the Istio Mesh. NOTE: In order to call this service, and have the appropriate routing take place, the Client _must_ also be _inside_ the mesh. This is accomplished by injecting the Istio Sidecar into the pod of the client. + +From within the cluster, and inside a pod that is inside the mesh, a call like the following will work, as well as split traffic between the two predictors: +``` shell +curl -X POST -H 'Content-Type: application/json' \ + -d '{"data": { "names": ["a", "b"], "ndarray": [[1,2]]}}' \ + http://mysvcname:8000/seldon/batest/canary-example-1/api/v1.0/predictions +``` \ No newline at end of file diff --git a/operator/controllers/cleaners.go b/operator/controllers/cleaners.go index 9a59055166..2e32d5ded2 100644 --- a/operator/controllers/cleaners.go +++ b/operator/controllers/cleaners.go @@ -23,7 +23,7 @@ func (r *ResourceCleaner) cleanUnusedVirtualServices() ([]*istio.VirtualService, err := r.client.List(context.Background(), vlist, &client.ListOptions{Namespace: r.instance.Namespace}) for _, vsvc := range vlist.Items { for _, ownerRef := range vsvc.OwnerReferences { - if ownerRef.Name == r.instance.Name { + if ownerRef.UID == r.instance.GetUID() { found := false for _, expectedVsvc := range r.virtualServices { if expectedVsvc.Name == vsvc.Name { diff --git a/operator/controllers/seldondeployment_controller.go b/operator/controllers/seldondeployment_controller.go index ec0b6517d3..827ef56757 100644 --- a/operator/controllers/seldondeployment_controller.go +++ b/operator/controllers/seldondeployment_controller.go @@ -221,9 +221,9 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment, return nil, nil, err } } - httpVsvc := &istio.VirtualService{ + vsvc := &istio.VirtualService{ ObjectMeta: metav1.ObjectMeta{ - Name: seldonId + "-http", + Name: seldonId, Namespace: namespace, }, Spec: istio_networking.VirtualService{ @@ -238,19 +238,6 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment, }, Rewrite: &istio_networking.HTTPRewrite{Uri: "/"}, }, - }, - }, - } - - grpcVsvc := &istio.VirtualService{ - ObjectMeta: metav1.ObjectMeta{ - Name: seldonId + "-grpc", - Namespace: namespace, - }, - Spec: istio_networking.VirtualService{ - Hosts: []string{getAnnotation(mlDep, ANNOTATION_ISTIO_HOST, "*")}, - Gateways: []string{getAnnotation(mlDep, ANNOTATION_ISTIO_GATEWAY, istio_gateway)}, - Http: []*istio_networking.HTTPRoute{ { Match: []*istio_networking.HTTPMatchRequest{ { @@ -265,10 +252,11 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment, }, }, } + // Add retries if istioRetries > 0 { - httpVsvc.Spec.Http[0].Retries = &istio_networking.HTTPRetry{Attempts: int32(istioRetries), PerTryTimeout: &types2.Duration{Seconds: int64(istioRetriesTimeout)}, RetryOn: "gateway-error,connect-failure,refused-stream"} - grpcVsvc.Spec.Http[0].Retries = &istio_networking.HTTPRetry{Attempts: int32(istioRetries), PerTryTimeout: &types2.Duration{Seconds: int64(istioRetriesTimeout)}, RetryOn: "gateway-error,connect-failure,refused-stream"} + vsvc.Spec.Http[0].Retries = &istio_networking.HTTPRetry{Attempts: int32(istioRetries), PerTryTimeout: &types2.Duration{Seconds: int64(istioRetriesTimeout)}, RetryOn: "gateway-error,connect-failure,refused-stream"} + vsvc.Spec.Http[1].Retries = &istio_networking.HTTPRetry{Attempts: int32(istioRetries), PerTryTimeout: &types2.Duration{Seconds: int64(istioRetriesTimeout)}, RetryOn: "gateway-error,connect-failure,refused-stream"} } // shadows don't get destinations in the vs as a shadow is a mirror instead @@ -322,7 +310,7 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment, if p.Shadow == true { //if there's a shadow then add a mirror section to the VirtualService - httpVsvc.Spec.Http[0].Mirror = &istio_networking.Destination{ + vsvc.Spec.Http[0].Mirror = &istio_networking.Destination{ Host: pSvcName, Subset: p.Name, Port: &istio_networking.PortSelector{ @@ -330,7 +318,7 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment, }, } - grpcVsvc.Spec.Http[0].Mirror = &istio_networking.Destination{ + vsvc.Spec.Http[1].Mirror = &istio_networking.Destination{ Host: pSvcName, Subset: p.Name, Port: &istio_networking.PortSelector{ @@ -366,12 +354,11 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment, routesIdx += 1 } - httpVsvc.Spec.Http[0].Route = routesHttp - grpcVsvc.Spec.Http[0].Route = routesGrpc + vsvc.Spec.Http[0].Route = routesHttp + vsvc.Spec.Http[1].Route = routesGrpc - vscs := make([]*istio.VirtualService, 2) - vscs[0] = httpVsvc - vscs[1] = grpcVsvc + vscs := make([]*istio.VirtualService, 1) + vscs[0] = vsvc return vscs, drules, nil }