diff --git a/doc/source/ingress/ambassador.md b/doc/source/ingress/ambassador.md index ad6db37b18..24bfd46453 100644 --- a/doc/source/ingress/ambassador.md +++ b/doc/source/ingress/ambassador.md @@ -26,7 +26,26 @@ Assuming a Seldon Deployment ```mymodel``` with Ambassador exposed on `0.0.0.0:8 curl -v 0.0.0.0:8003/seldon/mymodel/api/v1.0/predictions -d '{"data":{"names":["a","b"],"tensor":{"shape":[2,2],"values":[0,0,1,1]}}}' -H "Content-Type: application/json" ``` -## Canary Deployments +## Ambassador Configuration Annotations Reference + +| Annotation | Description | +|------------|-------------| +|`seldon.io/ambassador-config:`| Custom Ambassador Configuration | +|`seldon.io/ambassador-header:
`| The header to add to Ambassador configuration | +|`seldon.io/ambassador-id:`| The instance id to be added to Ambassador `ambassador_id` configuration | +|`seldon.io/ambassador-regex-header:`| The regular expression header to use for routing via headers| +|`seldon.io/ambassador-retries:` | The number of times ambassador will retry request on connect-failure. Default 0. Use custom configuration if more control needed.| +|`seldon.io/ambassador-service-name:`| The name of the existing Seldon Deployment for shadow or header based routing | +|`seldon.io/ambassador-shadow:true` | Activate shadowing for this deployment | +|`seldon.io/grpc-read-timeout: ` | gRPC read timeout | +|`seldon.io/rest-read-timeout:` | REST read timeout | + +All annotations should be placed in `spec.annotations`. + +See below for details. + + +### Canary Deployments Canary rollouts are available where you wish to push a certain percentage of traffic to a new model to test whether it works ok in production. To add a canary to your SeldonDeployment simply add a new predictor section and set the traffic levels for the main and canary to desired levels. For example: @@ -73,7 +92,7 @@ The above example has a "main" predictor with 75% of traffic and a "canary" with A worked example for [canary deployments](../examples/ambassador_canary.html) is provided. -## Shadow Deployments +### Shadow Deployments Shadow deployments allow you to send duplicate requests to a parallel deployment but throw away the response. This allows you to test machine learning models under load and compare the results to the live deployment. @@ -83,7 +102,7 @@ A worked example for [shadow deployments](../examples/ambassador_shadow.html) is To understand more about the Ambassador configuration for this see [their docs on shadow deployments](https://www.getambassador.io/reference/shadowing/). -## Header based Routing +### Header based Routing Header based routing allows you to route requests to particular Seldon Deployments based on headers in the incoming requests. @@ -91,7 +110,9 @@ You simply need to add some annotations to your Seldon Deployment resource. * `seldon.io/ambassador-header:
` : The header to add to Ambassador configuration * Example: `"seldon.io/ambassador-header":"location: london" ` - * `seldon.io/ambassador-service-name:` : The name of the existing Seldon you want to attach to as an alternative mapping for requests. + * `seldon.io/ambassador-regex-header:
` : The regular expression header to add to Ambassador configuration + * Example: `"seldon.io/ambassador-header":"location: lond.*" ` + * `seldon.io/ambassador-service-name:` : The name of the existing Seldon Deployment you want to attach to as an alternative mapping for requests. * Example: `"seldon.io/ambassador-service-name":"example"` A worked example for [header based routing](../examples/ambassador_headers.html) is provided. @@ -118,9 +139,10 @@ spec: ``` Note that your Ambassador instance must be configured with matching `ambassador_id`. + See [AMBASSADOR_ID](https://github.com/datawire/ambassador/blob/master/docs/reference/running.md#ambassador_id) for details -## Custom Amabassador configuration +### Custom Amabassador configuration The above discussed configurations should cover most cases but there maybe a case where you want to have a very particular Ambassador configuration under your control. You can acheieve this by adding your confguration as an annotation to your Seldon Deployment resource. @@ -129,3 +151,5 @@ The above discussed configurations should cover most cases but there maybe a cas A worked example for [custom Ambassador config](../examples/ambassador_custom.html) is provided. + + diff --git a/operator/controllers/ambassador.go b/operator/controllers/ambassador.go index 8dc4d499d2..bf1f130de9 100644 --- a/operator/controllers/ambassador.go +++ b/operator/controllers/ambassador.go @@ -17,10 +17,12 @@ const ( ANNOTATION_AMBASSADOR_HEADER = "seldon.io/ambassador-header" ANNOTATION_AMBASSADOR_REGEX_HEADER = "seldon.io/ambassador-regex-header" ANNOTATION_AMBASSADOR_ID = "seldon.io/ambassador-id" + ANNOTATION_AMBASSADOR_RETRIES = "seldon.io/ambassador-retries" YAML_SEP = "---\n" - AMBASSADOR_IDLE_TIMEOUT = 300000 + AMBASSADOR_IDLE_TIMEOUT = 300000 + AMBASSADOR_DEFAULT_RETRIES = "0" ) // Struct for Ambassador configuration @@ -67,7 +69,12 @@ func getAmbassadorRestConfig(mlDep *machinelearningv1.SeldonDeployment, // Set timeout timeout, err := strconv.Atoi(getAnnotation(mlDep, ANNOTATION_REST_TIMEOUT, "3000")) if err != nil { - return "", nil + return "", err + } + + retries, err := strconv.Atoi(getAnnotation(mlDep, ANNOTATION_AMBASSADOR_RETRIES, AMBASSADOR_DEFAULT_RETRIES)) + if err != nil { + return "", err } name := p.Name @@ -84,10 +91,13 @@ func getAmbassadorRestConfig(mlDep *machinelearningv1.SeldonDeployment, Rewrite: "/", Service: serviceName + "." + namespace + ":" + strconv.Itoa(engine_http_port), TimeoutMs: timeout, - RetryPolicy: &AmbassadorRetryPolicy{ + } + + if retries != 0 { + c.RetryPolicy = &AmbassadorRetryPolicy{ RetryOn: "connect-failure", - NumRetries: 3, - }, + NumRetries: retries, + } } if weight != nil { @@ -158,6 +168,11 @@ func getAmbassadorGrpcConfig(mlDep *machinelearningv1.SeldonDeployment, return "", nil } + retries, err := strconv.Atoi(getAnnotation(mlDep, ANNOTATION_AMBASSADOR_RETRIES, AMBASSADOR_DEFAULT_RETRIES)) + if err != nil { + return "", err + } + name := p.Name if nameOverride != "" { name = nameOverride @@ -175,10 +190,13 @@ func getAmbassadorGrpcConfig(mlDep *machinelearningv1.SeldonDeployment, Headers: map[string]string{"seldon": serviceNameExternal}, Service: serviceName + "." + namespace + ":" + strconv.Itoa(engine_grpc_port), TimeoutMs: timeout, - RetryPolicy: &AmbassadorRetryPolicy{ + } + + if retries != 0 { + c.RetryPolicy = &AmbassadorRetryPolicy{ RetryOn: "connect-failure", - NumRetries: 3, - }, + NumRetries: retries, + } } if weight != nil { diff --git a/operator/controllers/ambassador_test.go b/operator/controllers/ambassador_test.go index ea8cbda974..ff26035488 100644 --- a/operator/controllers/ambassador_test.go +++ b/operator/controllers/ambassador_test.go @@ -1,107 +1,53 @@ package controllers import ( - "strings" - "testing" - + . "github.com/onsi/gomega" machinelearningv1 "github.com/seldonio/seldon-core/operator/apis/machinelearning/v1" "gopkg.in/yaml.v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" + "testing" ) -func TestAmbassadorBasic(t *testing.T) { - p := machinelearningv1.PredictorSpec{Name: "p"} - mlDep := machinelearningv1.SeldonDeployment{ObjectMeta: metav1.ObjectMeta{Name: "mymodel"}, - Spec: machinelearningv1.SeldonDeploymentSpec{ - Predictors: []machinelearningv1.PredictorSpec{ - p, - }, - }, - } - s, err := getAmbassadorConfigs(&mlDep, &p, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts := strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } +const ( + TEST_DEFAULT_EXPECTED_RETRIES = 0 +) - if c.InstanceId != "" { - t.Fatalf("Found ambassador_id %s", c.InstanceId) - } +func basicAbassadorTests(t *testing.T, mlDep *machinelearningv1.SeldonDeployment, p *machinelearningv1.PredictorSpec, expectedWeight int32, expectedInstanceId string, expectedRetries int) { + g := NewGomegaWithT(t) + s, err := getAmbassadorConfigs(mlDep, p, "myservice", 9000, 5000, "") + g.Expect(err).To(BeNil()) + parts := strings.Split(s, "---\n")[1:] + g.Expect(len(parts)).To(Equal(2)) + c := AmbassadorConfig{} + err = yaml.Unmarshal([]byte(parts[0]), &c) + g.Expect(err).To(BeNil()) + g.Expect(c.Prefix).To(Equal("/seldon/default/mymodel/")) + g.Expect(c.Weight).To(Equal(expectedWeight)) + g.Expect(c.InstanceId).To(Equal(expectedInstanceId)) + if expectedRetries > 0 { + g.Expect(c.RetryPolicy.NumRetries).To(Equal(expectedRetries)) + } else { + g.Expect(c.RetryPolicy).To(BeNil()) } } func TestAmbassadorSingle(t *testing.T) { - p := machinelearningv1.PredictorSpec{Name: "p"} + p1 := machinelearningv1.PredictorSpec{Name: "p1"} mlDep := machinelearningv1.SeldonDeployment{ObjectMeta: metav1.ObjectMeta{Name: "mymodel"}, Spec: machinelearningv1.SeldonDeploymentSpec{ Predictors: []machinelearningv1.PredictorSpec{ - p, + p1, }, }, } - s, err := getAmbassadorConfigs(&mlDep, &p, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts := strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if c.Weight > 0 { - t.Fatalf("Bad weight for Ambassador config %d", c.Weight) - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } - - if c.InstanceId != "" { - t.Fatalf("Found ambassador_id %s", c.InstanceId) - } - } - + basicAbassadorTests(t, &mlDep, &p1, 0, "", TEST_DEFAULT_EXPECTED_RETRIES) } func TestAmbassadorCanary(t *testing.T) { - p1 := machinelearningv1.PredictorSpec{Name: "p", Traffic: 20} - p2 := machinelearningv1.PredictorSpec{Name: "p", Traffic: 80} + p1 := machinelearningv1.PredictorSpec{Name: "p1", Traffic: 20} + p2 := machinelearningv1.PredictorSpec{Name: "p2", Traffic: 80} mlDep := machinelearningv1.SeldonDeployment{ObjectMeta: metav1.ObjectMeta{Name: "mymodel"}, Spec: machinelearningv1.SeldonDeploymentSpec{ Predictors: []machinelearningv1.PredictorSpec{ @@ -110,79 +56,8 @@ func TestAmbassadorCanary(t *testing.T) { }, }, } - - s, err := getAmbassadorConfigs(&mlDep, &p1, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts := strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if c.Weight > 0 { - t.Fatalf("Bad weight for Ambassador config %d", c.Weight) - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } - - if c.InstanceId != "" { - t.Fatalf("Found ambassador_id %s", c.InstanceId) - } - } - - s, err = getAmbassadorConfigs(&mlDep, &p2, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts = strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if c.Weight > 0 { - t.Fatalf("Bad weight for Ambassador config %d", c.Weight) - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } - - if c.InstanceId != "" { - t.Fatalf("Found ambassador_id %s", c.InstanceId) - } - } - + basicAbassadorTests(t, &mlDep, &p1, 0, "", TEST_DEFAULT_EXPECTED_RETRIES) + basicAbassadorTests(t, &mlDep, &p2, 80, "", TEST_DEFAULT_EXPECTED_RETRIES) } func TestAmbassadorCanaryEqual(t *testing.T) { @@ -196,79 +71,8 @@ func TestAmbassadorCanaryEqual(t *testing.T) { }, }, } - - s, err := getAmbassadorConfigs(&mlDep, &p1, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts := strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if c.Weight > 0 { - t.Fatalf("Bad weight for Ambassador config %d", c.Weight) - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } - - if c.InstanceId != "" { - t.Fatalf("Found ambassador_id %s", c.InstanceId) - } - } - - s, err = getAmbassadorConfigs(&mlDep, &p2, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts = strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if c.Weight != 50 { - t.Fatalf("Bad weight for Ambassador config %d", c.Weight) - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } - - if c.InstanceId != "" { - t.Fatalf("Found ambassador_id %s", c.InstanceId) - } - } - + basicAbassadorTests(t, &mlDep, &p1, 0, "", TEST_DEFAULT_EXPECTED_RETRIES) + basicAbassadorTests(t, &mlDep, &p2, 50, "", TEST_DEFAULT_EXPECTED_RETRIES) } func TestAmbassadorCanaryThree(t *testing.T) { @@ -284,79 +88,9 @@ func TestAmbassadorCanaryThree(t *testing.T) { }, }, } - - s, err := getAmbassadorConfigs(&mlDep, &p1, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts := strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if c.Weight != 0 { - t.Fatalf("Bad weight for Ambassador config %d", c.Weight) - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } - - if c.InstanceId != "" { - t.Fatalf("Found ambassador_id %s", c.InstanceId) - } - } - - s, err = getAmbassadorConfigs(&mlDep, &p2, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts = strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if c.Weight != 20 { - t.Fatalf("Bad weight for Ambassador config %d", c.Weight) - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } - - if c.InstanceId != "" { - t.Fatalf("Found ambassador_id %s", c.InstanceId) - } - } - + basicAbassadorTests(t, &mlDep, &p1, 0, "", TEST_DEFAULT_EXPECTED_RETRIES) + basicAbassadorTests(t, &mlDep, &p2, 20, "", TEST_DEFAULT_EXPECTED_RETRIES) + basicAbassadorTests(t, &mlDep, &p3, 20, "", TEST_DEFAULT_EXPECTED_RETRIES) } func TestAmbassadorCanaryThreeEqual(t *testing.T) { @@ -372,121 +106,34 @@ func TestAmbassadorCanaryThreeEqual(t *testing.T) { }, }, } + basicAbassadorTests(t, &mlDep, &p1, 0, "", TEST_DEFAULT_EXPECTED_RETRIES) + basicAbassadorTests(t, &mlDep, &p2, 33, "", TEST_DEFAULT_EXPECTED_RETRIES) + basicAbassadorTests(t, &mlDep, &p3, 33, "", TEST_DEFAULT_EXPECTED_RETRIES) +} - s, err := getAmbassadorConfigs(&mlDep, &p1, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts := strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if c.Weight > 0 { - t.Fatalf("Bad weight for Ambassador config %d", c.Weight) - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } - - if c.InstanceId != "" { - t.Fatalf("Found ambassador_id %s", c.InstanceId) - } - } - - s, err = getAmbassadorConfigs(&mlDep, &p2, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts = strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if c.Weight != 33 { - t.Fatalf("Bad weight for Ambassador config %d", c.Weight) - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } - - if c.InstanceId != "" { - t.Fatalf("Found ambassador_id %s", c.InstanceId) - } +func TestAmbassadorID(t *testing.T) { + const instanceId = "myinstance_id" + p1 := machinelearningv1.PredictorSpec{Name: "p"} + mlDep := machinelearningv1.SeldonDeployment{ObjectMeta: metav1.ObjectMeta{Name: "mymodel"}, + Spec: machinelearningv1.SeldonDeploymentSpec{ + Annotations: map[string]string{ANNOTATION_AMBASSADOR_ID: instanceId}, + Predictors: []machinelearningv1.PredictorSpec{ + p1, + }, + }, } - + basicAbassadorTests(t, &mlDep, &p1, 0, instanceId, TEST_DEFAULT_EXPECTED_RETRIES) } -func TestAmbassadorID(t *testing.T) { +func TestAmbassadorRetriesAnnotation(t *testing.T) { p := machinelearningv1.PredictorSpec{Name: "p"} mlDep := machinelearningv1.SeldonDeployment{ObjectMeta: metav1.ObjectMeta{Name: "mymodel"}, Spec: machinelearningv1.SeldonDeploymentSpec{ - Annotations: map[string]string{ANNOTATION_AMBASSADOR_ID: "myinstance_id"}, + Annotations: map[string]string{ANNOTATION_AMBASSADOR_RETRIES: "2"}, Predictors: []machinelearningv1.PredictorSpec{ p, }, }, } - - s, err := getAmbassadorConfigs(&mlDep, &p, "myservice", 9000, 5000, "") - if err != nil { - t.Fatalf("Config format error") - } - t.Logf("%s\n\n", s) - parts := strings.Split(s, "---\n")[1:] - - if len(parts) != 2 { - t.Fatalf("Bad number of configs returned %d", len(parts)) - } - - for _, part := range parts { - c := AmbassadorConfig{} - t.Logf("Config: %s", part) - - err = yaml.Unmarshal([]byte(s), &c) - if err != nil { - t.Fatalf("Failed to unmarshall") - } - - if len(c.Headers) > 0 { - t.Fatalf("Found headers") - } - if c.Prefix != "/seldon/default/mymodel/" { - t.Fatalf("Found bad prefix %s", c.Prefix) - } - - if c.InstanceId != "myinstance_id" { - t.Fatalf("Found mismatch ambassador_id %s", c.InstanceId) - } - } + basicAbassadorTests(t, &mlDep, &p, 0, "", 2) }