Bump mlserver to 0.5.3 (kubeflow#1853)

* Bump mlserver to 0.5.0

* Change umask and set 777 for new folders

* Remove comment

* Add changes in go.mod

* Update storage URI in MMS tests

* Update expected results

* Fix linter

* Update storage URIs in other tests and docs

* Try extending sleep

* Add more debug info

* Reduce sleep

* uncomment lines

* Disable errexit temporarily to get debug output

* Add more logs to e2e script

* Kick-off tests again

* Try increasing memory reqs

* Remove breakpoint

* Bump up memory reqs to 512

* Bump MLServer image to 0.5.3

* Revert some of the debugging changes

* Change memory requirements for MMS

* Try to increase memory in case is OOO again

* Try reducing amount of memory

* Reduce memory usage on MMS test

config/configmap/inferenceservice.yaml

@@ -35,11 +35,11 @@ data:
           },
           "v2": {
             "image": "docker.io/seldonio/mlserver",
-            "defaultImageVersion": "0.2.1",
+            "defaultImageVersion": "0.5.3",
             "supportedFrameworks": [
               "sklearn"
             ],
-            "multiModelServer": false
+            "multiModelServer": true
           }
         },
         "xgboost": {
@@ -53,11 +53,11 @@ data:
           },
           "v2": {
             "image": "docker.io/seldonio/mlserver",
-            "defaultImageVersion": "0.2.1",
+            "defaultImageVersion": "0.5.3",
             "supportedFrameworks": [
               "xgboost"
             ],
-            "multiModelServer": false
+            "multiModelServer": true
           }
         },
         "pytorch": {

config/overlays/test/configmap/inferenceservice.yaml

@@ -35,7 +35,7 @@ data:
           },
           "v2": {
             "image": "docker.io/seldonio/mlserver",
-            "defaultImageVersion": "0.2.1",
+            "defaultImageVersion": "0.5.3",
             "supportedFrameworks": [
               "sklearn"
             ],
@@ -53,7 +53,7 @@ data:
           },
           "v2": {
             "image": "docker.io/seldonio/mlserver",
-            "defaultImageVersion": "0.2.1",
+            "defaultImageVersion": "0.5.3",
             "supportedFrameworks": [
               "xgboost"
             ],

docs/samples/v1beta1/sklearn/v2/README.md

@@ -97,7 +97,7 @@ spec:
   predictor:
     sklearn:
       protocolVersion: "v2"
-      storageUri: "gs://seldon-models/sklearn/iris"
+      storageUri: "gs://seldon-models/sklearn/mms/lr_model"
 ```
 
 Note that this makes the following assumptions:

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/stretchr/testify v1.5.1
 	go.uber.org/zap v1.15.0
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73
+	golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f
 	google.golang.org/api v0.31.0
 	google.golang.org/grpc v1.31.1
 	google.golang.org/protobuf v1.25.0

pkg/agent/puller.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"path/filepath"
 	"sync"
+	"syscall"
 
 	"github.com/kserve/kserve/pkg/agent/storage"
 	v1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1"
@@ -64,6 +65,12 @@ func StartPullerAndProcessModels(downloader *Downloader, commands <-chan ModelOp
 		Downloader:  downloader,
 		logger:      logger,
 	}
+
+	// Change umask to ensure we have control over the downloaded file
+	// permissions:
+	// https://stackoverflow.com/a/61645606/5015573
+	syscall.Umask(0)
+
 	puller.waitGroup.wg.Add(len(commands))
 	go puller.processCommands(commands)
 	puller.waitGroup.wg.Wait()

pkg/agent/storage/utils.go

@@ -56,7 +56,12 @@ func AsSha256(o interface{}) string {
 }
 
 func Create(fileName string) (*os.File, error) {
-	if err := os.MkdirAll(filepath.Dir(fileName), 0770); err != nil {
+	// Set the permissions to `777` so that the downloaded files are still
+	// readable by every other user and group. This ensures that the agent is
+	// compatible with any model / server container, using any user ID. Note we
+	// also need to enable the `+x` bit to ensure the folder is "listable":
+	// https://stackoverflow.com/a/30788944/5015573
+	if err := os.MkdirAll(filepath.Dir(fileName), 0777); err != nil {
 		return nil, err
 	}
 	return os.Create(fileName)

pkg/agent/storage/utils_test.go

@@ -0,0 +1,34 @@
+package storage
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"syscall"
+	"testing"
+
+	"github.com/onsi/gomega"
+)
+
+func TestCreate(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+
+	// This would get called in StartPullerAndProcessModels
+	syscall.Umask(0)
+
+	tmpDir, _ := ioutil.TempDir("", "test-create-")
+	defer os.RemoveAll(tmpDir)
+
+	folderPath := path.Join(tmpDir, "foo")
+	filePath := path.Join(folderPath, "bar.txt")
+	f, err := Create(filePath)
+	defer f.Close()
+
+	g.Expect(err).To(gomega.BeNil())
+	g.Expect(folderPath).To(gomega.BeADirectory())
+
+	info, _ := os.Stat(folderPath)
+	mode := info.Mode()
+	expectedMode := os.FileMode(0777)
+	g.Expect(mode.Perm()).To(gomega.Equal(expectedMode))
+}

test/e2e/predictor/test_multi_model_serving.py

@@ -14,7 +14,6 @@
 import os
 import pytest
 
-from typing import List
 from kubernetes import client
 from kserve import (
     constants,
@@ -36,33 +35,27 @@
 
 
 @pytest.mark.parametrize(
-    "protocol_version,storage_uris",
+    "protocol_version,storage_uri",
     [
         (
             "v1",
-            [
-                "gs://kfserving-samples/models/sklearn/iris",
-                "gs://kfserving-samples/models/sklearn/iris",
-            ],
+            "gs://kfserving-samples/models/sklearn/iris",
         ),
         (
             "v2",
-            [
-                "gs://seldon-models/sklearn/mms/model1-sklearn-v2",
-                "gs://seldon-models/sklearn/mms/model2-sklearn-v2",
-            ],
+            "gs://seldon-models/sklearn/mms/lr_model",
         ),
     ],
 )
-def test_mms_sklearn_kserve(protocol_version: str, storage_uris: List[str]):
+def test_mms_sklearn_kserve(protocol_version: str, storage_uri: str):
     # Define an inference service
     predictor = V1beta1PredictorSpec(
         min_replicas=1,
         sklearn=V1beta1SKLearnSpec(
             protocol_version=protocol_version,
             resources=client.V1ResourceRequirements(
-                requests={"cpu": "100m", "memory": "256Mi"},
-                limits={"cpu": "100m", "memory": "256Mi"},
+                requests={"cpu": "100m", "memory": "512Mi"},
+                limits={"cpu": "100m", "memory": "1024Mi"},
             ),
         ),
     )
@@ -88,7 +81,7 @@ def test_mms_sklearn_kserve(protocol_version: str, storage_uris: List[str]):
         f"model2-sklearn-{protocol_version}",
     ]
 
-    for model_name, storage_uri in zip(model_names, storage_uris):
+    for model_name in model_names:
         model_spec = V1alpha1ModelSpec(
             storage_uri=storage_uri,
             memory="128Mi",
@@ -136,8 +129,8 @@ def test_mms_sklearn_kserve(protocol_version: str, storage_uris: List[str]):
         assert responses[0]["predictions"] == [1, 1]
         assert responses[1]["predictions"] == [1, 1]
     elif protocol_version == "v2":
-        assert responses[0]["outputs"][0]["data"] == [1, 2]
-        assert responses[1]["outputs"][0]["data"] == [1, 2]
+        assert responses[0]["outputs"][0]["data"] == [1, 1]
+        assert responses[1]["outputs"][0]["data"] == [1, 1]
 
     # Clean up inference service and trained models
     for model_name in model_names:
@@ -146,33 +139,28 @@ def test_mms_sklearn_kserve(protocol_version: str, storage_uris: List[str]):
 
 
 @pytest.mark.parametrize(
-    "protocol_version,storage_uris",
+    "protocol_version,storage_uri",
     [
         (
             "v1",
-            [
-                "gs://kfserving-samples/models/xgboost/iris",
-                "gs://kfserving-samples/models/xgboost/iris",
-            ],
+            "gs://kfserving-samples/models/xgboost/iris",
         ),
         (
             "v2",
-            [
-                "gs://seldon-models/xgboost/mms/model1-xgboost-v2",
-                "gs://seldon-models/xgboost/mms/model2-xgboost-v2",
-            ],
+            "gs://seldon-models/xgboost/mms/iris",
         ),
     ],
 )
-def test_mms_xgboost_kserve(protocol_version: str, storage_uris: List[str]):
+def test_mms_xgboost_kserve(protocol_version: str, storage_uri: str):
     # Define an inference service
     predictor = V1beta1PredictorSpec(
         min_replicas=1,
         xgboost=V1beta1XGBoostSpec(
+            env=[client.V1EnvVar(name="MLSERVER_MODEL_PARALLEL_WORKERS", value="0")],
             protocol_version=protocol_version,
             resources=client.V1ResourceRequirements(
-                requests={"cpu": "100m", "memory": "256Mi"},
-                limits={"cpu": "100m", "memory": "256Mi"},
+                requests={"cpu": "100m", "memory": "512Mi"},
+                limits={"cpu": "100m", "memory": "1024Mi"},
             ),
         ),
     )
@@ -197,7 +185,7 @@ def test_mms_xgboost_kserve(protocol_version: str, storage_uris: List[str]):
         f"model2-xgboost-{protocol_version}",
     ]
 
-    for model_name, storage_uri in zip(model_names, storage_uris):
+    for model_name in model_names:
         # Define trained models
         model_spec = V1alpha1ModelSpec(
             storage_uri=storage_uri,

test/e2e/predictor/test_sklearn.py

@@ -63,11 +63,11 @@ def test_sklearn_v2_kserve():
     predictor = V1beta1PredictorSpec(
         min_replicas=1,
         sklearn=V1beta1SKLearnSpec(
-            storage_uri="gs://kfserving-examples/models/sklearn/irisv2",
+            storage_uri="gs://seldon-models/sklearn/mms/lr_model",
             protocol_version="v2",
             resources=V1ResourceRequirements(
                 requests={"cpu": "100m", "memory": "256Mi"},
-                limits={"cpu": "100m", "memory": "256Mi"},
+                limits={"cpu": "100m", "memory": "512Mi"},
             ),
         ),
     )
@@ -85,6 +85,6 @@ def test_sklearn_v2_kserve():
     kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
 
     res = predict(service_name, "./data/iris_input_v2.json", protocol_version="v2")
-    assert res["outputs"][0]["data"] == [1, 2]
+    assert res["outputs"][0]["data"] == [1, 1]
 
     kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)

test/e2e/predictor/test_xgboost.py

@@ -21,7 +21,7 @@
     V1beta1PredictorSpec,
     V1beta1XGBoostSpec,
 )
-from kubernetes.client import V1ResourceRequirements
+from kubernetes.client import V1ResourceRequirements, V1EnvVar
 
 from ..common.utils import predict, KSERVE_TEST_NAMESPACE
 
@@ -63,10 +63,11 @@ def test_xgboost_v2_kserve():
         min_replicas=1,
         xgboost=V1beta1XGBoostSpec(
             storage_uri="gs://kfserving-samples/models/xgboost/iris",
+            env=[V1EnvVar(name="MLSERVER_MODEL_PARALLEL_WORKERS", value="0")],
             protocol_version="v2",
             resources=V1ResourceRequirements(
                 requests={"cpu": "100m", "memory": "256Mi"},
-                limits={"cpu": "100m", "memory": "256Mi"},
+                limits={"cpu": "100m", "memory": "1024Mi"},
             ),
         ),
     )