fvt: add basic load and infer test for TorchServe runtime

Signed-off-by: Rafael Vasquez <[email protected]>

fvt/fvtclient.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -54,6 +54,7 @@ import (
 
 	inference "github.com/kserve/modelmesh-serving/fvt/generated"
 	tfsapi "github.com/kserve/modelmesh-serving/fvt/generated/tensorflow_serving/apis"
+	torchserveapi "github.com/kserve/modelmesh-serving/fvt/generated/torchserve/apis"
 )
 
 const predictorTimeout = time.Second * 120
@@ -480,6 +481,18 @@ func (fvt *FVTClient) RunTfsInference(req *tfsapi.PredictRequest) (*tfsapi.Predi
 	return grpcClient.Predict(ctx, req)
 }
 
+func (fvt *FVTClient) RunTorchserveInference(req *torchserveapi.PredictionsRequest) (*torchserveapi.PredictionResponse, error) {
+	if fvt.grpcConn == nil {
+		return nil, errors.New("you must connect to model mesh before running an inference")
+	}
+
+	grpcClient := torchserveapi.NewInferenceAPIsServiceClient(fvt.grpcConn)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	return grpcClient.Predictions(ctx, req)
+}
+
 func (fvt *FVTClient) ConnectToModelServing(connectionType ModelServingConnectionType) error {
 	if fvt.grpcPortForward == nil {
 		podName := fvt.GetRandomReadyRuntimePodNameFromEndpoints()

fvt/inference.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,6 +17,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"encoding/json"
+	"io/ioutil"
 	"math"
 	"os"
 
@@ -29,6 +30,7 @@ import (
 
 	tfsframework "github.com/kserve/modelmesh-serving/fvt/generated/tensorflow/core/framework"
 	tfsapi "github.com/kserve/modelmesh-serving/fvt/generated/tensorflow_serving/apis"
+	torchserveapi "github.com/kserve/modelmesh-serving/fvt/generated/torchserve/apis"
 )
 
 // Used for checking if floats are sufficiently close enough.
@@ -103,6 +105,20 @@ func ExpectSuccessfulInference_openvinoMnistTFSPredict(predictorName string) {
 	Expect(err).ToNot(HaveOccurred())
 }
 
+func ExpectSuccessfulInference_torchserveMARPredict(predictorName string) {
+	imageBytes, err := ioutil.ReadFile(TestDataPath("0.png"))
+	Expect(err).ToNot(HaveOccurred())
+
+	inferRequest := &torchserveapi.PredictionsRequest{
+		ModelName: predictorName,
+		Input:     map[string][]byte{"data": imageBytes},
+	}
+
+	inferResponse, err := FVTClientInstance.RunTorchserveInference(inferRequest)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(inferResponse).ToNot(BeNil())
+}
+
 // PyTorch CIFAR
 // COS path: fvt/pytorch/pytorch-cifar
 func ExpectSuccessfulInference_pytorchCifar(predictorName string) {

fvt/predictor/predictor_test.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -112,6 +112,14 @@ var predictorsArray = []FVTPredictor{
 		differentPredictorName:     "xgboost",
 		differentPredictorFilename: "xgboost-predictor.yaml",
 	},
+	{
+		predictorName:              "pytorch-mar",
+		predictorFilename:          "pytorch-mar-predictor.yaml",
+		currentModelPath:           "fvt/pytorch/pytorch-mar/mnist.mar",
+		updatedModelPath:           "fvt/pytorch/pytorch-mar-dup/mnist.mar",
+		differentPredictorName:     "pytorch",
+		differentPredictorFilename: "pytorch-predictor.yaml",
+	},
 }
 
 var _ = Describe("Predictor", func() {
@@ -576,6 +584,30 @@ var _ = Describe("Predictor", func() {
 		})
 	})
 
+	var _ = Describe("TorchServe Inference", Ordered, func() {
+		var torchservePredictorObject *unstructured.Unstructured
+		var torchservePredictorName string
+
+		BeforeAll(func() {
+			// load the test predictor object
+			torchservePredictorObject = NewPredictorForFVT("pytorch-mar-predictor.yaml")
+			torchservePredictorName = torchservePredictorObject.GetName()
+
+			CreatePredictorAndWaitAndExpectLoaded(torchservePredictorObject)
+
+			err := FVTClientInstance.ConnectToModelServing(Insecure)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		AfterAll(func() {
+			FVTClientInstance.DeletePredictor(torchservePredictorName)
+		})
+
+		It("should successfully run an inference", func() {
+			ExpectSuccessfulInference_torchserveMARPredict(torchservePredictorName)
+		})
+	})
+
 	var _ = Describe("MLServer inference", Ordered, func() {
 		var mlsPredictorObject *unstructured.Unstructured
 		var mlsPredictorName string

fvt/testdata/predictors/pytorch-mar-predictor.yaml

@@ -0,0 +1,24 @@
+# Copyright 2022 IBM Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: serving.kserve.io/v1alpha1
+kind: Predictor
+metadata:
+  name: pytorch-mar-predictor
+spec:
+  modelType:
+    name: pytorch-mar
+  path: fvt/pytorch/pytorch-mar/mnist.mar
+  storage:
+    s3:
+      secretKey: localMinIO