test: Add TorchServe FVT (#294)

#### Motivation Support for TorchServe was added in #250 and kserve/modelmesh-runtime-adapter#34. A test should be added for it as well. #### Modifications - Adds basic FVT for load/inference with a TorchServe MAR model using the native TorchServe gRPC API - Disables OVMS runtime and tests to allow TorchServe to be tested due to resource constraints #### Result Closes #280 Signed-off-by: Rafael Vasquez <[email protected]>
kserve · Feb 21, 2023 · ff0d455 · ff0d455
1 parent 346634f
commit ff0d455
Show file tree

Hide file tree

Showing 8 changed files with 606 additions and 3 deletions.
diff --git a/fvt/fvtclient.go b/fvt/fvtclient.go
@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -54,6 +54,7 @@ import (
 
 	inference "github.com/kserve/modelmesh-serving/fvt/generated"
 	tfsapi "github.com/kserve/modelmesh-serving/fvt/generated/tensorflow_serving/apis"
+	torchserveapi "github.com/kserve/modelmesh-serving/fvt/generated/torchserve/apis"
 )
 
 const predictorTimeout = time.Second * 120
@@ -480,6 +481,18 @@ func (fvt *FVTClient) RunTfsInference(req *tfsapi.PredictRequest) (*tfsapi.Predi
 	return grpcClient.Predict(ctx, req)
 }
 
+func (fvt *FVTClient) RunTorchserveInference(req *torchserveapi.PredictionsRequest) (*torchserveapi.PredictionResponse, error) {
+	if fvt.grpcConn == nil {
+		return nil, errors.New("you must connect to model mesh before running an inference")
+	}
+
+	grpcClient := torchserveapi.NewInferenceAPIsServiceClient(fvt.grpcConn)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	return grpcClient.Predictions(ctx, req)
+}
+
 func (fvt *FVTClient) ConnectToModelServing(connectionType ModelServingConnectionType) error {
 	if fvt.grpcPortForward == nil {
 		podName := fvt.GetRandomReadyRuntimePodNameFromEndpoints()

diff --git a/fvt/generated/torchserve/apis/inference.pb.go b/fvt/generated/torchserve/apis/inference.pb.go