From 1f36e40310a863d0a586d0949a69e1fc69ef927c Mon Sep 17 00:00:00 2001
From: Chad Roberts <croberts@redhat.com>
Date: Wed, 31 Aug 2022 12:19:33 -0400
Subject: [PATCH] Adding test for modelmesh-serving that utilizes the triton
 servingruntime

---
 tests/basictests/modelmesh.sh               | 45 ++++++++++++++++++---
 tests/resources/modelmesh/onnx-request.json |  1 +
 tests/resources/modelmesh/triton-onnx.yaml  | 14 +++++++
 3 files changed, 55 insertions(+), 5 deletions(-)
 create mode 100644 tests/resources/modelmesh/onnx-request.json
 create mode 100644 tests/resources/modelmesh/triton-onnx.yaml

diff --git a/tests/basictests/modelmesh.sh b/tests/basictests/modelmesh.sh
index 35eea09a7..f5edf55a2 100755
--- a/tests/basictests/modelmesh.sh
+++ b/tests/basictests/modelmesh.sh
@@ -8,7 +8,9 @@ source ${MY_DIR}/../util
 RESOURCEDIR="${MY_DIR}/../resources"
 
 MODEL_PROJECT="${ODHPROJECT}-model"
+TRITON_MODEL_PROJECT="${ODHPROJECT}-model-triton"
 PREDICTOR_NAME="example-sklearn-isvc"
+TRITON_ONNX_INF_NAME="example-onnx-mnist"
 
 
 os::test::junit::declare_suite_start "$MY_SCRIPT"
@@ -21,7 +23,7 @@ function check_resources() {
     os::cmd::try_until_text "oc get service modelmesh-serving" "modelmesh-serving" $odhdefaulttimeout $odhdefaultinterval
 }
 
-function setup_test_serving_namespace() {
+function setup_test_serving_namespace_mlserver() {
     oc new-project ${MODEL_PROJECT}
     header "Setting up test modelmesh serving in ${MODEL_PROJECT}"
     SECRETKEY=$(openssl rand -hex 32)
@@ -35,6 +37,20 @@ function setup_test_serving_namespace() {
     oc project ${ODHPROJECT}
 }
 
+function setup_test_serving_namespace_triton_onnx() {
+    oc new-project ${TRITON_MODEL_PROJECT}
+    header "Setting up triton onnx serving in ${TRITON_MODEL_PROJECT}"
+    SECRETKEY=$(openssl rand -hex 32)
+    sed -i "s/<secretkey>/$SECRETKEY/g" ${RESOURCEDIR}/modelmesh/sample-minio.yaml
+    os::cmd::expect_success "oc apply -f ${RESOURCEDIR}/modelmesh/sample-minio.yaml -n ${TRITON_MODEL_PROJECT}"
+    os::cmd::try_until_text "oc get pods -n ${TRITON_MODEL_PROJECT} -l app=minio --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}' | wc -w" "1" $odhdefaulttimeout $odhdefaultinterval
+    os::cmd::expect_success "oc apply -f ${RESOURCEDIR}/modelmesh/triton-onnx.yaml -n ${TRITON_MODEL_PROJECT}"
+    os::cmd::try_until_text "oc get pods -n ${ODHPROJECT} -l app=odh-model-controller --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}' | wc -w" "3" $odhdefaulttimeout $odhdefaultinterval
+    os::cmd::try_until_text "oc get pods -n ${TRITON_MODEL_PROJECT} -l name=modelmesh-serving-triton-2.x --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}' | wc -w" "2" $odhdefaulttimeout $odhdefaultinterval
+    os::cmd::try_until_text "oc get inferenceservice -n ${TRITON_MODEL_PROJECT} example-onnx-mnist -o jsonpath='{$.status.modelStatus.states.activeModelState}'" "Loaded" $odhdefaulttimeout $odhdefaultinterval
+    oc project ${ODHPROJECT}
+}
+
 function teardown_test_serving() {
     header "Tearing down test modelmesh serving"
     oc project ${MODEL_PROJECT}
@@ -43,6 +59,13 @@ function teardown_test_serving() {
     os::cmd::expect_success "oc delete -f ${RESOURCEDIR}/modelmesh/mnist-sklearn.yaml"
     os::cmd::try_until_text "oc get pods -l name=modelmesh-serving-mlserver-0.x --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}' | wc -w" "0" $odhdefaulttimeout $odhdefaultinterval
     os::cmd::expect_success "oc delete project ${MODEL_PROJECT}"
+
+    oc project ${TRITON_MODEL_PROJECT}
+    os::cmd::expect_success "oc delete -f ${RESOURCEDIR}/modelmesh/sample-minio.yaml"
+    os::cmd::try_until_text "oc get pods -l app=minio --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}' | wc -w" "0" $odhdefaulttimeout $odhdefaultinterval
+    os::cmd::expect_success "oc delete -f ${RESOURCEDIR}/modelmesh/triton-onnx.yaml"
+    os::cmd::try_until_text "oc get pods -l name=modelmesh-serving-triton-2.x --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}' | wc -w" "0" $odhdefaulttimeout $odhdefaultinterval
+    os::cmd::expect_success "oc delete project ${TRITON_MODEL_PROJECT}"
     oc project ${ODHPROJECT}
 }
 
@@ -50,8 +73,18 @@ function test_inferences() {
     header "Testing inference from example mnist model"
     oc project ${MODEL_PROJECT}
     route=$(oc get route ${PREDICTOR_NAME} --template={{.spec.host}})
+    path=$(oc get route ${PREDICTOR_NAME} --template={{.spec.path}})
     # The result of the inference call should give us back an "8" for the data part of the output
-    os::cmd::try_until_text "curl -X POST -k ${route}/infer -d '{\"inputs\": [{ \"name\": \"predict\", \"shape\": [1, 64], \"datatype\": \"FP32\", \"data\": [0.0, 0.0, 1.0, 11.0, 14.0, 15.0, 3.0, 0.0, 0.0, 1.0, 13.0, 16.0, 12.0, 16.0, 8.0, 0.0, 0.0, 8.0, 16.0, 4.0, 6.0, 16.0, 5.0, 0.0, 0.0, 5.0, 15.0, 11.0, 13.0, 14.0, 0.0, 0.0, 0.0, 0.0, 2.0, 12.0, 16.0, 13.0, 0.0, 0.0, 0.0, 0.0, 0.0, 13.0, 16.0, 16.0, 6.0, 0.0, 0.0, 0.0, 0.0, 16.0, 16.0, 16.0, 7.0, 0.0, 0.0, 0.0, 0.0, 11.0, 13.0, 12.0, 1.0, 0.0]}]}' | jq '.outputs[0].data[0]'" "8" $odhdefaulttimeout $odhdefaultinterval
+    os::cmd::try_until_text "curl -X POST -k https://${route}${path}/infer -d '{\"inputs\": [{ \"name\": \"predict\", \"shape\": [1, 64], \"datatype\": \"FP32\", \"data\": [0.0, 0.0, 1.0, 11.0, 14.0, 15.0, 3.0, 0.0, 0.0, 1.0, 13.0, 16.0, 12.0, 16.0, 8.0, 0.0, 0.0, 8.0, 16.0, 4.0, 6.0, 16.0, 5.0, 0.0, 0.0, 5.0, 15.0, 11.0, 13.0, 14.0, 0.0, 0.0, 0.0, 0.0, 2.0, 12.0, 16.0, 13.0, 0.0, 0.0, 0.0, 0.0, 0.0, 13.0, 16.0, 16.0, 6.0, 0.0, 0.0, 0.0, 0.0, 16.0, 16.0, 16.0, 7.0, 0.0, 0.0, 0.0, 0.0, 11.0, 13.0, 12.0, 1.0, 0.0]}]}' | jq '.outputs[0].data[0]'" "8" $odhdefaulttimeout $odhdefaultinterval
+    oc project ${ODHPROJECT}
+}
+
+function test_inferences_triton_onnx() {
+    header "Testing inference from example mnist model"
+    oc project ${TRITON_MODEL_PROJECT}
+    route=$(oc get route ${TRITON_ONNX_INF_NAME} --template={{.spec.host}})
+    path=$(oc get route ${TRITON_ONNX_INF_NAME} --template={{.spec.path}})
+    os::cmd::expect_success "curl -X POST -k https://${route}${path}/infer -d @${RESOURCEDIR}/modelmesh/onnx-request.json"
     oc project ${ODHPROJECT}
 }
 
@@ -61,15 +94,17 @@ function setup_monitoring() {
 }
 
 function test_metrics() {
-    header "Checking metrics for total models loaded, should be 1 since we have 1 model being served"   
+    header "Checking metrics for total models loaded, should be 2 since we have 2 models being served"
     monitoring_token=$(oc sa get-token prometheus-k8s -n openshift-monitoring)
-    os::cmd::try_until_text "oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- curl -k -H \"Authorization: Bearer $monitoring_token\" 'https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query?query=modelmesh_models_loaded_total' | jq '.data.result[0].value[1]'" "1" $odhdefaulttimeout $odhdefaultinterval
+    os::cmd::try_until_text "oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- curl -k -H \"Authorization: Bearer $monitoring_token\" 'https://thanos-querier.openshift-monitoring.svc:9091/api/v1/query?query=modelmesh_models_loaded_total' | jq '.data.result[0].value[1]'" "2" $odhdefaulttimeout $odhdefaultinterval
 }
 
 setup_monitoring
 check_resources
-setup_test_serving_namespace
+setup_test_serving_namespace_mlserver
+setup_test_serving_namespace_triton_onnx
 test_inferences
+test_inferences_triton_onnx
 test_metrics
 teardown_test_serving
 
diff --git a/tests/resources/modelmesh/onnx-request.json b/tests/resources/modelmesh/onnx-request.json
new file mode 100644
index 000000000..22fb1198a
--- /dev/null
+++ b/tests/resources/modelmesh/onnx-request.json
@@ -0,0 +1 @@
+{ "model_name": "example-onnx-mnist", "inputs": [{ "name": "Input3", "shape": [1, 1, 28, 28], "datatype": "FP32", "data": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01176471, 0.07058824, 0.07058824, 0.07058824, 0.49411765, 0.53333336, 0.6862745, 0.10196079, 0.6509804, 1.0, 0.96862745, 0.49803922, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11764706, 0.14117648, 0.36862746, 0.6039216, 0.6666667, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.88235295, 0.6745098, 0.99215686, 0.9490196, 0.7647059, 0.2509804, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19215687, 0.93333334, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.9843137, 0.3647059, 0.32156864, 0.32156864, 0.21960784, 0.15294118, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07058824, 0.85882354, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.7764706, 0.7137255, 0.96862745, 0.94509804, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3137255, 0.6117647, 0.41960785, 0.99215686, 0.99215686, 0.8039216, 0.04313726, 0.0, 0.16862746, 0.6039216, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05490196, 0.00392157, 0.6039216, 0.99215686, 0.3529412, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.54509807, 0.99215686, 0.74509805, 0.00784314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04313726, 0.74509805, 0.99215686, 0.27450982, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.13725491, 0.94509804, 0.88235295, 0.627451, 0.42352942, 0.00392157, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.31764707, 0.9411765, 0.99215686, 0.99215686, 0.46666667, 0.09803922, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1764706, 0.7294118, 0.99215686, 0.99215686, 0.5882353, 0.10588235, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0627451, 0.3647059, 0.9882353, 0.99215686, 0.73333335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.9764706, 0.99215686, 0.9764706, 0.2509804, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.18039216, 0.50980395, 0.7176471, 0.99215686, 0.99215686, 0.8117647, 0.00784314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.15294118, 0.5803922, 0.8980392, 0.99215686, 0.99215686, 0.99215686, 0.98039216, 0.7137255, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.09411765, 0.44705883, 0.8666667, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.7882353, 0.30588236, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.09019608, 0.25882354, 0.8352941, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.7764706, 0.31764707, 0.00784314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07058824, 0.67058825, 0.85882354, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.7647059, 0.3137255, 0.03529412, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.21568628, 0.6745098, 0.8862745, 0.99215686, 0.99215686, 0.99215686, 0.99215686, 0.95686275, 0.52156866, 0.04313726, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.53333336, 0.99215686, 0.99215686, 0.99215686, 0.83137256, 0.5294118, 0.5176471, 0.0627451, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] }]}
\ No newline at end of file
diff --git a/tests/resources/modelmesh/triton-onnx.yaml b/tests/resources/modelmesh/triton-onnx.yaml
new file mode 100644
index 000000000..d746bdeae
--- /dev/null
+++ b/tests/resources/modelmesh/triton-onnx.yaml
@@ -0,0 +1,14 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  name: example-onnx-mnist
+  annotations:
+    serving.kserve.io/deploymentMode: ModelMesh
+spec:
+  predictor:
+    model:
+      modelFormat:
+        name: onnx
+      storage:
+        key: localMinIO
+        path: onnx/mnist.onnx