Support Caikit Serving Runtime with REST

opendatahub-io · Nov 10, 2023 · eb6bab1 · eb6bab1
1 parent 555f354
commit eb6bab1
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 25 deletions.
diff --git a/manifests/modelserving/caikit-ootb.yaml b/manifests/modelserving/caikit-ootb.yaml
@@ -17,30 +17,33 @@ objects:
 - apiVersion: serving.kserve.io/v1alpha1
   kind: ServingRuntime
   metadata:
-    name: caikit-runtime
-    annotations:
-      openshift.io/display-name: Caikit TGIS ServingRuntime for KServe
-    labels:
-      opendatahub.io/dashboard: 'true'
+    name: caikit-tgis-runtime
+  annotations:
+    openshift.io/display-name: Caikit TGIS ServingRuntime for KServe
+  labels:
+    opendatahub.io/dashboard: 'true'
   spec:
+    multiModel: false
+    supportedModelFormats:
+      - autoSelect: true
+        name: caikit
     containers:
-      - env:
+      - name: kserve-container
+        image: text-generation-inference
+        command:
+          - text-generation-launcher
+        args:
+          - --model-name=/mnt/models/artifacts/
+        env:
+          - name: TRANSFORMERS_CACHE
+            value: /tmp/transformers_cache
+      - name: transformer-container
+        image: caikit-tgis-serving
+        env:
           - name: RUNTIME_LOCAL_MODELS_DIR
             value: /mnt/models
-        image: caikit-1
-        name: kserve-container
         ports:
-        - containerPort: 8085
-          name: h2c
-          protocol: TCP
-          resources:
-            limits:
-              cpu: '0'
-              memory: 0Gi
-            requests:
-              cpu: '0'
-              memory: 0Gi
-    multiModel: false
-    supportedModelFormats:
-    - autoSelect: true
-      name: caikit
+          - containerPort: 8080
+            name: h2c
+            protocol: TCP
+parameters: []
diff --git a/manifests/modelserving/kustomization.yaml b/manifests/modelserving/kustomization.yaml
@@ -10,6 +10,9 @@ images:
   - name: ovms-1
     newName: quay.io/opendatahub/openvino_model_server
     digest: sha256:2cbe8a48ab0bc6fe7fb76919bf33253e83a6218a9c4b486b744c3dcf30679616
-  - name: caikit-1
-    newName: quay.io/opendatahub/caikit-tgis-serving:stable
-    digest: sha256:794adc22d52cb3ac4b5aadfb286e8431cca829acdc4909719329cf8c4fabb4ec
+  - name: caikit-tgis-serving
+    newName: quay.io/opendatahub/caikit-tgis-serving:pr-131
+    digest: sha256:217783efa662d2cc6569bd600f94bfb1f11acc6e3437fa7855617c7b4a6bdfb8
+  - name: text-generation-inference
+    newName: quay.io/opendatahub/text-generation-inference:fast
+    digest: sha256:quay.io/opendatahub/text-generation-inference