Fix the replica count for kserve

backend/src/types.ts

@@ -893,7 +893,7 @@ export type ServingRuntime = K8sResourceCommon & {
       volumeMounts?: VolumeMount[];
     }[];
     supportedModelFormats: SupportedModelFormats[];
-    replicas: number;
+    replicas?: number;
     volumes?: Volume[];
   };
 };

frontend/src/__mocks__/mockServingRuntimeTemplateK8sResource.ts

@@ -5,13 +5,17 @@ type MockResourceConfigType = {
   name?: string;
   namespace?: string;
   displayName?: string;
+  replicas?: number;
   platforms?: ServingRuntimePlatform[];
+  isModelmesh?: boolean;
 };
 
 export const mockServingRuntimeTemplateK8sResource = ({
   name = 'template-1',
   namespace = 'opendatahub',
   displayName = 'New OVMS Server',
+  replicas = 1,
+  isModelmesh = false,
   platforms,
 }: MockResourceConfigType): TemplateKind => ({
   apiVersion: 'template.openshift.io/v1',
@@ -76,7 +80,7 @@ export const mockServingRuntimeTemplateK8sResource = ({
         grpcEndpoint: 'port:8085',
         multiModel: true,
         protocolVersions: ['grpc-v1'],
-        replicas: 1,
+        ...(isModelmesh && { replicas }),
         supportedModelFormats: [
           {
             autoSelect: true,

frontend/src/api/__tests__/inferenceServices.spec.ts

@@ -105,4 +105,53 @@ describe('assembleInferenceService', () => {
     expect(inferenceService.spec.predictor.tolerations).toBeUndefined();
     expect(inferenceService.spec.predictor.model.resources).toBeUndefined();
   });
+
+  it('should provide max and min replicas if provided', async () => {
+    const replicaCount = 2;
+
+    const acceleratorProfileState: AcceleratorProfileState = {
+      acceleratorProfile: mockAcceleratorProfile({}),
+      acceleratorProfiles: [mockAcceleratorProfile({})],
+      initialAcceleratorProfile: mockAcceleratorProfile({}),
+      count: 1,
+      additionalOptions: {},
+      useExisting: false,
+    };
+
+    const inferenceService = assembleInferenceService(
+      mockInferenceServiceModalData({}),
+      undefined,
+      undefined,
+      true,
+      undefined,
+      acceleratorProfileState,
+      replicaCount,
+    );
+
+    expect(inferenceService.spec.predictor.maxReplicas).toBe(replicaCount);
+    expect(inferenceService.spec.predictor.minReplicas).toBe(replicaCount);
+  });
+
+  it('should omit replica count if not provided', async () => {
+    const acceleratorProfileState: AcceleratorProfileState = {
+      acceleratorProfile: mockAcceleratorProfile({}),
+      acceleratorProfiles: [mockAcceleratorProfile({})],
+      initialAcceleratorProfile: mockAcceleratorProfile({}),
+      count: 1,
+      additionalOptions: {},
+      useExisting: false,
+    };
+
+    const inferenceService = assembleInferenceService(
+      mockInferenceServiceModalData({}),
+      undefined,
+      undefined,
+      true,
+      undefined,
+      acceleratorProfileState,
+    );
+
+    expect(inferenceService.spec.predictor.maxReplicas).toBeUndefined();
+    expect(inferenceService.spec.predictor.minReplicas).toBeUndefined();
+  });
 });

frontend/src/api/__tests__/servingRuntimes.spec.ts

@@ -134,4 +134,40 @@ describe('assembleServingRuntime', () => {
       servingRuntime.spec.containers[0].resources?.requests?.['nvidia.com/gpu'],
     ).toBeUndefined();
   });
+
+  it('should have replica count on modelmesh', async () => {
+    const replicaCount = 2;
+    const servingRuntime = assembleServingRuntime(
+      mockServingRuntimeModalData({
+        externalRoute: true,
+        tokenAuth: true,
+        numReplicas: replicaCount,
+      }),
+      'test',
+      mockServingRuntimeK8sResource({ auth: false, route: false }),
+      true,
+      false,
+      undefined,
+      true,
+    );
+
+    expect(servingRuntime.spec.replicas).toBe(replicaCount);
+  });
+
+  it('should have replica count on modelmesh', async () => {
+    const servingRuntime = assembleServingRuntime(
+      mockServingRuntimeModalData({
+        externalRoute: true,
+        tokenAuth: true,
+      }),
+      'test',
+      mockServingRuntimeK8sResource({ auth: false, route: false }),
+      true,
+      false,
+      undefined,
+      false,
+    );
+
+    expect(servingRuntime.spec.replicas).toBeUndefined();
+  });
 });

frontend/src/api/k8s/inferenceServices.ts

@@ -23,6 +23,7 @@ export const assembleInferenceService = (
   isModelMesh?: boolean,
   inferenceService?: InferenceServiceKind,
   acceleratorState?: AcceleratorProfileState,
+  replicaCount?: number,
 ): InferenceServiceKind => {
   const { storage, format, servingRuntimeName, project } = data;
   const name = editName || translateDisplayNameForK8s(data.name);
@@ -70,6 +71,8 @@ export const assembleInferenceService = (
         },
         spec: {
           predictor: {
+            ...(replicaCount && { minReplicas: replicaCount }),
+            ...(replicaCount && { maxReplicas: replicaCount }),
             model: {
               modelFormat: {
                 name: format.name,
@@ -149,6 +152,7 @@ export const createInferenceService = (
   secretKey?: string,
   isModelMesh?: boolean,
   acceleratorState?: AcceleratorProfileState,
+  replicaCount?: number,
 ): Promise<InferenceServiceKind> => {
   const inferenceService = assembleInferenceService(
     data,
@@ -157,6 +161,7 @@ export const createInferenceService = (
     isModelMesh,
     undefined,
     acceleratorState,
+    replicaCount,
   );
   return k8sCreateResource<InferenceServiceKind>({
     model: InferenceServiceModel,
@@ -170,6 +175,7 @@ export const updateInferenceService = (
   secretKey?: string,
   isModelMesh?: boolean,
   acceleratorState?: AcceleratorProfileState,
+  replicaCount?: number,
 ): Promise<InferenceServiceKind> => {
   const inferenceService = assembleInferenceService(
     data,
@@ -178,6 +184,7 @@ export const updateInferenceService = (
     isModelMesh,
     existingData,
     acceleratorState,
+    replicaCount,
   );
 
   return k8sUpdateResource<InferenceServiceKind>({

frontend/src/api/k8s/servingRuntimes.ts

@@ -86,7 +86,11 @@ export const assembleServingRuntime = (
       },
     };
   }
-  updatedServingRuntime.spec.replicas = numReplicas;
+
+  delete updatedServingRuntime.spec.replicas;
+  if (isModelMesh) {
+    updatedServingRuntime.spec.replicas = numReplicas;
+  }
 
   // Accelerator support
 

frontend/src/k8sTypes.ts

@@ -341,7 +341,7 @@ export type ServingRuntimeKind = K8sResourceCommon & {
     };
     containers: ServingContainer[];
     supportedModelFormats: SupportedModelFormats[];
-    replicas: number;
+    replicas?: number;
     tolerations?: Toleration[];
     volumes?: Volume[];
   };
@@ -387,6 +387,8 @@ export type InferenceServiceKind = K8sResourceCommon & {
           schemaPath?: string;
         };
       };
+      maxReplicas?: number;
+      minReplicas?: number;
     };
   };
   status?: {

frontend/src/pages/modelServing/screens/projects/kServeModal/ManageKServeModal.tsx

@@ -145,6 +145,8 @@ const ManageKServeModal: React.FC<ManageKServeModalProps> = ({
       editInfo?.inferenceServiceEditInfo?.spec.predictor.model.runtime ||
       translateDisplayNameForK8s(createDataInferenceService.name);
 
+    const replicaCount = createDataServingRuntime.numReplicas;
+
     submitServingRuntimeResources(
       servingRuntimeSelected,
       createDataServingRuntime,
@@ -165,6 +167,7 @@ const ManageKServeModal: React.FC<ManageKServeModalProps> = ({
           servingRuntimeName,
           false,
           acceleratorProfileState,
+          replicaCount,
         ),
       )
       .then(() => onSuccess())

frontend/src/pages/modelServing/screens/projects/utils.ts

@@ -258,6 +258,7 @@ const createInferenceServiceAndDataConnection = (
   editInfo?: InferenceServiceKind,
   isModelMesh?: boolean,
   acceleratorProfileState?: AcceleratorProfileState,
+  replicaCount?: number,
 ) => {
   if (!existingStorage) {
     return createAWSSecret(inferenceServiceData).then((secret) =>
@@ -268,12 +269,14 @@ const createInferenceServiceAndDataConnection = (
             secret.metadata.name,
             isModelMesh,
             acceleratorProfileState,
+            replicaCount,
           )
         : createInferenceService(
             inferenceServiceData,
             secret.metadata.name,
             isModelMesh,
             acceleratorProfileState,
+            replicaCount,
           ),
     );
   }
@@ -284,8 +287,15 @@ const createInferenceServiceAndDataConnection = (
         undefined,
         isModelMesh,
         acceleratorProfileState,
+        replicaCount,
       )
-    : createInferenceService(inferenceServiceData, undefined, isModelMesh, acceleratorProfileState);
+    : createInferenceService(
+        inferenceServiceData,
+        undefined,
+        isModelMesh,
+        acceleratorProfileState,
+        replicaCount,
+      );
 };
 
 export const submitInferenceServiceResource = (
@@ -294,6 +304,7 @@ export const submitInferenceServiceResource = (
   servingRuntimeName?: string,
   isModelMesh?: boolean,
   acceleratorProfileState?: AcceleratorProfileState,
+  replicaCount?: number,
 ): Promise<InferenceServiceKind> => {
   const inferenceServiceData = {
     ...createData,
@@ -311,6 +322,7 @@ export const submitInferenceServiceResource = (
     editInfo,
     isModelMesh,
     acceleratorProfileState,
+    replicaCount,
   );
 };