[Sync - 20240909] Upstream master to ODH master

.github/actions/minikube-setup/action.yml

@@ -7,16 +7,14 @@ runs:
     - name: Install kubectl
       uses: azure/[email protected]
       with:
-        version: 'v1.27.4'
+        version: 'v1.29.7'
 
     - name: Setup Minikube
       uses: medyagh/setup-minikube@latest
       with:
-        minikube-version: '1.32.0'
-        kubernetes-version: 'v1.27.4'
+        minikube-version: '1.33.1'
+        kubernetes-version: 'v1.29.7'
         driver: 'none'
-        network-plugin: 'cni'
-        cni: 'calico'
         wait: 'all'
         start-args: --wait-timeout=6m0s
 

.github/workflows/python-publish.yml

@@ -21,16 +21,14 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: "3.x"
+          python-version: "3.11"
 
       - name: Install Poetry and version plugin
         run: ./test/scripts/gh-actions/setup-poetry.sh
 
       - name: Build and publish
         env:
-          POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }}
         run: |
           cd python/kserve
-          poetry build
-          poetry publish
+          poetry publish --build

.github/workflows/scheduled-go-security-scan.yml

@@ -17,12 +17,12 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Run Gosec Security Scanner
-        uses: securego/gosec@master
+        uses: securego/gosec@v2.20.0
         with:
           args: '-no-fail -fmt=sarif -out=go-security-scan-results.sarif -exclude-dir=pkg/client -exclude-dir=pkg/clientv1alpha1 ./...'
 
       - name: Upload SARIF file to Github Code Scanning
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: go-security-scan-results.sarif
           category: gosec-tool

.github/workflows/scheduled-image-scan.yml

@@ -43,6 +43,14 @@ jobs:
             --sarif-file-output=./application/${{ matrix.image.name }}/docker.snyk.sarif
           sarif: false
 
+      # Replace any "undefined" security severity values with 0. The undefined value is used in the case
+      # of license-related findings, which do not indicate a security vulnerability.
+      # See https://github.com/github/codeql-action/issues/2187 for more context.
+      # This can be removed once https://github.com/snyk/cli/pull/5409 is merged.
+      - name: Replace security-severity undefined for license-related findings
+        run: |
+          sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' ./application/${{ matrix.image.name }}/docker.snyk.sarif
+
       - name: Upload sarif file to Github Code Scanning
         if: always()
         continue-on-error: true #avoid fail the pipeline if the SARIF upload fails.
@@ -80,9 +88,17 @@ jobs:
             --sarif-file-output=./application/${{ matrix.image.name }}/docker.snyk.sarif
           sarif: false
 
+      # Replace any "undefined" security severity values with 0. The undefined value is used in the case
+      # of license-related findings, which do not indicate a security vulnerability.
+      # See https://github.com/github/codeql-action/issues/2187 for more context.
+      # This can be removed once https://github.com/snyk/cli/pull/5409 is merged.
+      - name: Replace security-severity undefined for license-related findings
+        run: |
+          sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' ./application/${{ matrix.image.name }}/docker.snyk.sarif
+
       - name: Upload sarif file to Github Code Scanning
         if: always()
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: application/${{ matrix.image.name }}/docker.snyk.sarif
 
@@ -113,8 +129,16 @@ jobs:
             --sarif-file-output=./application/${{ matrix.image.name }}/docker.snyk.sarif
           sarif: false
 
+      # Replace any "undefined" security severity values with 0. The undefined value is used in the case
+      # of license-related findings, which do not indicate a security vulnerability.
+      # See https://github.com/github/codeql-action/issues/2187 for more context.
+      # This can be removed once https://github.com/snyk/cli/pull/5409 is merged.
+      - name: Replace security-severity undefined for license-related findings
+        run: |
+          sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' ./application/${{ matrix.image.name }}/docker.snyk.sarif
+
       - name: Upload sarif file to Github Code Scanning
         if: always()
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: application/${{ matrix.image.name }}/docker.snyk.sarif

Dockerfile

@@ -25,4 +25,4 @@ COPY third_party/ /third_party/
 COPY --from=builder /go/src/github.com/kserve/kserve/manager /
 USER 1000:1000
 
-ENTRYPOINT ["/manager"]
+ENTRYPOINT ["/manager"]

Makefile

@@ -1,7 +1,7 @@
 
 # Base Image URL
-BASE_IMG ?= python:3.9-slim-bullseye
-PMML_BASE_IMG ?= openjdk:11-slim
+BASE_IMG ?= python:3.11-slim-bookworm
+PMML_BASE_IMG ?= openjdk:21-slim-bookworm
 
 # Image URL to use all building/pushing image targets
 IMG ?= kserve-controller:latest
@@ -24,7 +24,7 @@ QPEXT_IMG ?= qpext:latest
 CRD_OPTIONS ?= "crd:maxDescLen=0"
 KSERVE_ENABLE_SELF_SIGNED_CA ?= false
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.27
+ENVTEST_K8S_VERSION = 1.29
 SUCCESS_200_ISVC_IMG ?= success-200-isvc
 ERROR_404_ISVC_IMG ?= error-404-isvc
 
@@ -74,10 +74,10 @@ deploy: manifests
 	cd config/default && if [ ${KSERVE_ENABLE_SELF_SIGNED_CA} != false ]; then \
 	echo > ../certmanager/certificate.yaml; \
 	else git checkout HEAD -- ../certmanager/certificate.yaml; fi;
-	kubectl apply -k config/default
+	kubectl apply --server-side=true -k config/default
 	if [ ${KSERVE_ENABLE_SELF_SIGNED_CA} != false ]; then ./hack/self-signed-ca.sh; fi;
 	kubectl wait --for=condition=ready pod -l control-plane=kserve-controller-manager -n kserve --timeout=300s
-	kubectl apply -k config/clusterresources
+	kubectl apply  --server-side=true  -k config/clusterresources
 	git checkout HEAD -- config/certmanager/certificate.yaml
 
 
@@ -87,11 +87,11 @@ deploy-dev: manifests
 	cd config/default && if [ ${KSERVE_ENABLE_SELF_SIGNED_CA} != false ]; then \
 	echo > ../certmanager/certificate.yaml; \
 	else git checkout HEAD -- ../certmanager/certificate.yaml; fi;
-	kubectl apply -k config/overlays/development
+	kubectl apply --server-side=true -k config/overlays/development
 	if [ ${KSERVE_ENABLE_SELF_SIGNED_CA} != false ]; then ./hack/self-signed-ca.sh; fi;
 	# TODO: Add runtimes as part of default deployment
 	kubectl wait --for=condition=ready pod -l control-plane=kserve-controller-manager -n kserve --timeout=300s
-	kubectl apply -k config/clusterresources
+	kubectl apply --server-side=true -k config/clusterresources
 	git checkout HEAD -- config/certmanager/certificate.yaml
 
 deploy-dev-sklearn: docker-push-sklearn
@@ -114,13 +114,13 @@ deploy-dev-huggingface: docker-push-huggingface
 
 deploy-dev-storageInitializer: docker-push-storageInitializer
 	./hack/storageInitializer_patch_dev.sh ${KO_DOCKER_REPO}/${STORAGE_INIT_IMG}
-	kubectl apply -k config/overlays/dev-image-config
+	kubectl apply --server-side=true -k config/overlays/dev-image-config
 
 deploy-ci: manifests
-	kubectl apply -k config/overlays/test
+	kubectl apply --server-side=true -k config/overlays/test
 	# TODO: Add runtimes as part of default deployment
 	kubectl wait --for=condition=ready pod -l control-plane=kserve-controller-manager -n kserve --timeout=300s
-	kubectl apply -k config/overlays/test/clusterresources
+	kubectl apply --server-side=true -k config/overlays/test/clusterresources
 
 deploy-helm: manifests
 	helm install kserve-crd charts/kserve-crd/ --wait --timeout 180s

ROADMAP.md

@@ -1,16 +1,38 @@
-# KServe 2023 Roadmap
+# KServe 2024-2025 Roadmap
+## Objective: "Support GenAI inference"
+- LLM Serving Runtimes
+   * Support Speculative Decoding with vLLM runtime [https://github.com/kserve/kserve/issues/3800].
+   * Support LoRA adapters [https://github.com/kserve/kserve/issues/3750].
+   * Support LLM Serving runtimes for TensorRT-LLM, TGI and provide benchmarking comparisons [https://github.com/kserve/kserve/issues/3868].
+   * Support multi-host, multi-GPU inference runtime [https://github.com/kserve/kserve/issues/2145].
+
+- LLM Autoscaling
+   * Support Model Caching with automatic PV/PVC provisioning [https://github.com/kserve/kserve/issues/3869].
+   * Support Autoscaling settings for serving runtimes.
+   * Support Autoscaling based on custom metrics [https://github.com/kserve/kserve/issues/3561].
+
+- LLM RAG/Agent Pipeline Orchestration
+   * Support declarative RAG/Agent workflow using KServe Inference Graph [https://github.com/kserve/kserve/issues/3829].
+
+-  Open Inference Protocol extension to GenAI Task APIs
+   * Community-maintained Open Inference Protocol repo for OpenAI schema [https://docs.google.com/document/d/1odTMdIFdm01CbRQ6CpLzUIGVppHSoUvJV_zwcX6GuaU].
+   * Support vertical GenAI Task APIs such as embedding, Text-to-Image, Text-To-Code, Doc-To-Text [https://github.com/kserve/kserve/issues/3572].
+
+- LLM Gateway
+   * Support multiple LLM providers.
+   * Support token based rate limiting.
+   * Support LLM router with traffic shaping, fallback, load balancing.
+   * LLM Gateway observability for metrics and cost reporting
 
 ## Objective: "Graduate core inference capability to stable/GA"
-- Promote `InferenceService` and `ClusterServingRuntime`/`ServingRuntime` CRD from v1beta1 to v1 
+- Promote `InferenceService` and `ClusterServingRuntime`/`ServingRuntime` CRD to v1
   * Improve `InferenceService` CRD for REST/gRPC protocol interface
-  * Unify model storage spec and implementation between KServe and ModelMesh
-  * Add Status to `ServingRuntime` for both ModelMesh and KServe, surface `ServingRuntime` validation errors and deployment status
-  * Deprecate `TrainedModel` CRD and use `InferenceService` annotation to allow dynamic model updates as alternative option to storage initializer
-  * Collocate transformer and predictor in the pod to reduce sidecar resources and networking latency
-  * Stablize `RawDeployment` mode with comprehensive testing for supported features
-
-- All model formats to support v2 inference protocol including custom serving runtime
-  * TorchServe to support v2 gRPC inference protocol
+  * Improve model storage interface 
+  * Deprecate `TrainedModel` CRD and add multiple model support for co-hosting, draft model, LoRA adapters to InferenceService.
+  * Improve YAML UX for predictor and transformer container collocation.
+  * Close the feature gap between `RawDeployment` and `Serverless` mode.
+
+- Open Inference Protocol 
   * Support batching for v2 inference protocol
   * Transformer and Explainer v2 inference protocol interoperability
   * Improve codec for v2 inference protocol
@@ -19,30 +41,19 @@ Reference: [Control plane issues](https://github.com/kserve/kserve/issues?q=is%3
 
 ## Objective: "Graduate KServe Python SDK to 1.0“
 
-- Improve KServe Python SDK dependency management with Poetry
-- Create standarized model packaging API
-- Improve KServe model server observability with metrics and distruted tracing
+- Create standardized model packaging API
+- Improve KServe model server observability with metrics and distributed tracing
 - Support batch inference
 
 Reference：[Python SDK issues](https://github.com/kserve/kserve/issues?q=is%3Aissue+is%3Aopen+label%3Akserve%2Fsdk), [Storage issues](https://github.com/kserve/kserve/issues?q=is%3Aissue+is%3Aopen+label%3Akfserving%2Fstorage)
 
-## Objective: "Graduate ModelMesh to beta"
-- Support TorchServe ServingRuntime
-- Add PVC support and unify storage implementation with KServe
-- Add optional ingress for ModelMesh deployments
-- Etcd secret security for multi-namespace mode
-- Add estimated model size field
-
-Reference: [ModelMesh issues](https://github.com/kserve/modelmesh-serving/issues?page=1&q=is%3Aissue+is%3Aopen)
-
-## Objective: "Graduate InferenceGraph to beta"
+## Objective: "Graduate InferenceGraph"
 - Improve `InferenceGraph` spec for replica and concurrency control
-- Allow setting resource limits per `InferenceGraph`
 - Support distributed tracing
 - Support gRPC for `InferenceGraph`
 - Standalone `Transformer` support for `InferenceGraph`
 - Support traffic mirroring node
-- Support `RawDeployment` mode for `InferenceGraph`
+- Improve `RawDeployment` mode for `InferenceGraph`
 
 Reference: [InferenceGraph issues](https://github.com/kserve/kserve/issues?q=is%3Aissue+is%3Aopen+label%3Akserve%2Finference_graph)
 
@@ -58,5 +69,5 @@ Reference: [Auth related issues](https://github.com/kserve/kserve/issues?q=is%3A
 - Add ModelMesh docs and explain the use cases for classic KServe and ModelMesh
 - Unify the data plane v1 and v2 page formats
 - Improve v2 data plane docs to tell the story why and what changed
-- Clean up the examples in kserve repo and unify them with the website's by creating one source of truth for example documentation
+- Clean up the examples in kserve repo and unify them with the website's by creating one source of truth for documentation
 - Update any out-of-date documentation and make sure the website as a whole is consistent and cohesive

agent.Dockerfile

@@ -27,4 +27,4 @@ WORKDIR /ko-app
 COPY --from=builder /go/src/github.com/kserve/kserve/agent /ko-app/
 USER 1000:1000
 
-ENTRYPOINT ["/ko-app/agent"]
+ENTRYPOINT ["/ko-app/agent"]

charts/kserve-crd-minimal/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: kserve-crd-minimal
-version: v0.13.0
+version: v0.14.0-rc0
 description: Helm chart for deploying minimal kserve crds without validation
 keywords:
   - kserve

charts/kserve-crd-minimal/README.md

@@ -2,12 +2,12 @@
 
 Helm chart for deploying minimal kserve crds without validation
 
-![Version: v0.13.0](https://img.shields.io/badge/Version-v0.13.0-informational?style=flat-square)
+![Version: v0.14.0-rc0](https://img.shields.io/badge/Version-v0.14.0--rc0-informational?style=flat-square)
 
 ## Installing the Chart
 
 To install the chart, run the following:
 
 ```console
-$ helm install kserve-crd-minimal oci://ghcr.io/kserve/charts/kserve-crd-minimal --version v0.13.0
+$ helm install kserve-crd-minimal oci://ghcr.io/kserve/charts/kserve-crd-minimal --version v0.14.0-rc0
 ```

charts/kserve-crd/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: kserve-crd
-version: v0.13.0
+version: v0.14.0-rc0
 description: Helm chart for deploying kserve crds 
 keywords:
   - kserve

charts/kserve-crd/README.md

@@ -2,12 +2,12 @@
 
 Helm chart for deploying kserve crds
 
-![Version: v0.13.0](https://img.shields.io/badge/Version-v0.13.0-informational?style=flat-square)
+![Version: v0.14.0-rc0](https://img.shields.io/badge/Version-v0.14.0--rc0-informational?style=flat-square)
 
 ## Installing the Chart
 
 To install the chart, run the following:
 
 ```console
-$ helm install kserve-crd oci://ghcr.io/kserve/charts/kserve-crd --version v0.13.0
+$ helm install kserve-crd oci://ghcr.io/kserve/charts/kserve-crd --version v0.14.0-rc0
 ```

charts/kserve-crd/templates/serving.kserve.io_inferenceservices.yaml

@@ -2385,6 +2385,10 @@ spec:
                     type: object
                   logger:
                     properties:
+                      metadataHeaders:
+                        items:
+                          type: string
+                        type: array
                       mode:
                         enum:
                         - all
@@ -6302,6 +6306,10 @@ spec:
                     type: object
                   logger:
                     properties:
+                      metadataHeaders:
+                        items:
+                          type: string
+                        type: array
                       mode:
                         enum:
                         - all
@@ -14619,6 +14627,10 @@ spec:
                     type: object
                   logger:
                     properties:
+                      metadataHeaders:
+                        items:
+                          type: string
+                        type: array
                       mode:
                         enum:
                         - all

charts/kserve-resources/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: kserve 
-version: v0.13.0
+version: v0.14.0-rc0
 description: Helm chart for deploying kserve resources
 keywords:
   - kserve