diff --git a/installer/helm/chart/volcano/templates/controllers.yaml b/installer/helm/chart/volcano/templates/controllers.yaml
index 5354fa06b4..f4e412c880 100644
--- a/installer/helm/chart/volcano/templates/controllers.yaml
+++ b/installer/helm/chart/volcano/templates/controllers.yaml
@@ -28,7 +28,7 @@ rules:
     verbs: ["create", "list", "watch", "update", "patch"]
   - apiGroups: [""]
     resources: ["pods"]
-    verbs: ["create", "get", "list", "watch", "update", "bind", "delete"]
+    verbs: ["create", "get", "list", "watch", "update", "bind", "delete", "patch"]
   - apiGroups: [""]
     resources: ["pods/finalizers"]
     verbs: ["update", "patch"]
diff --git a/installer/volcano-development-arm64.yaml b/installer/volcano-development-arm64.yaml
index 2d916331ee..cac04ac38b 100644
--- a/installer/volcano-development-arm64.yaml
+++ b/installer/volcano-development-arm64.yaml
@@ -8433,7 +8433,7 @@ rules:
     verbs: ["create", "list", "watch", "update", "patch"]
   - apiGroups: [""]
     resources: ["pods"]
-    verbs: ["create", "get", "list", "watch", "update", "bind", "delete"]
+    verbs: ["create", "get", "list", "watch", "update", "bind", "delete", "patch"]
   - apiGroups: [""]
     resources: ["pods/finalizers"]
     verbs: ["update", "patch"]
diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml
index 94b7a385e6..9acdddf1a5 100644
--- a/installer/volcano-development.yaml
+++ b/installer/volcano-development.yaml
@@ -8433,7 +8433,7 @@ rules:
     verbs: ["create", "list", "watch", "update", "patch"]
   - apiGroups: [""]
     resources: ["pods"]
-    verbs: ["create", "get", "list", "watch", "update", "bind", "delete"]
+    verbs: ["create", "get", "list", "watch", "update", "bind", "delete", "patch"]
   - apiGroups: [""]
     resources: ["pods/finalizers"]
     verbs: ["update", "patch"]
diff --git a/pkg/controllers/podgroup/pg_controller_handler.go b/pkg/controllers/podgroup/pg_controller_handler.go
index a5bac2ee48..931fba7bb7 100644
--- a/pkg/controllers/podgroup/pg_controller_handler.go
+++ b/pkg/controllers/podgroup/pg_controller_handler.go
@@ -18,6 +18,7 @@ package podgroup
 
 import (
 	"context"
+	"encoding/json"
 	"strings"
 
 	v1 "k8s.io/api/core/v1"
@@ -28,6 +29,7 @@ import (
 	quotacore "k8s.io/kubernetes/pkg/quota/v1/evaluator/core"
 	"k8s.io/utils/clock"
 
+	"k8s.io/apimachinery/pkg/types"
 	"volcano.sh/apis/pkg/apis/helpers"
 	scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
 )
@@ -37,6 +39,14 @@ type podRequest struct {
 	podNamespace string
 }
 
+type metadataForMergePatch struct {
+	Metadata annotationForMergePatch `json:"metadata"`
+}
+
+type annotationForMergePatch struct {
+	Annotations map[string]string `json:"annotations"`
+}
+
 func (pg *pgcontroller) addPod(obj interface{}) {
 	pod, ok := obj.(*v1.Pod)
 	if !ok {
@@ -57,20 +67,30 @@ func (pg *pgcontroller) updatePodAnnotations(pod *v1.Pod, pgName string) error {
 		pod.Annotations = make(map[string]string)
 	}
 	if pod.Annotations[scheduling.KubeGroupNameAnnotationKey] == "" {
-		pod.Annotations[scheduling.KubeGroupNameAnnotationKey] = pgName
+		patch := metadataForMergePatch{
+			Metadata: annotationForMergePatch{
+				Annotations: map[string]string{
+					scheduling.KubeGroupNameAnnotationKey: pgName,
+				},
+			},
+		}
+
+		patchBytes, err := json.Marshal(&patch)
+		if err != nil {
+			klog.Errorf("Failed to json.Marshal pod annotation: %v", err)
+			return err
+		}
+
+		if _, err := pg.kubeClient.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}); err != nil {
+			klog.Errorf("Failed to update pod <%s/%s>: %v", pod.Namespace, pod.Name, err)
+			return err
+		}
 	} else {
 		if pod.Annotations[scheduling.KubeGroupNameAnnotationKey] != pgName {
 			klog.Errorf("normal pod %s/%s annotations %s value is not %s, but %s", pod.Namespace, pod.Name,
 				scheduling.KubeGroupNameAnnotationKey, pgName, pod.Annotations[scheduling.KubeGroupNameAnnotationKey])
 		}
-		return nil
-	}
-
-	if _, err := pg.kubeClient.CoreV1().Pods(pod.Namespace).Update(context.TODO(), pod, metav1.UpdateOptions{}); err != nil {
-		klog.Errorf("Failed to update pod <%s/%s>: %v", pod.Namespace, pod.Name, err)
-		return err
 	}
-
 	return nil
 }
 
diff --git a/pkg/controllers/podgroup/pg_controller_test.go b/pkg/controllers/podgroup/pg_controller_test.go
index a79021d9b4..ecb9f5cf17 100644
--- a/pkg/controllers/podgroup/pg_controller_test.go
+++ b/pkg/controllers/podgroup/pg_controller_test.go
@@ -170,7 +170,12 @@ func TestAddPodGroup(t *testing.T) {
 			t.Errorf("Case %s failed, expect %v, got %v", testCase.name, testCase.expectedPodGroup, pg)
 		}
 
-		podAnnotation := pod.Annotations[scheduling.KubeGroupNameAnnotationKey]
+		newpod, err := c.kubeClient.CoreV1().Pods(testCase.pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
+		if err != nil {
+			t.Errorf("Case %s failed when creating pod for %v", testCase.name, err)
+		}
+
+		podAnnotation := newpod.Annotations[scheduling.KubeGroupNameAnnotationKey]
 		if testCase.expectedPodGroup.Name != podAnnotation {
 			t.Errorf("Case %s failed, expect %v, got %v", testCase.name,
 				testCase.expectedPodGroup.Name, podAnnotation)