diff --git a/pkg/controllers/podgroup/pg_controller.go b/pkg/controllers/podgroup/pg_controller.go
index 003e150da7a..6512872237d 100644
--- a/pkg/controllers/podgroup/pg_controller.go
+++ b/pkg/controllers/podgroup/pg_controller.go
@@ -18,7 +18,8 @@ package podgroup
 
 import (
 	"slices"
-
+	
+	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
@@ -108,9 +109,23 @@ func (pg *pgcontroller) Initialize(opt *framework.ControllerOption) error {
 	if utilfeature.DefaultFeatureGate.Enabled(features.WorkLoadSupport) {
 		pg.rsInformer = pg.informerFactory.Apps().V1().ReplicaSets()
 		pg.rsSynced = pg.rsInformer.Informer().HasSynced
-		pg.rsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-			AddFunc:    pg.addReplicaSet,
-			UpdateFunc: pg.updateReplicaSet,
+		pg.rsInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
+			FilterFunc: func(obj interface{}) bool {
+				switch v := obj.(type) {
+				case *appsv1.ReplicaSet:
+					if slices.Contains(pg.schedulerNames, v.Spec.Template.Spec.SchedulerName) {
+						return true
+					}
+
+					return false
+				default:
+					return false
+				}
+			},
+			Handler: cache.ResourceEventHandlerFuncs{
+				AddFunc:    pg.addReplicaSet,
+				UpdateFunc: pg.updateReplicaSet,
+			},
 		})
 	}
 	return nil