diff --git a/pkg/controller/syncset/syncset_controller.go b/pkg/controller/syncset/syncset_controller.go
index 51c0e52fca1..d818a4f324d 100644
--- a/pkg/controller/syncset/syncset_controller.go
+++ b/pkg/controller/syncset/syncset_controller.go
@@ -130,17 +130,28 @@ func (r *ReconcileSyncSet) Reconcile(ctx context.Context, request reconcile.Requ
 			return reconcile.Result{}, err
 		}
 	}
+	sk := aggregator.Key{Source: "syncset", ID: request.NamespacedName.String()}
+
+	if !exists || !syncset.GetDeletionTimestamp().IsZero() {
+		log.V(logging.DebugLevel).Info("handling SyncSet delete", "instance", syncset)
 
-	gvks := make([]schema.GroupVersionKind, 0)
-	if exists && syncset.GetDeletionTimestamp().IsZero() {
-		log.V(logging.DebugLevel).Info("handling SyncSet update", "instance", syncset)
+		if err := r.cacheManager.RemoveSource(ctx, sk); err != nil {
+			syncsetTr.TryCancelExpect(syncset)
 
-		for _, entry := range syncset.Spec.GVKs {
-			gvks = append(gvks, schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind})
+			return reconcile.Result{}, fmt.Errorf("synceset-controller: error removing source: %w", err)
 		}
+
+		syncsetTr.CancelExpect(syncset)
+
+		return reconcile.Result{}, nil
+	}
+
+	log.V(logging.DebugLevel).Info("handling SyncSet update", "instance", syncset)
+	gvks := []schema.GroupVersionKind{}
+	for _, entry := range syncset.Spec.GVKs {
+		gvks = append(gvks, schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind})
 	}
 
-	sk := aggregator.Key{Source: "syncset", ID: request.NamespacedName.String()}
 	if err := r.cacheManager.UpsertSource(ctx, sk, gvks); err != nil {
 		syncsetTr.TryCancelExpect(syncset)