diff --git a/rollout/canary.go b/rollout/canary.go
index a003b092a3..ed27bd0a79 100644
--- a/rollout/canary.go
+++ b/rollout/canary.go
@@ -184,7 +184,7 @@ func (c *rolloutContext) scaleDownOldReplicaSetsForCanary(oldRSs []*appsv1.Repli
 
 	annotationedRSs := int32(0)
 	for _, targetRS := range oldRSs {
-		if c.isReplicaSetReferenced(targetRS) {
+		if c.rollout.Spec.Strategy.Canary.TrafficRouting != nil && c.isReplicaSetReferenced(targetRS) {
 			// We might get here if user interrupted an an update in order to move back to stable.
 			c.log.Infof("Skip scale down of older RS '%s': still referenced", targetRS.Name)
 			continue
diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go
index bc5e60b6c3..d42abb6a79 100644
--- a/test/e2e/canary_test.go
+++ b/test/e2e/canary_test.go
@@ -565,6 +565,23 @@ func (s *CanarySuite) TestCanaryScaleDownOnAbortNoTrafficRouting() {
 		ExpectRevisionPodCount("2", 0)
 }
 
+func (s *CanarySuite) TestCanaryWithPausedRollout() {
+	(s.Given().
+		HealthyRollout(`@functional/rollout-canary-with-pause.yaml`).
+		When().
+		ApplyManifests().
+		MarkPodsReady("1", 3). // mark all 3 pods ready
+		WaitForRolloutStatus("Healthy").
+		UpdateSpec(). // update to revision 2
+		WaitForRolloutStatus("Paused").
+		UpdateSpec(). // update to revision 3
+		WaitForRolloutStatus("Paused").
+		Then().
+		ExpectRevisionPodCount("1", 3).
+		ExpectRevisionPodCount("2", 0).
+		ExpectRevisionPodCount("3", 1))
+}
+
 func (s *CanarySuite) TestCanaryUnScaleDownOnAbort() {
 	s.Given().
 		HealthyRollout(`@functional/canary-unscaledownonabort.yaml`).
diff --git a/test/e2e/functional/rollout-canary-with-pause.yaml b/test/e2e/functional/rollout-canary-with-pause.yaml
new file mode 100644
index 0000000000..b92ec9e1c1
--- /dev/null
+++ b/test/e2e/functional/rollout-canary-with-pause.yaml
@@ -0,0 +1,77 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: rollout-canary-with-pause-root
+spec:
+  type: NodePort
+  ports:
+    - port: 80
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: rollout-canary-with-pause
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rollout-canary-with-pause-canary
+spec:
+  type: NodePort
+  ports:
+    - port: 80
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: rollout-canary-with-pause
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rollout-canary-with-pause-stable
+spec:
+  type: NodePort
+  ports:
+    - port: 80
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: rollout-canary-with-pause
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  name: rollout-canary-with-pause
+spec:
+  replicas: 3
+  revisionHistoryLimit: 3
+  progressDeadlineSeconds: 5
+  selector:
+    matchLabels:
+      app: rollout-canary-with-pause
+  template:
+    metadata:
+      labels:
+        app: rollout-canary-with-pause
+    spec:
+      containers:
+        - name: rollouts-demo
+          image: nginx:1.19-alpine
+          ports:
+            - containerPort: 80
+          readinessProbe:
+            initialDelaySeconds: 10
+            httpGet:
+              path: /
+              port: 80
+            periodSeconds: 30
+  strategy:
+    canary:
+      canaryService: rollout-canary-with-pause-canary
+      stableService: rollout-canary-with-pause-stable
+      steps:
+        - setWeight: 20
+        - pause: {}
+        
\ No newline at end of file