From 8f386604217dc3718b0fb1ea9f11c2ae0b39e545 Mon Sep 17 00:00:00 2001 From: khhirani Date: Tue, 23 Feb 2021 03:12:42 -0800 Subject: [PATCH 01/13] fix: Cllear ProgressDeadlineExceeded Condition in paused BlueGreen Rollout Signed-off-by: khhirani --- rollout/sync.go | 4 ---- utils/conditions/conditions.go | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/rollout/sync.go b/rollout/sync.go index 3aec9c4676..f3ff80158f 100644 --- a/rollout/sync.go +++ b/rollout/sync.go @@ -484,10 +484,6 @@ func (c *rolloutContext) cleanupRollouts(oldRSs []*appsv1.ReplicaSet) error { // that were paused for longer than progressDeadlineSeconds. func (c *rolloutContext) checkPausedConditions() error { cond := conditions.GetRolloutCondition(c.rollout.Status, v1alpha1.RolloutProgressing) - if cond != nil && cond.Reason == conditions.TimedOutReason { - // If we have reported lack of progress, do not overwrite it with a paused condition. - return nil - } pausedCondExists := cond != nil && cond.Reason == conditions.PausedRolloutReason isPaused := len(c.rollout.Status.PauseConditions) > 0 || c.rollout.Spec.Paused diff --git a/utils/conditions/conditions.go b/utils/conditions/conditions.go index a9338d20f7..871ab9f866 100644 --- a/utils/conditions/conditions.go +++ b/utils/conditions/conditions.go @@ -178,8 +178,8 @@ func RolloutProgressing(rollout *v1alpha1.Rollout, newStatus *v1alpha1.RolloutSt strategySpecificProgress := false if rollout.Spec.Strategy.BlueGreen != nil { - activeSelectorChange := rollout.Status.BlueGreen.ActiveSelector != newStatus.BlueGreen.ActiveSelector - previewSelectorChange := rollout.Status.BlueGreen.PreviewSelector != newStatus.BlueGreen.PreviewSelector + activeSelectorChange := oldStatus.BlueGreen.ActiveSelector != newStatus.BlueGreen.ActiveSelector + previewSelectorChange := oldStatus.BlueGreen.PreviewSelector != newStatus.BlueGreen.PreviewSelector strategySpecificProgress = activeSelectorChange || previewSelectorChange } From 616c52408aec418c46b5e1f9cc7950c448e81e03 Mon Sep 17 00:00:00 2001 From: khhirani Date: Tue, 23 Feb 2021 19:10:28 -0800 Subject: [PATCH 02/13] Create e2e tests Signed-off-by: khhirani --- .../rollouts/v1alpha1/openapi_generated.go | 18 ++- test/e2e/bluegreen_test.go | 118 ++++++++++++++++++ test/e2e/canary_test.go | 50 ++++++++ 3 files changed, 183 insertions(+), 3 deletions(-) diff --git a/pkg/apis/rollouts/v1alpha1/openapi_generated.go b/pkg/apis/rollouts/v1alpha1/openapi_generated.go index 5cc5a0c258..8a50cc550e 100644 --- a/pkg/apis/rollouts/v1alpha1/openapi_generated.go +++ b/pkg/apis/rollouts/v1alpha1/openapi_generated.go @@ -767,12 +767,24 @@ func schema_pkg_apis_rollouts_v1alpha1_BlueGreenStrategy(ref common.ReferenceCal Ref: ref("github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutAnalysis"), }, }, + "previewMetadata": { + SchemaProps: spec.SchemaProps{ + Description: "PreviewMetadata specify labels and annotations which will be attached to the preview pods for the duration which they act as a preview pod, and will be removed after", + Ref: ref("github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PodTemplateMetadata"), + }, + }, + "activeMetadata": { + SchemaProps: spec.SchemaProps{ + Description: "ActiveMetadata specify labels and annotations which will be attached to the active pods for the duration which they act as a active pod, and will be removed after", + Ref: ref("github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PodTemplateMetadata"), + }, + }, }, Required: []string{"activeService"}, }, }, Dependencies: []string{ - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.AntiAffinity", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutAnalysis", "k8s.io/apimachinery/pkg/util/intstr.IntOrString"}, + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.AntiAffinity", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PodTemplateMetadata", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutAnalysis", "k8s.io/apimachinery/pkg/util/intstr.IntOrString"}, } } @@ -912,13 +924,13 @@ func schema_pkg_apis_rollouts_v1alpha1_CanaryStrategy(ref common.ReferenceCallba }, "maxUnavailable": { SchemaProps: spec.SchemaProps{ - Description: "MaxUnavailable The maximum number of pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of total pods at the start of update (ex: 10%). Absolute number is calculated from percentage by rounding down. This can not be 0 if MaxSurge is 0. By default, a fixed value of 1 is used. Example: when this is set to 30%, the old RC can be scaled down by 30% immediately when the rolling update starts. Once new pods are ready, old RC can be scaled down further, followed by scaling up the new RC, ensuring that at least 70% of original number of pods are available at all times during the update.", + Description: "MaxUnavailable The maximum number of pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of total pods at the start of update (ex: 10%). Absolute number is calculated from percentage by rounding down. This can not be 0 if MaxSurge is 0. By default, a fixed value of 25% is used. Example: when this is set to 30%, the old RC can be scaled down by 30% immediately when the rolling update starts. Once new pods are ready, old RC can be scaled down further, followed by scaling up the new RC, ensuring that at least 70% of original number of pods are available at all times during the update.", Ref: ref("k8s.io/apimachinery/pkg/util/intstr.IntOrString"), }, }, "maxSurge": { SchemaProps: spec.SchemaProps{ - Description: "MaxSurge The maximum number of pods that can be scheduled above the original number of pods. Value can be an absolute number (ex: 5) or a percentage of total pods at the start of the update (ex: 10%). This can not be 0 if MaxUnavailable is 0. Absolute number is calculated from percentage by rounding up. By default, a value of 1 is used. Example: when this is set to 30%, the new RC can be scaled up by 30% immediately when the rolling update starts. Once old pods have been killed, new RC can be scaled up further, ensuring that total number of pods running at any time during the update is at most 130% of original pods.", + Description: "MaxSurge The maximum number of pods that can be scheduled above the original number of pods. Value can be an absolute number (ex: 5) or a percentage of total pods at the start of the update (ex: 10%). This can not be 0 if MaxUnavailable is 0. Absolute number is calculated from percentage by rounding up. By default, a value of 25% is used. Example: when this is set to 30%, the new RC can be scaled up by 30% immediately when the rolling update starts. Once old pods have been killed, new RC can be scaled up further, ensuring that total number of pods running at any time during the update is at most 130% of original pods.", Ref: ref("k8s.io/apimachinery/pkg/util/intstr.IntOrString"), }, }, diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index bf4ed23743..71f649dc1b 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -158,3 +158,121 @@ spec: Then(). ExpectRevisionPods("revision 2 has active metadata2", "2", podsHaveActiveMetadata2) } + +func (s *BlueGreenSuite) TestBlueGreenProgressDeadlineExceededWithPause() { + s.Given(). + RolloutObjects(` +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: rollout-bluegreen +spec: + replicas: 2 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 10 # note this is less than initialDelaySeconds + selector: + matchLabels: + app: rollout-bluegreen + template: + metadata: + labels: + app: rollout-bluegreen + spec: + containers: + - name: rollouts-demo + image: argoproj/rollouts-demo:blue + imagePullPolicy: Always + ports: + - containerPort: 8080 + readinessProbe: + initialDelaySeconds: 20 + httpGet: + path: /color + port: 8080 + periodSeconds: 30 + strategy: + blueGreen: + autoPromotionEnabled: false + activeService: rollout-bluegreen-active +--- +kind: Service +apiVersion: v1 +metadata: + name: rollout-bluegreen-active +spec: + selector: + app: rollout-bluegreen + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +`). + When(). + ApplyManifests(). + WaitForRolloutReplicas(2). + WaitForRolloutStatus("Healthy"). + UpdateSpec(). + WaitForRolloutStatus("Paused"). + PromoteRollout(). + WaitForRolloutStatus("Healthy"). + Then(). + ExpectActiveRevision("2") +} + +func (s *BlueGreenSuite) TestBlueGreenProgressDeadlineExceededWithoutPause() { + s.Given(). + RolloutObjects(` +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: rollout-bluegreen +spec: + replicas: 2 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 10 # note this is less than initialDelaySeconds + selector: + matchLabels: + app: rollout-bluegreen + template: + metadata: + labels: + app: rollout-bluegreen + spec: + containers: + - name: rollouts-demo + image: argoproj/rollouts-demo:blue + imagePullPolicy: Always + ports: + - containerPort: 8080 + readinessProbe: + initialDelaySeconds: 20 + httpGet: + path: /color + port: 8080 + periodSeconds: 30 + strategy: + blueGreen: + autoPromotionEnabled: true + activeService: rollout-bluegreen-active +--- +kind: Service +apiVersion: v1 +metadata: + name: rollout-bluegreen-active +spec: + selector: + app: rollout-bluegreen + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +`). + When(). + ApplyManifests(). + WaitForRolloutReplicas(2). + WaitForRolloutStatus("Healthy"). + UpdateSpec(). + WaitForRolloutStatus("Healthy"). + Then(). + ExpectActiveRevision("2") +} \ No newline at end of file diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go index 7f3da3ae3a..2beb054728 100644 --- a/test/e2e/canary_test.go +++ b/test/e2e/canary_test.go @@ -355,3 +355,53 @@ spec: Then(). ExpectRevisionPods("revision 2 has stable metadata2", "2", podsHaveStableMetadata2) } + +func (s *CanarySuite) TestCanaryProgressDeadlineExceededWithPause() { + s.Given(). + RolloutObjects(` +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: rollout-canary +spec: + replicas: 3 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 10 + selector: + matchLabels: + app: rollout-canary + template: + metadata: + labels: + app: rollout-canary + spec: + containers: + - name: rollouts-demo + image: argoproj/rollouts-demo:blue + imagePullPolicy: Always + ports: + - containerPort: 8080 + readinessProbe: + initialDelaySeconds: 20 + httpGet: + path: /color + port: 8080 + periodSeconds: 5 + strategy: + canary: + steps: + - setWeight: 20 + - pause: {} +`). + When(). + ApplyManifests(). + WaitForRolloutStatus("Healthy"). + WaitForRolloutReplicas(3). + UpdateSpec(). + WaitForRolloutStatus("Paused"). + Then(). + ExpectCanaryStablePodCount(1, 3). + When(). + PromoteRollout(). + WaitForRolloutStatus("Healthy") +} From 3722ea140f734c707f226f1830d09a97ed0699e3 Mon Sep 17 00:00:00 2001 From: khhirani Date: Wed, 24 Feb 2021 23:28:29 -0800 Subject: [PATCH 03/13] Modified e2e tests. Set progressDeadlineExceeded to 1 to guarantee that it will always be exceeded Signed-off-by: khhirani --- test/e2e/bluegreen_test.go | 96 +++++++++++++++++--------------------- test/e2e/canary_test.go | 34 ++++++-------- 2 files changed, 58 insertions(+), 72 deletions(-) diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index 71f649dc1b..3357d3fdce 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -162,6 +162,18 @@ spec: func (s *BlueGreenSuite) TestBlueGreenProgressDeadlineExceededWithPause() { s.Given(). RolloutObjects(` +kind: Service +apiVersion: v1 +metadata: + name: rollout-bluegreen-active +spec: + selector: + app: rollout-bluegreen-progress-deadline-exceeded-with-pause + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +--- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: @@ -169,43 +181,26 @@ metadata: spec: replicas: 2 revisionHistoryLimit: 2 - progressDeadlineSeconds: 10 # note this is less than initialDelaySeconds + progressDeadlineSeconds: 1 selector: matchLabels: - app: rollout-bluegreen + app: rollout-bluegreen-progress-deadline-exceeded-with-pause template: metadata: labels: - app: rollout-bluegreen + app: rollout-bluegreen-progress-deadline-exceeded-with-pause spec: containers: - - name: rollouts-demo - image: argoproj/rollouts-demo:blue - imagePullPolicy: Always - ports: - - containerPort: 8080 - readinessProbe: - initialDelaySeconds: 20 - httpGet: - path: /color - port: 8080 - periodSeconds: 30 + - name: rollout-bluegreen-progress-deadline-exceeded-with-pause + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 1m strategy: blueGreen: autoPromotionEnabled: false activeService: rollout-bluegreen-active ---- -kind: Service -apiVersion: v1 -metadata: - name: rollout-bluegreen-active -spec: - selector: - app: rollout-bluegreen - ports: - - protocol: TCP - port: 80 - targetPort: 8080 `). When(). ApplyManifests(). @@ -222,50 +217,45 @@ spec: func (s *BlueGreenSuite) TestBlueGreenProgressDeadlineExceededWithoutPause() { s.Given(). RolloutObjects(` +kind: Service +apiVersion: v1 +metadata: + name: rollout-bluegreen-active +spec: + selector: + app: rollout-bluegreen-progress-deadline-exceeded-without-pause + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +--- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: - name: rollout-bluegreen + name: rollout-bluegreen-progress-deadline-exceeded-without-pause spec: replicas: 2 revisionHistoryLimit: 2 - progressDeadlineSeconds: 10 # note this is less than initialDelaySeconds + progressDeadlineSeconds: 1 selector: matchLabels: - app: rollout-bluegreen + app: rollout-bluegreen-progress-deadline-exceeded-without-pause template: metadata: labels: - app: rollout-bluegreen + app: rollout-bluegreen-progress-deadline-exceeded-without-pause spec: containers: - - name: rollouts-demo - image: argoproj/rollouts-demo:blue - imagePullPolicy: Always - ports: - - containerPort: 8080 - readinessProbe: - initialDelaySeconds: 20 - httpGet: - path: /color - port: 8080 - periodSeconds: 30 + - name: rollout-bluegreen-progress-deadline-exceeded-without-pause + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 1m strategy: blueGreen: autoPromotionEnabled: true activeService: rollout-bluegreen-active ---- -kind: Service -apiVersion: v1 -metadata: - name: rollout-bluegreen-active -spec: - selector: - app: rollout-bluegreen - ports: - - protocol: TCP - port: 80 - targetPort: 8080 `). When(). ApplyManifests(). diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go index 2beb054728..2ab700c9df 100644 --- a/test/e2e/canary_test.go +++ b/test/e2e/canary_test.go @@ -362,45 +362,41 @@ func (s *CanarySuite) TestCanaryProgressDeadlineExceededWithPause() { apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: - name: rollout-canary + name: rollout-canary-progress-deadline-exceeded-with-pause spec: - replicas: 3 + replicas: 2 revisionHistoryLimit: 2 - progressDeadlineSeconds: 10 + progressDeadlineSeconds: 1 selector: matchLabels: - app: rollout-canary + app: rollout-canary-progress-deadline-exceeded-with-pause template: metadata: labels: - app: rollout-canary + app: rollout-canary-progress-deadline-exceeded-with-pause spec: containers: - - name: rollouts-demo - image: argoproj/rollouts-demo:blue - imagePullPolicy: Always - ports: - - containerPort: 8080 - readinessProbe: - initialDelaySeconds: 20 - httpGet: - path: /color - port: 8080 - periodSeconds: 5 + - name: rollout-canary-progress-deadline-exceeded-with-pause + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 1m strategy: canary: steps: - - setWeight: 20 + - setWeight: 50 - pause: {} `). When(). ApplyManifests(). WaitForRolloutStatus("Healthy"). - WaitForRolloutReplicas(3). + WaitForRolloutReplicas(2). UpdateSpec(). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Paused"). Then(). - ExpectCanaryStablePodCount(1, 3). + ExpectCanaryStablePodCount(1, 1). When(). PromoteRollout(). WaitForRolloutStatus("Healthy") From 6edb43df76960ef206111da6ad0f5a93ec28ef7f Mon Sep 17 00:00:00 2001 From: khhirani Date: Wed, 24 Feb 2021 23:35:13 -0800 Subject: [PATCH 04/13] Modify tests Signed-off-by: khhirani --- test/e2e/bluegreen_test.go | 12 ++++++++---- test/e2e/canary_test.go | 2 ++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index 3357d3fdce..3e1ad52497 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -179,7 +179,7 @@ kind: Rollout metadata: name: rollout-bluegreen spec: - replicas: 2 + replicas: 1 revisionHistoryLimit: 2 progressDeadlineSeconds: 1 selector: @@ -204,9 +204,11 @@ spec: `). When(). ApplyManifests(). - WaitForRolloutReplicas(2). + WaitForRolloutReplicas(1). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy"). UpdateSpec(). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Paused"). PromoteRollout(). WaitForRolloutStatus("Healthy"). @@ -234,7 +236,7 @@ kind: Rollout metadata: name: rollout-bluegreen-progress-deadline-exceeded-without-pause spec: - replicas: 2 + replicas: 1 revisionHistoryLimit: 2 progressDeadlineSeconds: 1 selector: @@ -259,9 +261,11 @@ spec: `). When(). ApplyManifests(). - WaitForRolloutReplicas(2). + WaitForRolloutReplicas(1). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy"). UpdateSpec(). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy"). Then(). ExpectActiveRevision("2") diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go index 2ab700c9df..9a61e4445f 100644 --- a/test/e2e/canary_test.go +++ b/test/e2e/canary_test.go @@ -390,6 +390,7 @@ spec: `). When(). ApplyManifests(). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy"). WaitForRolloutReplicas(2). UpdateSpec(). @@ -399,5 +400,6 @@ spec: ExpectCanaryStablePodCount(1, 1). When(). PromoteRollout(). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy") } From 3166147bcd8b3e8f1b8993cb93aaff32984d4616 Mon Sep 17 00:00:00 2001 From: khhirani Date: Wed, 24 Feb 2021 23:37:39 -0800 Subject: [PATCH 05/13] change RO name in test Signed-off-by: khhirani --- test/e2e/bluegreen_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index 3e1ad52497..54cef567e6 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -177,7 +177,7 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: - name: rollout-bluegreen + name: rollout-bluegreen-progress-deadline-exceeded-with-pause spec: replicas: 1 revisionHistoryLimit: 2 From 415bd4c84e7bfb70903867d4b6197ba86f5e34ac Mon Sep 17 00:00:00 2001 From: khhirani Date: Wed, 24 Feb 2021 23:38:22 -0800 Subject: [PATCH 06/13] add newline Signed-off-by: khhirani --- test/e2e/bluegreen_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index 54cef567e6..b0f28bdb19 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -269,4 +269,4 @@ spec: WaitForRolloutStatus("Healthy"). Then(). ExpectActiveRevision("2") -} \ No newline at end of file +} From 02588ec5a8ab27898dc367fb0acd9314c214ee5a Mon Sep 17 00:00:00 2001 From: khhirani Date: Tue, 23 Feb 2021 03:12:42 -0800 Subject: [PATCH 07/13] fix: Cllear ProgressDeadlineExceeded Condition in paused BlueGreen Rollout Signed-off-by: khhirani --- rollout/sync.go | 4 ---- utils/conditions/conditions.go | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/rollout/sync.go b/rollout/sync.go index 9c62f58fb4..3b3d20e522 100644 --- a/rollout/sync.go +++ b/rollout/sync.go @@ -484,10 +484,6 @@ func (c *rolloutContext) cleanupRollouts(oldRSs []*appsv1.ReplicaSet) error { // that were paused for longer than progressDeadlineSeconds. func (c *rolloutContext) checkPausedConditions() error { cond := conditions.GetRolloutCondition(c.rollout.Status, v1alpha1.RolloutProgressing) - if cond != nil && cond.Reason == conditions.TimedOutReason { - // If we have reported lack of progress, do not overwrite it with a paused condition. - return nil - } pausedCondExists := cond != nil && cond.Reason == conditions.PausedRolloutReason isPaused := len(c.rollout.Status.PauseConditions) > 0 || c.rollout.Spec.Paused diff --git a/utils/conditions/conditions.go b/utils/conditions/conditions.go index a9338d20f7..871ab9f866 100644 --- a/utils/conditions/conditions.go +++ b/utils/conditions/conditions.go @@ -178,8 +178,8 @@ func RolloutProgressing(rollout *v1alpha1.Rollout, newStatus *v1alpha1.RolloutSt strategySpecificProgress := false if rollout.Spec.Strategy.BlueGreen != nil { - activeSelectorChange := rollout.Status.BlueGreen.ActiveSelector != newStatus.BlueGreen.ActiveSelector - previewSelectorChange := rollout.Status.BlueGreen.PreviewSelector != newStatus.BlueGreen.PreviewSelector + activeSelectorChange := oldStatus.BlueGreen.ActiveSelector != newStatus.BlueGreen.ActiveSelector + previewSelectorChange := oldStatus.BlueGreen.PreviewSelector != newStatus.BlueGreen.PreviewSelector strategySpecificProgress = activeSelectorChange || previewSelectorChange } From bc33238f93e3b77870d49b2dc67d32f37fbb25ac Mon Sep 17 00:00:00 2001 From: khhirani Date: Tue, 23 Feb 2021 19:10:28 -0800 Subject: [PATCH 08/13] Create e2e tests Signed-off-by: khhirani --- test/e2e/bluegreen_test.go | 118 +++++++++++++++++++++++++++++++++++++ test/e2e/canary_test.go | 50 ++++++++++++++++ 2 files changed, 168 insertions(+) diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index bf4ed23743..71f649dc1b 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -158,3 +158,121 @@ spec: Then(). ExpectRevisionPods("revision 2 has active metadata2", "2", podsHaveActiveMetadata2) } + +func (s *BlueGreenSuite) TestBlueGreenProgressDeadlineExceededWithPause() { + s.Given(). + RolloutObjects(` +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: rollout-bluegreen +spec: + replicas: 2 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 10 # note this is less than initialDelaySeconds + selector: + matchLabels: + app: rollout-bluegreen + template: + metadata: + labels: + app: rollout-bluegreen + spec: + containers: + - name: rollouts-demo + image: argoproj/rollouts-demo:blue + imagePullPolicy: Always + ports: + - containerPort: 8080 + readinessProbe: + initialDelaySeconds: 20 + httpGet: + path: /color + port: 8080 + periodSeconds: 30 + strategy: + blueGreen: + autoPromotionEnabled: false + activeService: rollout-bluegreen-active +--- +kind: Service +apiVersion: v1 +metadata: + name: rollout-bluegreen-active +spec: + selector: + app: rollout-bluegreen + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +`). + When(). + ApplyManifests(). + WaitForRolloutReplicas(2). + WaitForRolloutStatus("Healthy"). + UpdateSpec(). + WaitForRolloutStatus("Paused"). + PromoteRollout(). + WaitForRolloutStatus("Healthy"). + Then(). + ExpectActiveRevision("2") +} + +func (s *BlueGreenSuite) TestBlueGreenProgressDeadlineExceededWithoutPause() { + s.Given(). + RolloutObjects(` +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: rollout-bluegreen +spec: + replicas: 2 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 10 # note this is less than initialDelaySeconds + selector: + matchLabels: + app: rollout-bluegreen + template: + metadata: + labels: + app: rollout-bluegreen + spec: + containers: + - name: rollouts-demo + image: argoproj/rollouts-demo:blue + imagePullPolicy: Always + ports: + - containerPort: 8080 + readinessProbe: + initialDelaySeconds: 20 + httpGet: + path: /color + port: 8080 + periodSeconds: 30 + strategy: + blueGreen: + autoPromotionEnabled: true + activeService: rollout-bluegreen-active +--- +kind: Service +apiVersion: v1 +metadata: + name: rollout-bluegreen-active +spec: + selector: + app: rollout-bluegreen + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +`). + When(). + ApplyManifests(). + WaitForRolloutReplicas(2). + WaitForRolloutStatus("Healthy"). + UpdateSpec(). + WaitForRolloutStatus("Healthy"). + Then(). + ExpectActiveRevision("2") +} \ No newline at end of file diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go index 7f3da3ae3a..2beb054728 100644 --- a/test/e2e/canary_test.go +++ b/test/e2e/canary_test.go @@ -355,3 +355,53 @@ spec: Then(). ExpectRevisionPods("revision 2 has stable metadata2", "2", podsHaveStableMetadata2) } + +func (s *CanarySuite) TestCanaryProgressDeadlineExceededWithPause() { + s.Given(). + RolloutObjects(` +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: rollout-canary +spec: + replicas: 3 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 10 + selector: + matchLabels: + app: rollout-canary + template: + metadata: + labels: + app: rollout-canary + spec: + containers: + - name: rollouts-demo + image: argoproj/rollouts-demo:blue + imagePullPolicy: Always + ports: + - containerPort: 8080 + readinessProbe: + initialDelaySeconds: 20 + httpGet: + path: /color + port: 8080 + periodSeconds: 5 + strategy: + canary: + steps: + - setWeight: 20 + - pause: {} +`). + When(). + ApplyManifests(). + WaitForRolloutStatus("Healthy"). + WaitForRolloutReplicas(3). + UpdateSpec(). + WaitForRolloutStatus("Paused"). + Then(). + ExpectCanaryStablePodCount(1, 3). + When(). + PromoteRollout(). + WaitForRolloutStatus("Healthy") +} From 561dbaecb55f5e00a9bda031111c0e1ad22a10ab Mon Sep 17 00:00:00 2001 From: khhirani Date: Wed, 24 Feb 2021 23:28:29 -0800 Subject: [PATCH 09/13] Modified e2e tests. Set progressDeadlineExceeded to 1 to guarantee that it will always be exceeded Signed-off-by: khhirani --- test/e2e/bluegreen_test.go | 96 +++++++++++++++++--------------------- test/e2e/canary_test.go | 34 ++++++-------- 2 files changed, 58 insertions(+), 72 deletions(-) diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index 71f649dc1b..3357d3fdce 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -162,6 +162,18 @@ spec: func (s *BlueGreenSuite) TestBlueGreenProgressDeadlineExceededWithPause() { s.Given(). RolloutObjects(` +kind: Service +apiVersion: v1 +metadata: + name: rollout-bluegreen-active +spec: + selector: + app: rollout-bluegreen-progress-deadline-exceeded-with-pause + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +--- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: @@ -169,43 +181,26 @@ metadata: spec: replicas: 2 revisionHistoryLimit: 2 - progressDeadlineSeconds: 10 # note this is less than initialDelaySeconds + progressDeadlineSeconds: 1 selector: matchLabels: - app: rollout-bluegreen + app: rollout-bluegreen-progress-deadline-exceeded-with-pause template: metadata: labels: - app: rollout-bluegreen + app: rollout-bluegreen-progress-deadline-exceeded-with-pause spec: containers: - - name: rollouts-demo - image: argoproj/rollouts-demo:blue - imagePullPolicy: Always - ports: - - containerPort: 8080 - readinessProbe: - initialDelaySeconds: 20 - httpGet: - path: /color - port: 8080 - periodSeconds: 30 + - name: rollout-bluegreen-progress-deadline-exceeded-with-pause + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 1m strategy: blueGreen: autoPromotionEnabled: false activeService: rollout-bluegreen-active ---- -kind: Service -apiVersion: v1 -metadata: - name: rollout-bluegreen-active -spec: - selector: - app: rollout-bluegreen - ports: - - protocol: TCP - port: 80 - targetPort: 8080 `). When(). ApplyManifests(). @@ -222,50 +217,45 @@ spec: func (s *BlueGreenSuite) TestBlueGreenProgressDeadlineExceededWithoutPause() { s.Given(). RolloutObjects(` +kind: Service +apiVersion: v1 +metadata: + name: rollout-bluegreen-active +spec: + selector: + app: rollout-bluegreen-progress-deadline-exceeded-without-pause + ports: + - protocol: TCP + port: 80 + targetPort: 8080 +--- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: - name: rollout-bluegreen + name: rollout-bluegreen-progress-deadline-exceeded-without-pause spec: replicas: 2 revisionHistoryLimit: 2 - progressDeadlineSeconds: 10 # note this is less than initialDelaySeconds + progressDeadlineSeconds: 1 selector: matchLabels: - app: rollout-bluegreen + app: rollout-bluegreen-progress-deadline-exceeded-without-pause template: metadata: labels: - app: rollout-bluegreen + app: rollout-bluegreen-progress-deadline-exceeded-without-pause spec: containers: - - name: rollouts-demo - image: argoproj/rollouts-demo:blue - imagePullPolicy: Always - ports: - - containerPort: 8080 - readinessProbe: - initialDelaySeconds: 20 - httpGet: - path: /color - port: 8080 - periodSeconds: 30 + - name: rollout-bluegreen-progress-deadline-exceeded-without-pause + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 1m strategy: blueGreen: autoPromotionEnabled: true activeService: rollout-bluegreen-active ---- -kind: Service -apiVersion: v1 -metadata: - name: rollout-bluegreen-active -spec: - selector: - app: rollout-bluegreen - ports: - - protocol: TCP - port: 80 - targetPort: 8080 `). When(). ApplyManifests(). diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go index 2beb054728..2ab700c9df 100644 --- a/test/e2e/canary_test.go +++ b/test/e2e/canary_test.go @@ -362,45 +362,41 @@ func (s *CanarySuite) TestCanaryProgressDeadlineExceededWithPause() { apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: - name: rollout-canary + name: rollout-canary-progress-deadline-exceeded-with-pause spec: - replicas: 3 + replicas: 2 revisionHistoryLimit: 2 - progressDeadlineSeconds: 10 + progressDeadlineSeconds: 1 selector: matchLabels: - app: rollout-canary + app: rollout-canary-progress-deadline-exceeded-with-pause template: metadata: labels: - app: rollout-canary + app: rollout-canary-progress-deadline-exceeded-with-pause spec: containers: - - name: rollouts-demo - image: argoproj/rollouts-demo:blue - imagePullPolicy: Always - ports: - - containerPort: 8080 - readinessProbe: - initialDelaySeconds: 20 - httpGet: - path: /color - port: 8080 - periodSeconds: 5 + - name: rollout-canary-progress-deadline-exceeded-with-pause + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 1m strategy: canary: steps: - - setWeight: 20 + - setWeight: 50 - pause: {} `). When(). ApplyManifests(). WaitForRolloutStatus("Healthy"). - WaitForRolloutReplicas(3). + WaitForRolloutReplicas(2). UpdateSpec(). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Paused"). Then(). - ExpectCanaryStablePodCount(1, 3). + ExpectCanaryStablePodCount(1, 1). When(). PromoteRollout(). WaitForRolloutStatus("Healthy") From f537589a9c50db67bf2bfffa34a3b73165a31091 Mon Sep 17 00:00:00 2001 From: khhirani Date: Wed, 24 Feb 2021 23:35:13 -0800 Subject: [PATCH 10/13] Modify tests Signed-off-by: khhirani --- test/e2e/bluegreen_test.go | 12 ++++++++---- test/e2e/canary_test.go | 2 ++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index 3357d3fdce..3e1ad52497 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -179,7 +179,7 @@ kind: Rollout metadata: name: rollout-bluegreen spec: - replicas: 2 + replicas: 1 revisionHistoryLimit: 2 progressDeadlineSeconds: 1 selector: @@ -204,9 +204,11 @@ spec: `). When(). ApplyManifests(). - WaitForRolloutReplicas(2). + WaitForRolloutReplicas(1). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy"). UpdateSpec(). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Paused"). PromoteRollout(). WaitForRolloutStatus("Healthy"). @@ -234,7 +236,7 @@ kind: Rollout metadata: name: rollout-bluegreen-progress-deadline-exceeded-without-pause spec: - replicas: 2 + replicas: 1 revisionHistoryLimit: 2 progressDeadlineSeconds: 1 selector: @@ -259,9 +261,11 @@ spec: `). When(). ApplyManifests(). - WaitForRolloutReplicas(2). + WaitForRolloutReplicas(1). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy"). UpdateSpec(). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy"). Then(). ExpectActiveRevision("2") diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go index 2ab700c9df..9a61e4445f 100644 --- a/test/e2e/canary_test.go +++ b/test/e2e/canary_test.go @@ -390,6 +390,7 @@ spec: `). When(). ApplyManifests(). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy"). WaitForRolloutReplicas(2). UpdateSpec(). @@ -399,5 +400,6 @@ spec: ExpectCanaryStablePodCount(1, 1). When(). PromoteRollout(). + WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy") } From 6f1681c5dc9e6f70bad83f6a057aaf78a21b8fcf Mon Sep 17 00:00:00 2001 From: khhirani Date: Wed, 24 Feb 2021 23:37:39 -0800 Subject: [PATCH 11/13] change RO name in test Signed-off-by: khhirani --- test/e2e/bluegreen_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index 3e1ad52497..54cef567e6 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -177,7 +177,7 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: - name: rollout-bluegreen + name: rollout-bluegreen-progress-deadline-exceeded-with-pause spec: replicas: 1 revisionHistoryLimit: 2 From bb9ad58821304734b4abe379e566b63575e10461 Mon Sep 17 00:00:00 2001 From: khhirani Date: Wed, 24 Feb 2021 23:38:22 -0800 Subject: [PATCH 12/13] add newline Signed-off-by: khhirani --- test/e2e/bluegreen_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index 54cef567e6..b0f28bdb19 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -269,4 +269,4 @@ spec: WaitForRolloutStatus("Healthy"). Then(). ExpectActiveRevision("2") -} \ No newline at end of file +} From 2dae7472d2ce143ac740f48025801753e74ae307 Mon Sep 17 00:00:00 2001 From: khhirani Date: Fri, 26 Feb 2021 20:23:24 -0800 Subject: [PATCH 13/13] Modify tests Signed-off-by: khhirani --- test/e2e/bluegreen_test.go | 48 ++++++++++++++++++++++---------------- test/e2e/canary_test.go | 30 +++++++++++++----------- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/test/e2e/bluegreen_test.go b/test/e2e/bluegreen_test.go index b0f28bdb19..b6ebadad94 100644 --- a/test/e2e/bluegreen_test.go +++ b/test/e2e/bluegreen_test.go @@ -168,7 +168,7 @@ metadata: name: rollout-bluegreen-active spec: selector: - app: rollout-bluegreen-progress-deadline-exceeded-with-pause + app: rollout-bluegreen-with-pause ports: - protocol: TCP port: 80 @@ -177,26 +177,30 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: - name: rollout-bluegreen-progress-deadline-exceeded-with-pause + name: rollout-bluegreen-with-pause spec: replicas: 1 revisionHistoryLimit: 2 - progressDeadlineSeconds: 1 + progressDeadlineSeconds: 5 # note this is less than initialDelaySeconds selector: matchLabels: - app: rollout-bluegreen-progress-deadline-exceeded-with-pause + app: rollout-bluegreen-with-pause template: metadata: labels: - app: rollout-bluegreen-progress-deadline-exceeded-with-pause + app: rollout-bluegreen-with-pause spec: containers: - - name: rollout-bluegreen-progress-deadline-exceeded-with-pause + - name: rollouts-demo image: nginx:1.19-alpine - resources: - requests: - memory: 16Mi - cpu: 1m + ports: + - containerPort: 80 + readinessProbe: + initialDelaySeconds: 10 + httpGet: + path: / + port: 80 + periodSeconds: 30 strategy: blueGreen: autoPromotionEnabled: false @@ -225,7 +229,7 @@ metadata: name: rollout-bluegreen-active spec: selector: - app: rollout-bluegreen-progress-deadline-exceeded-without-pause + app: rollout-bluegreen-without-pause ports: - protocol: TCP port: 80 @@ -234,26 +238,30 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: - name: rollout-bluegreen-progress-deadline-exceeded-without-pause + name: rollout-bluegreen-without-pause spec: replicas: 1 revisionHistoryLimit: 2 - progressDeadlineSeconds: 1 + progressDeadlineSeconds: 5 # note this is less than initialDelaySeconds selector: matchLabels: - app: rollout-bluegreen-progress-deadline-exceeded-without-pause + app: rollout-bluegreen-without-pause template: metadata: labels: - app: rollout-bluegreen-progress-deadline-exceeded-without-pause + app: rollout-bluegreen-without-pause spec: containers: - - name: rollout-bluegreen-progress-deadline-exceeded-without-pause + - name: rollouts-demo image: nginx:1.19-alpine - resources: - requests: - memory: 16Mi - cpu: 1m + ports: + - containerPort: 80 + readinessProbe: + initialDelaySeconds: 10 + httpGet: + path: / + port: 80 + periodSeconds: 30 strategy: blueGreen: autoPromotionEnabled: true diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go index 9a61e4445f..1181799c4f 100644 --- a/test/e2e/canary_test.go +++ b/test/e2e/canary_test.go @@ -362,42 +362,46 @@ func (s *CanarySuite) TestCanaryProgressDeadlineExceededWithPause() { apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: - name: rollout-canary-progress-deadline-exceeded-with-pause + name: rollout-canary-with-pause spec: - replicas: 2 + replicas: 3 revisionHistoryLimit: 2 - progressDeadlineSeconds: 1 + progressDeadlineSeconds: 5 selector: matchLabels: - app: rollout-canary-progress-deadline-exceeded-with-pause + app: rollout-canary-with-pause template: metadata: labels: - app: rollout-canary-progress-deadline-exceeded-with-pause + app: rollout-canary-with-pause spec: containers: - - name: rollout-canary-progress-deadline-exceeded-with-pause + - name: rollouts-demo image: nginx:1.19-alpine - resources: - requests: - memory: 16Mi - cpu: 1m + ports: + - containerPort: 80 + readinessProbe: + initialDelaySeconds: 10 + httpGet: + path: / + port: 80 + periodSeconds: 30 strategy: canary: steps: - - setWeight: 50 + - setWeight: 20 - pause: {} `). When(). ApplyManifests(). WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Healthy"). - WaitForRolloutReplicas(2). + WaitForRolloutReplicas(3). UpdateSpec(). WaitForRolloutStatus("Degraded"). WaitForRolloutStatus("Paused"). Then(). - ExpectCanaryStablePodCount(1, 1). + ExpectCanaryStablePodCount(1, 3). When(). PromoteRollout(). WaitForRolloutStatus("Degraded").