From b8bd5b5b57fa6370b3b37025814ad8b5154da6bc Mon Sep 17 00:00:00 2001 From: yunbo Date: Wed, 4 Dec 2024 13:18:52 +0800 Subject: [PATCH] support bluegreen release: webhook update Signed-off-by: yunbo --- .../bluegreenstyle/cloneset/control.go | 29 +- .../bluegreenstyle/deployment/control.go | 120 +++-- .../bluegreenstyle/deployment/control_test.go | 7 +- pkg/controller/rollout/rollout_progressing.go | 16 +- pkg/util/client/delegating_client.go | 7 +- pkg/util/controller_finder.go | 6 +- pkg/webhook/util/writer/fs.go | 2 +- .../mutating/workload_update_handler.go | 47 +- .../mutating/workload_update_handler_test.go | 146 +++++- test/e2e/rollout_v1beta1_test.go | 429 +++++++----------- 10 files changed, 469 insertions(+), 340 deletions(-) diff --git a/pkg/controller/batchrelease/control/bluegreenstyle/cloneset/control.go b/pkg/controller/batchrelease/control/bluegreenstyle/cloneset/control.go index d52d6638..0af7e3d3 100644 --- a/pkg/controller/batchrelease/control/bluegreenstyle/cloneset/control.go +++ b/pkg/controller/batchrelease/control/bluegreenstyle/cloneset/control.go @@ -154,10 +154,6 @@ func (rc *realController) Finalize(release *v1beta1.BatchRelease) error { return errors.NewFatalError(fmt.Errorf("cannot get original setting for cloneset %v: %s from annotation", klog.KObj(rc.object), err.Error())) } patchData := patch.NewClonesetPatch() - // why we need a simple MinReadySeconds-based status machine? (ie. the if-else block) - // It's possible for Finalize to be called multiple times, if error returned is not nil. - // if we do all needed operations in a single code block, like, A->B->C, when C need retry, - // both A and B will be executed as well, however, operations like restoreHPA cost a lot(which calls LIST API) if rc.object.Spec.MinReadySeconds != setting.MinReadySeconds { // restore the hpa if err := hpa.RestoreHPA(rc.client, rc.object); err != nil { @@ -170,21 +166,18 @@ func (rc *realController) Finalize(release *v1beta1.BatchRelease) error { if err := rc.client.Patch(context.TODO(), c, patchData); err != nil { return err } - // we should return an error to trigger re-enqueue, so that we can go to the next if-else branch in the next reconcile - return errors.NewBenignError(fmt.Errorf("cloneset bluegreen: we should wait all pods updated and available")) - } else { - klog.InfoS("Finalize: cloneset bluegreen release: wait all pods updated and ready", "cloneset", klog.KObj(rc.object)) - // wait all pods updated and ready - if rc.object.Status.ReadyReplicas != rc.object.Status.UpdatedReadyReplicas { - return errors.NewBenignError(fmt.Errorf("cloneset %v finalize not done, readyReplicas %d != updatedReadyReplicas %d, current policy %s", - klog.KObj(rc.object), rc.object.Status.ReadyReplicas, rc.object.Status.UpdatedReadyReplicas, release.Spec.ReleasePlan.FinalizingPolicy)) - } - klog.InfoS("Finalize: cloneset bluegreen release: all pods updated and ready") - // restore annotation - patchData.DeleteAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation) - patchData.DeleteAnnotation(util.BatchReleaseControlAnnotation) - return rc.client.Patch(context.TODO(), c, patchData) } + klog.InfoS("Finalize: cloneset bluegreen release: wait all pods updated and ready", "cloneset", klog.KObj(rc.object)) + // wait all pods updated and ready + if rc.object.Status.ReadyReplicas != rc.object.Status.UpdatedReadyReplicas { + return errors.NewBenignError(fmt.Errorf("cloneset %v finalize not done, readyReplicas %d != updatedReadyReplicas %d, current policy %s", + klog.KObj(rc.object), rc.object.Status.ReadyReplicas, rc.object.Status.UpdatedReadyReplicas, release.Spec.ReleasePlan.FinalizingPolicy)) + } + klog.InfoS("Finalize: cloneset bluegreen release: all pods updated and ready") + // restore annotation + patchData.DeleteAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation) + patchData.DeleteAnnotation(util.BatchReleaseControlAnnotation) + return rc.client.Patch(context.TODO(), c, patchData) } func (rc *realController) finalized() bool { diff --git a/pkg/controller/batchrelease/control/bluegreenstyle/deployment/control.go b/pkg/controller/batchrelease/control/bluegreenstyle/deployment/control.go index 3c33b968..da93a2f3 100644 --- a/pkg/controller/batchrelease/control/bluegreenstyle/deployment/control.go +++ b/pkg/controller/batchrelease/control/bluegreenstyle/deployment/control.go @@ -47,12 +47,14 @@ type realController struct { pods []*corev1.Pod key types.NamespacedName object *apps.Deployment + finder *util.ControllerFinder } func NewController(cli client.Client, key types.NamespacedName, _ schema.GroupVersionKind) bluegreenstyle.Interface { return &realController{ key: key, client: cli, + finder: util.NewControllerFinder(cli), } } @@ -82,39 +84,29 @@ func (rc *realController) ListOwnedPods() ([]*corev1.Pod, error) { return rc.pods, err } -// Add OriginalDeploymentStrategyAnnotation to workload +// Initialize prepares the Deployment for the BatchRelease process func (rc *realController) Initialize(release *v1beta1.BatchRelease) error { if rc.object == nil || control.IsControlledByBatchRelease(release, rc.object) { return nil } - // disable the hpa + // Disable the HPA if err := hpa.DisableHPA(rc.client, rc.object); err != nil { return err } - klog.InfoS("Initialize: disable hpa for deployment successfully", "deployment", klog.KObj(rc.object)) - // update the deployment - setting, err := control.GetOriginalSetting(rc.object) - if err != nil { - return errors.NewFatalError(fmt.Errorf("cannot get original setting for cloneset %v: %s from annotation", klog.KObj(rc.object), err.Error())) + klog.InfoS("Initialize: disabled HPA for deployment successfully", "deployment", klog.KObj(rc.object)) + + // Patch minReadySeconds for stable ReplicaSet + if err := rc.patchStableRSMinReadySeconds(v1beta1.MaxReadySeconds); err != nil { + return err } - control.InitOriginalSetting(&setting, rc.object) - klog.InfoS("Initialize deployment", "deployment", klog.KObj(rc.object), "setting", util.DumpJSON(&setting)) + klog.InfoS("Initialize: patched minReadySeconds for stable replicaset successfully", "deployment", klog.KObj(rc.object)) - patchData := patch.NewDeploymentPatch() - patchData.InsertAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation, util.DumpJSON(&setting)) - patchData.InsertAnnotation(util.BatchReleaseControlAnnotation, util.DumpJSON(metav1.NewControllerRef( - release, release.GetObjectKind().GroupVersionKind()))) - // update: MinReadySeconds, ProgressDeadlineSeconds, MaxSurge, MaxUnavailable - patchData.UpdateStrategy(apps.DeploymentStrategy{ - Type: apps.RollingUpdateDeploymentStrategyType, - RollingUpdate: &apps.RollingUpdateDeployment{ - MaxSurge: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, - MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 0}, - }, - }) - patchData.UpdateMinReadySeconds(v1beta1.MaxReadySeconds) - patchData.UpdateProgressDeadlineSeconds(utilpointer.Int32(v1beta1.MaxProgressSeconds)) - return rc.client.Patch(context.TODO(), util.GetEmptyObjectWithKey(rc.object), patchData) + // Patch Deplopyment + if err := rc.patchDeployment(release); err != nil { + return err + } + klog.InfoS("Initialize: patched deployment successfully", "deployment", klog.KObj(rc.object)) + return nil } func (rc *realController) UpgradeBatch(ctx *batchcontext.BatchContext) error { @@ -131,6 +123,7 @@ func (rc *realController) UpgradeBatch(ctx *batchcontext.BatchContext) error { klog.Infof("Ready to upgrade batch for deployment %v: current %d < desired %d", klog.KObj(rc.object), current, desired) patchData := patch.NewDeploymentPatch() // different with canary release, bluegreen don't need to set pause in the process of rollout + // because our webhook may pause the Deployment in some situations, we ensure that the Deployment is not paused patchData.UpdatePaused(false) patchData.UpdateStrategy(apps.DeploymentStrategy{ Type: apps.RollingUpdateDeploymentStrategyType, @@ -166,10 +159,6 @@ func (rc *realController) Finalize(release *v1beta1.BatchRelease) error { return errors.NewFatalError(fmt.Errorf("cannot get original setting for cloneset %v: %s from annotation", klog.KObj(rc.object), err.Error())) } patchData := patch.NewDeploymentPatch() - // why we need a simple MinReadySeconds-based status machine? (ie. the if-else block) - // It's possible for Finalize to be called multiple times, if error returned is not nil. - // if we do all needed operations in a single code block, like, A->B->C, when C need retry, - // both A and B will be executed as well, however, operations like restoreHPA cost a lot(which calls LIST API) if rc.object.Spec.MinReadySeconds != setting.MinReadySeconds { // restore the hpa if err := hpa.RestoreHPA(rc.client, rc.object); err != nil { @@ -184,21 +173,18 @@ func (rc *realController) Finalize(release *v1beta1.BatchRelease) error { if err := rc.client.Patch(context.TODO(), d, patchData); err != nil { return err } - // we should return an error to trigger re-enqueue, so that we can go to the next if-else branch in the next reconcile - return errors.NewBenignError(fmt.Errorf("deployment bluegreen: we should wait all pods updated and available")) - } else { - klog.InfoS("Finalize: deployment bluegreen release: wait all pods updated and ready", "cloneset", klog.KObj(rc.object)) - // wait all pods updated and ready - if err := waitAllUpdatedAndReady(d.(*apps.Deployment)); err != nil { - return errors.NewBenignError(err) - } - klog.InfoS("Finalize: deployment is ready to resume, restore the original setting", "deployment", klog.KObj(rc.object)) - // restore label and annotation - patchData.DeleteAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation) - patchData.DeleteLabel(v1alpha1.DeploymentStableRevisionLabel) - patchData.DeleteAnnotation(util.BatchReleaseControlAnnotation) - return rc.client.Patch(context.TODO(), d, patchData) } + klog.InfoS("Finalize: deployment bluegreen release: wait all pods updated and ready", "cloneset", klog.KObj(rc.object)) + // wait all pods updated and ready + if err := waitAllUpdatedAndReady(d.(*apps.Deployment)); err != nil { + return errors.NewBenignError(err) + } + klog.InfoS("Finalize: deployment is ready to resume, restore the original setting", "deployment", klog.KObj(rc.object)) + // restore label and annotation + patchData.DeleteAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation) + patchData.DeleteLabel(v1alpha1.DeploymentStableRevisionLabel) + patchData.DeleteAnnotation(util.BatchReleaseControlAnnotation) + return rc.client.Patch(context.TODO(), d, patchData) } func (rc *realController) finalized() bool { @@ -301,3 +287,53 @@ func waitAllUpdatedAndReady(deployment *apps.Deployment) error { } return nil } + +// Patch minReadySeconds for stable ReplicaSet +/* + Here is why: + For rollback scenario, we should set the stable rs minReadySeconds to infinity to make pods of the stable rs unavailable, + otherwise Pods in new version would be terminated immediately when rollback happens. + we want to keep them until traffic is switched to the stable version +*/ +func (rc *realController) patchStableRSMinReadySeconds(seconds int32) error { + if stableRS, err := rc.finder.GetDeploymentStableRs(rc.object); err != nil { + return fmt.Errorf("failed to get stable ReplicaSet: %v", err) + } else if stableRS == nil { + klog.Warningf("No stable ReplicaSet found for deployment %s/%s", rc.object.Namespace, rc.object.Name) + } else { + body := fmt.Sprintf(`{"spec":{"minReadySeconds":%v}}`, seconds) + if err = rc.client.Patch(context.TODO(), stableRS, client.RawPatch(types.MergePatchType, []byte(body))); err != nil { + return fmt.Errorf("failed to patch ReplicaSet %s/%s minReadySeconds to %v: %v", stableRS.Namespace, stableRS.Name, v1beta1.MaxReadySeconds, err) + } + } + return nil +} + +// Update deployment strategy: MinReadySeconds, ProgressDeadlineSeconds, MaxSurge, MaxUnavailable +func (rc *realController) patchDeployment(release *v1beta1.BatchRelease) error { + setting, err := control.GetOriginalSetting(rc.object) + if err != nil { + return errors.NewFatalError(fmt.Errorf("cannot get original setting for deployment %v: %s", klog.KObj(rc.object), err.Error())) + } + control.InitOriginalSetting(&setting, rc.object) + patchData := patch.NewDeploymentPatch() + patchData.InsertAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation, util.DumpJSON(&setting)) + patchData.InsertAnnotation(util.BatchReleaseControlAnnotation, util.DumpJSON(metav1.NewControllerRef( + release, release.GetObjectKind().GroupVersionKind()))) + + patchData.UpdateStrategy(apps.DeploymentStrategy{ + Type: apps.RollingUpdateDeploymentStrategyType, + RollingUpdate: &apps.RollingUpdateDeployment{ + MaxSurge: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, + MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 0}, + }, + }) + patchData.UpdateMinReadySeconds(v1beta1.MaxReadySeconds) + patchData.UpdateProgressDeadlineSeconds(utilpointer.Int32(v1beta1.MaxProgressSeconds)) + + // Apply the patch to the Deployment + if err := rc.client.Patch(context.TODO(), util.GetEmptyObjectWithKey(rc.object), patchData); err != nil { + return fmt.Errorf("failed to patch deployment %v: %v", klog.KObj(rc.object), err) + } + return nil +} diff --git a/pkg/controller/batchrelease/control/bluegreenstyle/deployment/control_test.go b/pkg/controller/batchrelease/control/bluegreenstyle/deployment/control_test.go index e076c101..4d1cb3da 100644 --- a/pkg/controller/batchrelease/control/bluegreenstyle/deployment/control_test.go +++ b/pkg/controller/batchrelease/control/bluegreenstyle/deployment/control_test.go @@ -388,7 +388,8 @@ func TestRealController(t *testing.T) { release := releaseDemo.DeepCopy() clone := deploymentDemo.DeepCopy() - cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(release, clone).Build() + stableRs, canaryRs := makeStableReplicaSets(clone), makeCanaryReplicaSets(clone) + cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(release, clone, stableRs, canaryRs).Build() // build new controller c := NewController(cli, deploymentKey, clone.GroupVersionKind()).(*realController) controller, err := c.BuildController() @@ -414,6 +415,10 @@ func TestRealController(t *testing.T) { MinReadySeconds: 0, ProgressDeadlineSeconds: pointer.Int32(600), }))) + // check minReadyseconds field of stable replicaset + fetchRS := &apps.ReplicaSet{} + Expect(cli.Get(context.TODO(), types.NamespacedName{Name: stableRs.GetName(), Namespace: stableRs.GetNamespace()}, fetchRS)).NotTo(HaveOccurred()) + Expect(fetchRS.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds))) c.object = fetch // mock diff --git a/pkg/controller/rollout/rollout_progressing.go b/pkg/controller/rollout/rollout_progressing.go index 4798cf29..97e49785 100644 --- a/pkg/controller/rollout/rollout_progressing.go +++ b/pkg/controller/rollout/rollout_progressing.go @@ -25,6 +25,7 @@ import ( "github.com/openkruise/rollouts/api/v1beta1" "github.com/openkruise/rollouts/pkg/trafficrouting" "github.com/openkruise/rollouts/pkg/util" + utilerrors "github.com/openkruise/rollouts/pkg/util/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -110,7 +111,12 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *v1beta1.Rollout case v1alpha1.ProgressingReasonInRolling: klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) err = r.doProgressingInRolling(rolloutContext) - if err != nil { + if utilerrors.IsFatal(err) { + // For fatal errors, do not retry as it wastes resources and has no effect. + // therefore, we don't propagate the error, but just log it. + // user should do sth instead, eg. for bluegreen continuous release scenario, user should do rollback + klog.Warningf("rollout(%s/%s) doProgressingInRolling error(%s)", rollout.Namespace, rollout.Name, err.Error()) + } else if err != nil { return nil, err } @@ -230,6 +236,14 @@ func (r *RolloutReconciler) handleContinuousRelease(c *RolloutContext) error { klog.Infof("rollout(%s/%s) workload continuous publishing canaryRevision from(%s) -> to(%s), then restart publishing", c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.GetCanaryRevision(), c.Workload.CanaryRevision) + // do nothing for blue-green release + if c.Rollout.Spec.Strategy.IsBlueGreenRelease() { + cond := util.GetRolloutCondition(*c.NewStatus, v1beta1.RolloutConditionProgressing) + cond.Message = "[warning] new version released in progress of blue-green release, please rollback first" + c.NewStatus.Message = cond.Message + return utilerrors.NewFatalError(fmt.Errorf("cannot do continuous release for blue-green release, rollback firstly")) + } + done, err := r.doProgressingReset(c) if err != nil { klog.Errorf("rollout(%s/%s) doProgressingReset failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) diff --git a/pkg/util/client/delegating_client.go b/pkg/util/client/delegating_client.go index d677d109..41bb074f 100644 --- a/pkg/util/client/delegating_client.go +++ b/pkg/util/client/delegating_client.go @@ -153,12 +153,7 @@ func (d *delegatingReader) List(ctx context.Context, list client.ObjectList, opt return d.CacheReader.List(ctx, list, opts...) } -var DisableDeepCopy = disableDeepCopy{} - -type disableDeepCopy struct{} - -func (_ disableDeepCopy) ApplyToList(_ *client.ListOptions) { -} +var DisableDeepCopy = client.UnsafeDisableDeepCopy func isDisableDeepCopy(opts []client.ListOption) bool { for _, opt := range opts { diff --git a/pkg/util/controller_finder.go b/pkg/util/controller_finder.go index 50030561..258f29e9 100644 --- a/pkg/util/controller_finder.go +++ b/pkg/util/controller_finder.go @@ -289,7 +289,7 @@ func (r *ControllerFinder) getDeployment(namespace string, ref *rolloutv1beta1.O return &Workload{IsStatusConsistent: false}, nil } // stable replicaSet - stableRs, err := r.getDeploymentStableRs(stable) + stableRs, err := r.GetDeploymentStableRs(stable) if err != nil || stableRs == nil { return &Workload{IsStatusConsistent: false}, err } @@ -318,7 +318,7 @@ func (r *ControllerFinder) getDeployment(namespace string, ref *rolloutv1beta1.O if err != nil || canary == nil { return workload, err } - canaryRs, err := r.getDeploymentStableRs(canary) + canaryRs, err := r.GetDeploymentStableRs(canary) if err != nil || canaryRs == nil { return workload, err } @@ -422,7 +422,7 @@ func (r *ControllerFinder) GetReplicaSetsForDeployment(obj *apps.Deployment) ([] return rss, nil } -func (r *ControllerFinder) getDeploymentStableRs(obj *apps.Deployment) (*apps.ReplicaSet, error) { +func (r *ControllerFinder) GetDeploymentStableRs(obj *apps.Deployment) (*apps.ReplicaSet, error) { rss, err := r.GetReplicaSetsForDeployment(obj) if err != nil { return nil, err diff --git a/pkg/webhook/util/writer/fs.go b/pkg/webhook/util/writer/fs.go index c003caa7..543e94f3 100644 --- a/pkg/webhook/util/writer/fs.go +++ b/pkg/webhook/util/writer/fs.go @@ -128,7 +128,7 @@ func prepareToWrite(dir string) error { // TODO: figure out if we can reduce the permission. (Now it's 0777) err = os.MkdirAll(dir, 0777) if err != nil { - return fmt.Errorf("can't create dir: %v", dir) + return fmt.Errorf("can't create dir: %v, err: %s", dir, err.Error()) } case err != nil: return err diff --git a/pkg/webhook/workload/mutating/workload_update_handler.go b/pkg/webhook/workload/mutating/workload_update_handler.go index 45ade337..01700046 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler.go +++ b/pkg/webhook/workload/mutating/workload_update_handler.go @@ -263,13 +263,13 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo // in rollout progressing if newObj.Annotations[util.InRolloutProgressingAnnotation] != "" { modified := false - if !newObj.Spec.Paused { - modified = true - newObj.Spec.Paused = true - } strategy := util.GetDeploymentStrategy(newObj) - switch strings.ToLower(string(strategy.RollingStyle)) { - case strings.ToLower(string(appsv1alpha1.PartitionRollingStyle)): + // partition + if strings.EqualFold(string(strategy.RollingStyle), string(appsv1alpha1.PartitionRollingStyle)) { + if !newObj.Spec.Paused { + modified = true + newObj.Spec.Paused = true + } // Make sure it is always Recreate to disable native controller if newObj.Spec.Strategy.Type == apps.RollingUpdateDeploymentStrategyType { modified = true @@ -287,7 +287,24 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo } appsv1alpha1.SetDefaultDeploymentStrategy(&strategy) setDeploymentStrategyAnnotation(strategy, newObj) - default: + // bluegreenStyle + } else if len(newObj.GetAnnotations()[appsv1beta1.OriginalDeploymentStrategyAnnotation]) > 0 { + if isEffectiveDeploymentRevisionChange(oldObj, newObj) { + newObj.Spec.Paused, modified = true, true + // disallow continuous release, allow rollback + klog.Warningf("rollback or continuous release detected in Deployment webhook, while only rollback is allowed for bluegreen release for now") + } + // not allow to modify Strategy.Type to Recreate + if newObj.Spec.Strategy.Type != apps.RollingUpdateDeploymentStrategyType { + modified = true + newObj.Spec.Strategy.Type = oldObj.Spec.Strategy.Type + klog.Warningf("Not allow to modify Strategy.Type to Recreate") + } + } else { // default + if !newObj.Spec.Paused { + modified = true + newObj.Spec.Paused = true + } // Do not allow to modify strategy as Recreate during rolling if newObj.Spec.Strategy.Type == apps.RecreateDeploymentStrategyType { modified = true @@ -369,6 +386,22 @@ func (h *WorkloadHandler) handleCloneSet(newObj, oldObj *kruiseappsv1alpha1.Clon } else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() { return false, nil } + /* + continuous release (or successive release) is not supported for bluegreen release, especially for cloneset, + here is why: + suppose we are releasing a cloneset, which has pods of both v1 and v2 for now. If we release v3 before + v2 release is done, the cloneset controller might scale down pods without distinguishing between v1 and v2. + This is because our implementation is based on the minReadySeconds, pods of both v1 and v2 are "unavailable" + in the progress of rollout. + Deployment actually has the same problem, however it is possible to bypass this issue for Deployment by setting + minReadySeconds for replicaset separately; unfortunately this workaround seems not work for cloneset + + // if rollout.Spec.Strategy.IsBlueGreenRelease() && revision.IsContinuousRelease(h.Client, oldObj, newObj) { + // err = fmt.Errorf("successive release is not supported for bluegreen for now, rollback first") + // return false, fmt.Errorf(err.Error()) + // } + */ + // if traffic routing, there must only be one version of Pods if rollout.Spec.Strategy.HasTrafficRoutings() && newObj.Status.Replicas != newObj.Status.UpdatedReplicas { klog.Warningf("Because cloneSet(%s/%s) have multiple versions of Pods, so can not enter rollout progressing", newObj.Namespace, newObj.Name) diff --git a/pkg/webhook/workload/mutating/workload_update_handler_test.go b/pkg/webhook/workload/mutating/workload_update_handler_test.go index 117d32ba..083c6242 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler_test.go +++ b/pkg/webhook/workload/mutating/workload_update_handler_test.go @@ -130,6 +130,51 @@ var ( }, } + rsDemoV2 = &apps.ReplicaSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "ReplicaSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "echoserver-v2", + Labels: map[string]string{ + "app": "echoserver", + "pod-template-hash": "verision2", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(deploymentDemo, schema.GroupVersionKind{ + Group: apps.SchemeGroupVersion.Group, + Version: apps.SchemeGroupVersion.Version, + Kind: "Deployment", + }), + }, + }, + Spec: apps.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "echoserver", + }, + }, + Replicas: pointer.Int32(5), + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "echoserver", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "echoserver", + Image: "echoserver:v2", + }, + }, + }, + }, + }, + } + cloneSetDemo = &kruisev1aplphal.CloneSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps.kruise.io/v1alpha1", @@ -519,6 +564,100 @@ func TestHandlerDeployment(t *testing.T) { return obj }, }, + { + name: "bluegreen: normal release", + getObjs: func() (*apps.Deployment, *apps.Deployment) { + oldObj := deploymentDemo.DeepCopy() + newObj := deploymentDemo.DeepCopy() + newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + return oldObj, newObj + }, + expectObj: func() *apps.Deployment { + obj := deploymentDemo.DeepCopy() + obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + obj.Spec.Paused = true + return obj + }, + getRs: func() []*apps.ReplicaSet { + rs := rsDemo.DeepCopy() + return []*apps.ReplicaSet{rs} + }, + getRollout: func() *appsv1beta1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.BlueGreen = &appsv1beta1.BlueGreenStrategy{} + return obj + }, + isError: false, + }, + { + name: "bluegreen: rollback", + getObjs: func() (*apps.Deployment, *apps.Deployment) { + oldObj := deploymentDemo.DeepCopy() + oldObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + oldObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + oldObj.Labels[appsv1alpha1.DeploymentStableRevisionLabel] = "5b494f7bf" + oldObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + newObj := deploymentDemo.DeepCopy() + newObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + newObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v1" + return oldObj, newObj + }, + expectObj: func() *apps.Deployment { + obj := deploymentDemo.DeepCopy() + obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v1" + obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + obj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + obj.Spec.Paused = true + return obj + }, + getRs: func() []*apps.ReplicaSet { + rs := rsDemo.DeepCopy() + rs2 := rsDemoV2.DeepCopy() + return []*apps.ReplicaSet{rs, rs2} + }, + getRollout: func() *appsv1beta1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.BlueGreen = &appsv1beta1.BlueGreenStrategy{} + return obj + }, + isError: false, + }, + { + name: "bluegreen: successive release", + getObjs: func() (*apps.Deployment, *apps.Deployment) { + oldObj := deploymentDemo.DeepCopy() + oldObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + oldObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + oldObj.Labels[appsv1alpha1.DeploymentStableRevisionLabel] = "5b494f7bf" + oldObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + newObj := deploymentDemo.DeepCopy() + newObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + newObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v3" + return oldObj, newObj + }, + expectObj: func() *apps.Deployment { + obj := deploymentDemo.DeepCopy() + obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v3" + obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + obj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}` + obj.Spec.Paused = true + return obj + }, + getRs: func() []*apps.ReplicaSet { + rs := rsDemo.DeepCopy() + rs2 := rsDemoV2.DeepCopy() + return []*apps.ReplicaSet{rs, rs2} + }, + getRollout: func() *appsv1beta1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.BlueGreen = &appsv1beta1.BlueGreenStrategy{} + return obj + }, + isError: false, + }, } decoder, _ := admission.NewDecoder(scheme) @@ -542,8 +681,11 @@ func TestHandlerDeployment(t *testing.T) { oldObj, newObj := cs.getObjs() _, err := h.handleDeployment(newObj, oldObj) - if cs.isError && err == nil { - t.Fatal("handlerDeployment failed") + if cs.isError { + if err == nil { + t.Fatal("handlerDeployment failed") + } + return //no need to check again } else if !cs.isError && err != nil { t.Fatalf(err.Error()) } diff --git a/test/e2e/rollout_v1beta1_test.go b/test/e2e/rollout_v1beta1_test.go index a3c3e3e1..1ed52170 100644 --- a/test/e2e/rollout_v1beta1_test.go +++ b/test/e2e/rollout_v1beta1_test.go @@ -112,25 +112,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { return clone } - // continuous release is not allowed for now, therefor we expect failure when updating - UpdateDeploymentFailed := func(object *apps.Deployment) *apps.Deployment { - var clone *apps.Deployment - Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { - clone = &apps.Deployment{} - err := GetObject(object.Name, clone) - if err != nil { - return err - } - clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) - clone.Spec.Template = *object.Spec.Template.DeepCopy() - clone.Labels = mergeMap(clone.Labels, object.Labels) - clone.Annotations = mergeMap(clone.Annotations, object.Annotations) - clone.Spec.Paused = object.Spec.Paused - return k8sClient.Update(context.TODO(), clone) - })).To(HaveOccurred()) - - return clone - } UpdateCloneSet := func(object *appsv1alpha1.CloneSet) *appsv1alpha1.CloneSet { var clone *appsv1alpha1.CloneSet @@ -150,25 +131,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { return clone } - // continuous release is not allowed for now, therefor we expect failure when updating - UpdateCloneSetFail := func(object *appsv1alpha1.CloneSet) *appsv1alpha1.CloneSet { - var clone *appsv1alpha1.CloneSet - Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { - clone = &appsv1alpha1.CloneSet{} - err := GetObject(object.Name, clone) - if err != nil { - return err - } - clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) - clone.Spec.Template = *object.Spec.Template.DeepCopy() - clone.Labels = mergeMap(clone.Labels, object.Labels) - clone.Annotations = mergeMap(clone.Annotations, object.Annotations) - return k8sClient.Update(context.TODO(), clone) - })).To(HaveOccurred()) - - return clone - } - // UpdateDaemonSet := func(object *appsv1alpha1.DaemonSet) *appsv1alpha1.DaemonSet { // var daemon *appsv1alpha1.DaemonSet // Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -353,8 +315,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { Eventually(func() bool { clone := &apps.Deployment{} Expect(GetObject(deployment.Name, clone)).NotTo(HaveOccurred()) - return clone.Status.ObservedGeneration == clone.Generation && - *clone.Spec.Replicas == clone.Status.AvailableReplicas && clone.Status.ReadyReplicas == clone.Status.Replicas + return clone.Status.ObservedGeneration == clone.Generation && clone.Status.ReadyReplicas == clone.Status.Replicas }, 10*time.Minute, time.Second).Should(BeTrue()) } @@ -1696,8 +1657,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status & paused Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) setting, _ := control.GetOriginalSetting(workload) @@ -1751,8 +1711,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10)) Expect(workload.Spec.Paused).Should(BeFalse()) Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) @@ -1777,8 +1736,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10)) Expect(workload.Spec.Paused).Should(BeFalse()) Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) @@ -1817,8 +1775,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10)) Expect(workload.Spec.Paused).Should(BeFalse()) Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) @@ -1868,7 +1825,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) for _, env := range workload.Spec.Template.Spec.Containers[0].Env { if env.Name == "NODE_NAME" { Expect(env.Value).Should(Equal("version2")) @@ -1946,8 +1902,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status & paused Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) setting, _ := control.GetOriginalSetting(workload) @@ -2001,8 +1956,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10)) Expect(workload.Spec.Paused).Should(BeFalse()) Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) @@ -2027,8 +1981,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10)) Expect(workload.Spec.Paused).Should(BeFalse()) Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) @@ -2067,8 +2020,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10)) Expect(workload.Spec.Paused).Should(BeFalse()) Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) @@ -2105,8 +2057,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10)) Expect(workload.Spec.Paused).Should(BeFalse()) Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) @@ -2143,8 +2094,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10)) Expect(workload.Spec.Paused).Should(BeFalse()) Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) @@ -2246,8 +2196,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status & paused Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) setting, _ := control.GetOriginalSetting(workload) @@ -2266,169 +2215,82 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // ----- Continuous Release ------ updatedRevision := rollout.Status.BlueGreenStatus.UpdatedRevision By(updatedRevision) - By("update workload env NODE_NAME from(version2) -> to(version3)") newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateDeploymentFailed(workload) - // the next code is used to test continuous release scenario, in case we need it in the future, keep it as comment - /* - UpdateDeployment(workload) - // from step 1 to step 1, we need to additionally check stepUpgrad to distinguish the two steps - WaitRolloutStepUpgrade(rollout.Name, 1) - WaitRolloutStepPaused(rollout.Name, 1) - // stable revision shouldn't change - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).ShouldNot(Equal(updatedRevision)) - Expect(workload.Labels[v1beta1.DeploymentStableRevisionLabel]).Should(Equal(stableRevision)) + UpdateDeployment(workload) + By("update workload env NODE_NAME from(version2) -> to(version3)") + WaitRolloutStepPaused(rollout.Name, 1) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus).Should(BeNil()) + Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision)) - // check workload status & paused - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) - - setting = control.GetOriginalSetting(workload) - Expect(setting.MinReadySeconds).Should(BeNumerically("==", int32(0))) - Expect(*setting.ProgressDeadlineSeconds).Should(BeNumerically("==", int32(600))) - Expect(reflect.DeepEqual(setting.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue()) - Expect(reflect.DeepEqual(setting.MaxSurge, &intstr.IntOrString{Type: intstr.Int, IntVal: 1})).Should(BeTrue()) - - Expect(workload.Spec.Paused).Should(BeFalse()) - Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) - Expect(workload.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds))) - Expect(*workload.Spec.ProgressDeadlineSeconds).Should(Equal(int32(v1beta1.MaxProgressSeconds))) - Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue()) - Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxSurge, &intstr.IntOrString{Type: intstr.String, StrVal: "50%"})).Should(BeTrue()) + // check workload status & paused + time.Sleep(time.Second * 1) // ensure the Deployment controller notice the update + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) // no version3 pods created, since we don't support continuous release yet + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) - // check rollout status - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) - Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) - Expect(rollout.Status.BlueGreenStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) - canaryRevision := rollout.Status.BlueGreenStatus.PodTemplateHash - Expect(rollout.Status.BlueGreenStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 2)) - Expect(rollout.Status.BlueGreenStatus.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(rollout.Status.BlueGreenStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) - // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) - //canary service - cService := &v1.Service{} - Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) - - // ------ step 2: replicas: 100%, traffic: 0% ------ - // resume rollout canary - ResumeRollout(rollout.Name) - By("resume rollout, and wait next step(2)") - WaitRolloutStepPaused(rollout.Name, 2) + setting, _ = control.GetOriginalSetting(workload) + Expect(setting.MinReadySeconds).Should(BeNumerically("==", int32(0))) + Expect(*setting.ProgressDeadlineSeconds).Should(BeNumerically("==", int32(600))) + Expect(reflect.DeepEqual(setting.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue()) + Expect(reflect.DeepEqual(setting.MaxSurge, &intstr.IntOrString{Type: intstr.Int, IntVal: 1})).Should(BeTrue()) - // workload - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10)) - Expect(workload.Spec.Paused).Should(BeFalse()) - Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) - Expect(workload.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds))) - Expect(*workload.Spec.ProgressDeadlineSeconds).Should(Equal(int32(v1beta1.MaxProgressSeconds))) - Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue()) - Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxSurge, &intstr.IntOrString{Type: intstr.String, StrVal: "100%"})).Should(BeTrue()) - - // rollout - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) - Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 3)) - Expect(rollout.Status.BlueGreenStatus.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(rollout.Status.BlueGreenStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) - // ----- Continuous Release, AGAIN------ - updatedRevision = rollout.Status.BlueGreenStatus.UpdatedRevision - By("update workload env NODE_NAME from(version3) -> to(version4)") - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version4"}) - workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateDeployment(workload) - WaitRolloutStepPaused(rollout.Name, 1) - // stable revision shouldn't change - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).ShouldNot(Equal(updatedRevision)) - Expect(workload.Labels[v1beta1.DeploymentStableRevisionLabel]).Should(Equal(stableRevision)) - // workload - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) - Expect(workload.Spec.Paused).Should(BeFalse()) - Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) - Expect(workload.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds))) - Expect(*workload.Spec.ProgressDeadlineSeconds).Should(Equal(int32(v1beta1.MaxProgressSeconds))) - Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue()) - Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxSurge, &intstr.IntOrString{Type: intstr.String, StrVal: "50%"})).Should(BeTrue()) - - // rollout - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 2)) - Expect(rollout.Status.BlueGreenStatus.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(rollout.Status.BlueGreenStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Spec.Paused).Should(BeTrue()) // paused in the webhook + Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) + Expect(workload.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds))) + Expect(*workload.Spec.ProgressDeadlineSeconds).Should(Equal(int32(v1beta1.MaxProgressSeconds))) + Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue()) + Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxSurge, &intstr.IntOrString{Type: intstr.String, StrVal: "50%"})).Should(BeTrue()) - // ------ step 4: replicas: 100%, traffic: 100% ------ - // resume rollout canary - By("Jump to step 4") - JumpRolloutStep(rollout.Name, 4) - WaitRolloutStepPaused(rollout.Name, 4) + // it's ok to patch the Deployment to version2 back, and even release remaining steps then + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + By("update workload env NODE_NAME from(version3) -> to(version2)") + WaitRolloutStepPaused(rollout.Name, 1) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus).Should(BeNil()) + Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision)) - // ------ Final approval ------ - // resume rollout canary - ResumeRollout(rollout.Name) - By("resume rollout, final approval") - // wait rollout complete - WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhase(v1beta1.RolloutPhaseHealthy)) - klog.Infof("rollout(%s) completed, and check", namespace) - // rollout - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", -1)) - // check service & ingress - // ingress - Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) - cIngress := &netv1.Ingress{} - Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) - // service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) - cService = &v1.Service{} - Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) - // workload - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) - Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas)) - Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) - for _, env := range workload.Spec.Template.Spec.Containers[0].Env { - if env.Name == "NODE_NAME" { - Expect(env.Value).Should(Equal("version4")) - } - } - // check progressing succeed - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) - cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) - Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - */ + // check workload status & paused + time.Sleep(time.Second * 1) // ensure the Deployment controller notice the update + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) + + setting, _ = control.GetOriginalSetting(workload) + Expect(setting.MinReadySeconds).Should(BeNumerically("==", int32(0))) + Expect(*setting.ProgressDeadlineSeconds).Should(BeNumerically("==", int32(600))) + Expect(reflect.DeepEqual(setting.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue()) + Expect(reflect.DeepEqual(setting.MaxSurge, &intstr.IntOrString{Type: intstr.Int, IntVal: 1})).Should(BeTrue()) + + Expect(workload.Spec.Paused).Should(BeTrue()) + Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) + Expect(workload.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds))) + Expect(*workload.Spec.ProgressDeadlineSeconds).Should(Equal(int32(v1beta1.MaxProgressSeconds))) + Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue()) + Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxSurge, &intstr.IntOrString{Type: intstr.String, StrVal: "50%"})).Should(BeTrue()) + + // of course user can rollback to version1 directly + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + By("rollback: update workload env NODE_NAME from(version2) -> to(version1)") + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitDeploymentAllPodsReady(workload) + + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(len(rollout.GetAnnotations()[v1beta1.OriginalDeploymentStrategyAnnotation])).Should(BeNumerically("==", 0)) // the annotation should be removed + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(string(cond.Reason)).Should(Equal(string(v1beta1.CanaryStepStateCompleted))) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) }) It("bluegreen scale up and down", func() { @@ -2480,8 +2342,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status & paused Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) // check rollout status @@ -2496,6 +2357,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // ------ 50% maxSurge, scale up: from 5 to 6 ------ workload.Spec.Replicas = utilpointer.Int32(6) UpdateDeployment(workload) + By("scale up: from 5 to 6") time.Sleep(time.Second * 3) WaitDeploymentBlueGreenReplicas(workload) @@ -2508,8 +2370,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 6)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 9)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 9)) // ------ scale up: from 6 to 7 ------ @@ -2524,13 +2385,13 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 4)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 7)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 11)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 11)) // ------ scale up: from 7 to 8 ------ workload.Spec.Replicas = utilpointer.Int32(8) UpdateDeployment(workload) + By("scale up: from 7 to 8") time.Sleep(time.Second * 3) WaitDeploymentBlueGreenReplicas(workload) // check rollout status @@ -2540,13 +2401,13 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 4)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 8)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 12)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 12)) // ------ scale down: from 8 to 4 ------ workload.Spec.Replicas = utilpointer.Int32(4) UpdateDeployment(workload) + By("scale down: from 8 to 4") time.Sleep(time.Second * 3) WaitDeploymentBlueGreenReplicas(workload) // check rollout status @@ -2556,8 +2417,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 2)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 2)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 6)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 6)) // ------ step 2: replicas: 100%, traffic: 0% ------ @@ -2569,8 +2429,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 4)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) Expect(workload.Spec.Paused).Should(BeFalse()) Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType)) @@ -2598,8 +2457,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 7)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 7)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 7)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 14)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 14)) // ------ scale up: from 7 to 8 ------ @@ -2614,8 +2472,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 8)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 8)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 16)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 16)) // ------ scale down: from 8 to 4 ------ @@ -2630,8 +2487,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 4)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) }) @@ -2685,8 +2541,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status & paused Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) // check rollout status @@ -2732,7 +2587,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) }) It("bluegreen disable rollout case", func() { @@ -2784,8 +2638,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status & paused Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) // check rollout status @@ -2846,7 +2699,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) }) }) @@ -2906,8 +2758,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status & paused Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) // check rollout status @@ -2959,7 +2810,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) // check hpa Expect(GetObject(hpa.Name, hpa)).NotTo(HaveOccurred()) @@ -3021,8 +2871,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // check workload status & paused Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8)) Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) // check rollout status @@ -3073,7 +2922,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // status Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) // check hpa Expect(GetObject(hpa.Name, hpa)).NotTo(HaveOccurred()) Expect(hpa.Spec.ScaleTargetRef.Name).Should(Equal(workload.Name)) @@ -3225,7 +3073,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // workload Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) for _, env := range workload.Spec.Template.Spec.Containers[0].Env { if env.Name == "NODE_NAME" { Expect(env.Value).Should(Equal("version2")) @@ -3525,12 +3372,13 @@ var _ = SIGDescribe("Rollout v1beta1", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision)) + revision2 := workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:] Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.BlueGreenStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 3)) Expect(rollout.Status.BlueGreenStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) - // if network configuration has restored + // check if network configuration has restored cIngress := &netv1.Ingress{} Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) @@ -3542,7 +3390,77 @@ var _ = SIGDescribe("Rollout v1beta1", func() { By("update workload env NODE_NAME from(version2) -> to(version3)") newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) workload.Spec.Template.Spec.Containers[0].Env = newEnvs - UpdateCloneSetFail(workload) + UpdateCloneSet(workload) + time.Sleep(time.Second * 1) + WaitRolloutStepPaused(rollout.Name, 2) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.Replicas).Should(BeNumerically("==", 10)) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) // unlike Deployment, cloneSet isn't paused + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision)) + /* + note: rollout.Status.BlueGreenStatus.UpdatedRevision won't update at all, since we disallow + continuous release for bluegreen release (it is designed to trigger a fatal error before status update) + however the workload.Status.UpdateRevision will always be update since it is calculated + directly from the Cloneset + */ + Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).Should(Equal(revision2)) + Expect(rollout.Status.BlueGreenStatus.PodTemplateHash).Should(Equal(revision2)) + Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.BlueGreenStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // it's ok to patch the CloneSet to version2 back, and even release remaining steps then + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + By("update workload env NODE_NAME from(version3) -> to(version2)") + time.Sleep(time.Second * 1) + WaitRolloutStepPaused(rollout.Name, 2) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.Replicas).Should(BeNumerically("==", 10)) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) // unlike Deployment, cloneSet isn't paused + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).Should(Equal(revision2)) + Expect(rollout.Status.BlueGreenStatus.PodTemplateHash).Should(Equal(revision2)) + Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.BlueGreenStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // of course user can rollback to version1 directly + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + By("rollback: update workload env NODE_NAME from(version2) -> to(version1)") + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + WaitCloneSetAllPodsReady(workload) + + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(string(cond.Reason)).Should(Equal(string(v1beta1.CanaryStepStateCompleted))) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + Expect(len(rollout.GetAnnotations()[v1beta1.OriginalDeploymentStrategyAnnotation])).Should(BeNumerically("==", 0)) // the annotation should be removed + CheckIngressRestored(service.Name) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.Replicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) }) // cloneset now only support single step, keep this case for future @@ -3631,7 +3549,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // // check workload status // Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) // Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 6)) - // Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 6)) // Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 12)) // // ------ scale up: from 6 to 7 ------ @@ -3650,7 +3567,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // // check workload status // Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) // Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 7)) - // Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 7)) // Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 14)) // // ------ scale up: from 7 to 8 ------ @@ -3669,7 +3585,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // // check workload status // Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) // Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 8)) - // Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 8)) // Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 16)) // // ------ scale down: from 8 to 4 ------ @@ -3688,7 +3603,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // // check workload status // Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) // Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) - // Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) // Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8)) // }) @@ -3808,7 +3722,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { } } Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) // check service & ingress & deployment // ingress @@ -3946,7 +3859,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { } } Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) // check service & ingress & deployment // ingress @@ -4638,7 +4550,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() { // cloneset Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) - Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) for _, env := range workload.Spec.Template.Spec.Containers[0].Env { if env.Name == "NODE_NAME" { Expect(env.Value).Should(Equal("version2"))