diff --git a/api/v1alpha1/rollout_types.go b/api/v1alpha1/rollout_types.go index d9baafcd..e41ccea4 100644 --- a/api/v1alpha1/rollout_types.go +++ b/api/v1alpha1/rollout_types.go @@ -25,6 +25,23 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +const ( + // RolloutIDLabel is set to workload labels. + // RolloutIDLabel is designed to distinguish each workload revision publications. + // The value of RolloutIDLabel corresponds Rollout.Spec.RolloutID. + RolloutIDLabel = "rollouts.kruise.io/rollout-id" + + // RolloutBatchIDLabel is patched in pod labels. + // RolloutBatchIDLabel is the label key of batch id that will be patched to pods during rollout. + // Only when RolloutIDLabel is set, RolloutBatchIDLabel will be patched. + // Users can use RolloutIDLabel and RolloutBatchIDLabel to select the pods that are upgraded in some certain batch and release. + RolloutBatchIDLabel = "rollouts.kruise.io/rollout-batch-id" + + // RollbackInBatchAnnotation is set to rollout annotations. + // RollbackInBatchAnnotation allow use disable quick rollback, and will roll back in batch style. + RollbackInBatchAnnotation = "rollouts.kruise.io/rollback-in-batch" +) + // RolloutSpec defines the desired state of Rollout type RolloutSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster @@ -33,9 +50,11 @@ type RolloutSpec struct { ObjectRef ObjectRef `json:"objectRef"` // rollout strategy Strategy RolloutStrategy `json:"strategy"` + // DeprecatedRolloutID is the deprecated field. + // It is recommended that configure RolloutId in workload.annotations[rollouts.kruise.io/rollout-id]. // RolloutID should be changed before each workload revision publication. // It is to distinguish consecutive multiple workload publications and rollout progress. - RolloutID string `json:"rolloutID,omitempty"` + DeprecatedRolloutID string `json:"rolloutID,omitempty"` } type ObjectRef struct { @@ -158,17 +177,12 @@ type RolloutStatus struct { // observedGeneration is the most recent generation observed for this Rollout. ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // CanaryRevision the hash of the canary pod template + // Canary describes the state of the canary rollout // +optional - //CanaryRevision string `json:"canaryRevision,omitempty"` - // StableRevision indicates the revision pods that has successfully rolled out - StableRevision string `json:"stableRevision,omitempty"` + CanaryStatus *CanaryStatus `json:"canaryStatus,omitempty"` // Conditions a list of conditions a rollout can have. // +optional Conditions []RolloutCondition `json:"conditions,omitempty"` - // Canary describes the state of the canary rollout - // +optional - CanaryStatus *CanaryStatus `json:"canaryStatus,omitempty"` // +optional //BlueGreenStatus *BlueGreenStatus `json:"blueGreenStatus,omitempty"` // Phase is the rollout phase. @@ -227,11 +241,10 @@ type CanaryStatus struct { ObservedRolloutID string `json:"observedRolloutID,omitempty"` // RolloutHash from rollout.spec object RolloutHash string `json:"rolloutHash,omitempty"` - // CanaryService holds the name of a service which selects pods with canary version and don't select any pods with stable version. - CanaryService string `json:"canaryService"` + // StableRevision indicates the revision of stable pods + StableRevision string `json:"stableRevision,omitempty"` // CanaryRevision is calculated by rollout based on podTemplateHash, and the internal logic flow uses // It may be different from rs podTemplateHash in different k8s versions, so it cannot be used as service selector label - // +optional CanaryRevision string `json:"canaryRevision"` // pod template hash is used as service selector label PodTemplateHash string `json:"podTemplateHash"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ff60b066..e124c408 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -643,6 +643,11 @@ func (in *RolloutSpec) DeepCopy() *RolloutSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RolloutStatus) DeepCopyInto(out *RolloutStatus) { *out = *in + if in.CanaryStatus != nil { + in, out := &in.CanaryStatus, &out.CanaryStatus + *out = new(CanaryStatus) + (*in).DeepCopyInto(*out) + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]RolloutCondition, len(*in)) @@ -650,11 +655,6 @@ func (in *RolloutStatus) DeepCopyInto(out *RolloutStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.CanaryStatus != nil { - in, out := &in.CanaryStatus, &out.CanaryStatus - *out = new(CanaryStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutStatus. diff --git a/config/crd/bases/rollouts.kruise.io_rollouts.yaml b/config/crd/bases/rollouts.kruise.io_rollouts.yaml index c7626aa9..8e42d9c4 100644 --- a/config/crd/bases/rollouts.kruise.io_rollouts.yaml +++ b/config/crd/bases/rollouts.kruise.io_rollouts.yaml @@ -81,9 +81,11 @@ spec: type: object type: object rolloutID: - description: RolloutID should be changed before each workload revision - publication. It is to distinguish consecutive multiple workload - publications and rollout progress. + description: DeprecatedRolloutID is the deprecated field. It is recommended + that configure RolloutId in workload.annotations[rollouts.kruise.io/rollout-id]. + RolloutID should be changed before each workload revision publication. + It is to distinguish consecutive multiple workload publications + and rollout progress. type: string strategy: description: rollout strategy @@ -202,11 +204,6 @@ spec: different from rs podTemplateHash in different k8s versions, so it cannot be used as service selector label type: string - canaryService: - description: CanaryService holds the name of a service which selects - pods with canary version and don't select any pods with stable - version. - type: string currentStepIndex: description: CurrentStepIndex defines the current step of the rollout is on. If the current step index is null, the controller @@ -235,10 +232,13 @@ spec: rolloutHash: description: RolloutHash from rollout.spec object type: string + stableRevision: + description: StableRevision indicates the revision of stable pods + type: string required: - canaryReadyReplicas - canaryReplicas - - canaryService + - canaryRevision - currentStepState - podTemplateHash type: object @@ -290,11 +290,6 @@ spec: description: BlueGreenStatus *BlueGreenStatus `json:"blueGreenStatus,omitempty"` Phase is the rollout phase. type: string - stableRevision: - description: CanaryRevision the hash of the canary pod template CanaryRevision - string `json:"canaryRevision,omitempty"` StableRevision indicates - the revision pods that has successfully rolled out - type: string type: object type: object served: true diff --git a/main.go b/main.go index 36b90c67..116d2e60 100644 --- a/main.go +++ b/main.go @@ -26,7 +26,6 @@ import ( br "github.com/openkruise/rollouts/pkg/controller/batchrelease" "github.com/openkruise/rollouts/pkg/controller/rollout" "github.com/openkruise/rollouts/pkg/controller/rollouthistory" - "github.com/openkruise/rollouts/pkg/util" utilclient "github.com/openkruise/rollouts/pkg/util/client" utilfeature "github.com/openkruise/rollouts/pkg/util/feature" "github.com/openkruise/rollouts/pkg/webhook" @@ -102,7 +101,6 @@ func main() { if err = (&rollout.RolloutReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - Finder: util.NewControllerFinder(mgr.GetClient()), Recorder: mgr.GetEventRecorderFor("rollout-controller"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Rollout") diff --git a/pkg/controller/batchrelease/batchrelease_status.go b/pkg/controller/batchrelease/batchrelease_status.go index 7c8ad451..d7185892 100644 --- a/pkg/controller/batchrelease/batchrelease_status.go +++ b/pkg/controller/batchrelease/batchrelease_status.go @@ -220,7 +220,7 @@ func isWorkloadRevisionChanged(event workloads.WorkloadEventType, release *v1alp } func isWorkloadRollbackInBatch(event workloads.WorkloadEventType, release *v1alpha1.BatchRelease) bool { - return (event == workloads.WorkloadRollbackInBatch || release.Annotations[util.RollbackInBatchAnnotation] != "") && + return (event == workloads.WorkloadRollbackInBatch || release.Annotations[v1alpha1.RollbackInBatchAnnotation] != "") && release.Status.CanaryStatus.NoNeedUpdateReplicas == nil && release.Status.Phase == v1alpha1.RolloutPhaseProgressing } @@ -236,5 +236,5 @@ func isRollbackInBatchSatisfied(workloadInfo *util.WorkloadInfo, release *v1alph if workloadInfo.Status == nil { return false } - return workloadInfo.Status.StableRevision == workloadInfo.Status.UpdateRevision && release.Annotations[util.RollbackInBatchAnnotation] != "" + return workloadInfo.Status.StableRevision == workloadInfo.Status.UpdateRevision && release.Annotations[v1alpha1.RollbackInBatchAnnotation] != "" } diff --git a/pkg/controller/batchrelease/workloads/cloneset_control_plane.go b/pkg/controller/batchrelease/workloads/cloneset_control_plane.go index 84d1f0d5..c9d6c5e3 100644 --- a/pkg/controller/batchrelease/workloads/cloneset_control_plane.go +++ b/pkg/controller/batchrelease/workloads/cloneset_control_plane.go @@ -66,7 +66,7 @@ func (c *CloneSetRolloutController) VerifyWorkload() (bool, error) { // - *int32: how many pods have been patched; // - err: whether error occurs. func (c *CloneSetRolloutController) prepareBeforeRollback() (bool, *int32, error) { - if c.release.Annotations[util.RollbackInBatchAnnotation] != "true" { + if c.release.Annotations[v1alpha1.RollbackInBatchAnnotation] != "true" { return true, nil, nil } diff --git a/pkg/controller/batchrelease/workloads/commons.go b/pkg/controller/batchrelease/workloads/commons.go index 990ca8a3..21417ff9 100644 --- a/pkg/controller/batchrelease/workloads/commons.go +++ b/pkg/controller/batchrelease/workloads/commons.go @@ -36,7 +36,7 @@ func filterPodsForUnorderedRollback(pods []*corev1.Pod, plannedBatchCanaryReplic if !util.IsConsistentWithRevision(pod, updateRevision) { continue } - podRolloutID := pod.Labels[util.RolloutIDLabel] + podRolloutID := pod.Labels[v1alpha1.RolloutIDLabel] podRollbackID := pod.Labels[util.NoNeedUpdatePodLabel] if podRollbackID == rolloutID && podRolloutID != rolloutID { noNeedRollbackReplicas++ @@ -122,14 +122,14 @@ func patchPodBatchLabel(c client.Client, pods []*corev1.Pod, rolloutID string, b continue } - podRolloutID := pod.Labels[util.RolloutIDLabel] + podRolloutID := pod.Labels[v1alpha1.RolloutIDLabel] if pod.DeletionTimestamp.IsZero() && podRolloutID == rolloutID { patchedUpdatedReplicas++ } } for _, pod := range pods { - podRolloutID := pod.Labels[util.RolloutIDLabel] + podRolloutID := pod.Labels[v1alpha1.RolloutIDLabel] if pod.DeletionTimestamp.IsZero() { // we don't patch label for the active old revision pod if !util.IsConsistentWithRevision(pod, updateRevision) { @@ -147,7 +147,7 @@ func patchPodBatchLabel(c client.Client, pods []*corev1.Pod, rolloutID string, b } podClone := pod.DeepCopy() - by := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s","%s":"%d"}}}`, util.RolloutIDLabel, rolloutID, util.RolloutBatchIDLabel, batchID) + by := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s","%s":"%d"}}}`, v1alpha1.RolloutIDLabel, rolloutID, v1alpha1.RolloutBatchIDLabel, batchID) err := c.Patch(context.TODO(), podClone, client.RawPatch(types.StrategicMergePatchType, []byte(by))) if err != nil { klog.Errorf("Failed to patch Pod(%v) batchID, err: %v", client.ObjectKeyFromObject(pod), err) diff --git a/pkg/controller/batchrelease/workloads/workload_rollout_control_plane.go b/pkg/controller/batchrelease/workloads/workload_rollout_control_plane.go index d2d50d2a..efbc25b0 100644 --- a/pkg/controller/batchrelease/workloads/workload_rollout_control_plane.go +++ b/pkg/controller/batchrelease/workloads/workload_rollout_control_plane.go @@ -75,7 +75,7 @@ func (c *UnifiedWorkloadRolloutControlPlane) VerifyWorkload() (bool, error) { // - *int32: how many pods have been patched; // - err: whether error occurs. func (c *UnifiedWorkloadRolloutControlPlane) prepareBeforeRollback() (bool, *int32, error) { - if c.release.Annotations[util.RollbackInBatchAnnotation] == "" { + if c.release.Annotations[v1alpha1.RollbackInBatchAnnotation] == "" { return true, nil, nil } diff --git a/pkg/controller/rollout/batchrelease/batchrelease.go b/pkg/controller/rollout/batchrelease/batchrelease.go deleted file mode 100644 index 1b0f3e7d..00000000 --- a/pkg/controller/rollout/batchrelease/batchrelease.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package batchrelease - -import rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - -// BatchRelease is not the actual controller of the BatchRelease controller, -// but rather the ability to interact with the BatchRelease controller through the BatchRelease CRD to achieve a batch release -type BatchRelease interface { - // Verify will create batchRelease or update batchRelease steps configuration and - // return whether the batchRelease configuration is consistent with the rollout step - Verify(index int32) (bool, error) - - // SyncRolloutID will sync rollout id from Rollout to BatchRelease - SyncRolloutID(currentID string) error - - // 1. Promote release workload in step(index), 1<=index<=len(step) - // 2. Promote will resume stable workload if the last batch(index=-1) is finished - Promote(index int32, isRollback, checkReady bool) (bool, error) - - // FetchBatchRelease fetch batchRelease - FetchBatchRelease() (*rolloutv1alpha1.BatchRelease, error) - - // Finalize clean up batchRelease - // 1. delete canary deployments - // 2. delete batchRelease CRD - Finalize() (bool, error) -} diff --git a/pkg/controller/rollout/batchrelease/inner_batchrelease.go b/pkg/controller/rollout/batchrelease/inner_batchrelease.go deleted file mode 100644 index f30b3740..00000000 --- a/pkg/controller/rollout/batchrelease/inner_batchrelease.go +++ /dev/null @@ -1,400 +0,0 @@ -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package batchrelease - -import ( - "context" - "fmt" - "reflect" - "strconv" - - appsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/util" - apps "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/util/retry" - "k8s.io/klog/v2" - utilpointer "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - // rollouts.kruise.io - BatchReleaseOwnerRefLabel = "rollouts.kruise.io/owner-ref" -) - -type innerBatchRelease struct { - client.Client - - rollout *rolloutv1alpha1.Rollout - - batchName string - rolloutID string -} - -func NewInnerBatchController(c client.Client, rollout *rolloutv1alpha1.Rollout, rolloutID string) BatchRelease { - r := &innerBatchRelease{ - Client: c, - rollout: rollout, - rolloutID: rolloutID, - batchName: rolloutBatchName(rollout), - } - - return r -} - -func (r *innerBatchRelease) SyncRolloutID(currentID string) error { - if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - batch := &rolloutv1alpha1.BatchRelease{} - if err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch); err != nil { - if errors.IsNotFound(err) { - return nil // just return nil if batchRelease not exist - } - return err - } - if batch.Spec.ReleasePlan.RolloutID == currentID { - return nil - } - batch.Spec.ReleasePlan.RolloutID = r.rolloutID - if err := r.Client.Update(context.TODO(), batch); err != nil { - return err - } - return nil - }); err != nil { - klog.Errorf("rollout(%s/%s) update batchRelease rolloutID %s failed: %s", r.rollout.Namespace, r.rollout.Name, currentID, err.Error()) - return err - } - return nil -} - -func (r *innerBatchRelease) Verify(index int32) (bool, error) { - index = index - 1 - batch := &rolloutv1alpha1.BatchRelease{} - err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch) - if errors.IsNotFound(err) { - // create new BatchRelease Crd - br := createBatchRelease(r.rollout, r.batchName, r.rolloutID) - if err = r.Create(context.TODO(), br); err != nil && !errors.IsAlreadyExists(err) { - klog.Errorf("rollout(%s/%s) create BatchRelease failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return false, err - } - data := util.DumpJSON(br) - klog.Infof("rollout(%s/%s) create BatchRelease(%s) success", r.rollout.Namespace, r.rollout.Name, data) - return false, nil - } else if err != nil { - klog.Errorf("rollout(%s/%s) fetch BatchRelease failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return false, err - } - - // check whether batchRelease configuration is the latest - newBr := createBatchRelease(r.rollout, r.batchName, r.rolloutID) - if batchPlanDeepEqual(batch, newBr, index) { - klog.Infof("rollout(%s/%s) batchRelease(generation:%d) configuration is the latest", r.rollout.Namespace, r.rollout.Name, batch.Generation) - return true, nil - } - - // update batchRelease to the latest version - if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if err = r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch); err != nil { - klog.Errorf("error getting updated BatchRelease(%s/%s) from client", batch.Namespace, batch.Name) - return err - } - batch.Spec.ReleasePlan.RolloutID = r.rolloutID - batch.Spec.ReleasePlan.Batches = newBr.Spec.ReleasePlan.Batches - batch.Spec.ReleasePlan.BatchPartition = utilpointer.Int32Ptr(index) - if err = r.Client.Update(context.TODO(), batch); err != nil { - return err - } - return nil - }); err != nil { - klog.Errorf("rollout(%s/%s) update batchRelease configuration failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return false, err - } - data := util.DumpJSON(batch) - klog.Infof("rollout(%s/%s) update batchRelease configuration(%s) to the latest", r.rollout.Namespace, r.rollout.Name, data) - return false, nil -} - -func (r *innerBatchRelease) FetchBatchRelease() (*rolloutv1alpha1.BatchRelease, error) { - batch := &rolloutv1alpha1.BatchRelease{} - err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch) - if err != nil { - klog.Errorf("rollout(%s/%s) fetch BatchRelease failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return nil, err - } - return batch, nil -} - -func (r *innerBatchRelease) Promote(index int32, isRollback, checkReady bool) (bool, error) { - // Promote will resume stable workload if the last batch(index=-1) is finished - if index == -1 { - return r.resumeStableWorkload(checkReady) - } - - // batch release workload's pods - index = index - 1 - batch := &rolloutv1alpha1.BatchRelease{} - if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch); err != nil { - klog.Errorf("error getting updated BatchRelease(%s/%s) from client", batch.Namespace, batch.Name) - return err - } - if IsPromoted(r.rollout, batch, isRollback) { - return nil - } - if isRollback && len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { - if batch.Annotations == nil { - batch.Annotations = map[string]string{} - } - // only rollback case should update this rollout id for BatchRelease. - batch.Spec.ReleasePlan.RolloutID = r.rolloutID - batch.Annotations[util.RollbackInBatchAnnotation] = r.rollout.Annotations[util.RollbackInBatchAnnotation] - } - - batch.Spec.Paused = false - if batch.Labels == nil { - batch.Labels = map[string]string{} - } - batch.Spec.ReleasePlan.BatchPartition = utilpointer.Int32Ptr(index) - if err := r.Client.Update(context.TODO(), batch); err != nil { - return err - } - klog.Infof("rollout(%s/%s) promote batchRelease BatchPartition(%d) success", r.rollout.Namespace, r.rollout.Name, index) - return nil - }); err != nil { - klog.Errorf("rollout(%s/%s) promote batchRelease BatchPartition(%d) failed: %s", r.rollout.Namespace, r.rollout.Name, index, err.Error()) - return false, err - } - return false, nil -} - -func (r *innerBatchRelease) resumeStableWorkload(checkReady bool) (bool, error) { - // cloneSet - switch r.rollout.Spec.ObjectRef.WorkloadRef.Kind { - case util.ControllerKruiseKindCS.Kind: - dName := r.rollout.Spec.ObjectRef.WorkloadRef.Name - obj := &appsv1alpha1.CloneSet{} - err := r.Get(context.TODO(), types.NamespacedName{Namespace: r.rollout.Namespace, Name: dName}, obj) - if err != nil { - if errors.IsNotFound(err) { - klog.Warningf("rollout(%s/%s) cloneSet(%s) not found, and return true", r.rollout.Namespace, r.rollout.Name, dName) - return true, nil - } - return false, err - } - // default partition.IntVal=0 - if !obj.Spec.UpdateStrategy.Paused && obj.Spec.UpdateStrategy.Partition.IntVal == 0 && obj.Spec.UpdateStrategy.Partition.Type == intstr.Int { - return true, nil - } - - err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if err = r.Get(context.TODO(), types.NamespacedName{Namespace: r.rollout.Namespace, Name: dName}, obj); err != nil { - return err - } - obj.Spec.UpdateStrategy.Paused = false - obj.Spec.UpdateStrategy.Partition = nil - return r.Update(context.TODO(), obj) - }) - if err != nil { - klog.Errorf("update rollout(%s/%s) cloneSet failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return false, err - } - klog.Infof("resume rollout(%s/%s) cloneSet(paused=false,partition=nil) success", r.rollout.Namespace, r.rollout.Name) - return true, nil - - case util.ControllerKindDep.Kind: - // deployment - dName := r.rollout.Spec.ObjectRef.WorkloadRef.Name - obj := &apps.Deployment{} - err := r.Get(context.TODO(), types.NamespacedName{Namespace: r.rollout.Namespace, Name: dName}, obj) - if err != nil { - if errors.IsNotFound(err) { - klog.Warningf("rollout(%s/%s) stable deployment(%s) not found, and return true", r.rollout.Namespace, r.rollout.Name, dName) - return true, nil - } - return false, err - } - // set deployment paused=false - if obj.Spec.Paused { - err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if err = r.Get(context.TODO(), types.NamespacedName{Namespace: r.rollout.Namespace, Name: dName}, obj); err != nil { - return err - } - obj.Spec.Paused = false - return r.Update(context.TODO(), obj) - }) - if err != nil { - klog.Errorf("update rollout(%s/%s) stable deployment failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return false, err - } - klog.Infof("resume rollout(%s/%s) stable deployment(paused=false) success", r.rollout.Namespace, r.rollout.Name) - } - - // Whether to wait for pods are ready - if !checkReady { - return true, nil - } - data := util.DumpJSON(obj.Status) - // wait for all pods are ready - maxUnavailable, _ := intstr.GetScaledValueFromIntOrPercent(obj.Spec.Strategy.RollingUpdate.MaxUnavailable, int(*obj.Spec.Replicas), true) - if obj.Status.ObservedGeneration != obj.Generation || obj.Status.UpdatedReplicas != *obj.Spec.Replicas || - obj.Status.Replicas != *obj.Spec.Replicas || *obj.Spec.Replicas-obj.Status.AvailableReplicas > int32(maxUnavailable) { - klog.Infof("rollout(%s/%s) stable deployment status(%s), and wait a moment", r.rollout.Namespace, r.rollout.Name, data) - return false, nil - } - klog.Infof("resume rollout(%s/%s) stable deployment(paused=false) status(%s) success", r.rollout.Namespace, r.rollout.Name, data) - return true, nil - - default: - // statefulset-like workloads - workloadRef := r.rollout.Spec.ObjectRef.WorkloadRef - workloadNsn := types.NamespacedName{Namespace: r.rollout.Namespace, Name: workloadRef.Name} - workloadGVK := schema.FromAPIVersionAndKind(workloadRef.APIVersion, workloadRef.Kind) - obj := &unstructured.Unstructured{} - obj.SetGroupVersionKind(workloadGVK) - err := r.Get(context.TODO(), workloadNsn, obj) - if err != nil { - if errors.IsNotFound(err) { - klog.Warningf("rollout(%s/%s) statefulset(%s) not found, and return true", r.rollout.Namespace, r.rollout.Name, workloadNsn.Name) - return true, nil - } - return false, err - } - - if util.GetStatefulSetPartition(obj) == 0 { - return true, nil - } - - cloneObj := obj.DeepCopy() - body := fmt.Sprintf(`{"spec":{"updateStrategy":{"rollingUpdate":{"partition":0}}}}`) - err = r.Patch(context.TODO(), cloneObj, client.RawPatch(types.MergePatchType, []byte(body))) - if err != nil { - klog.Errorf("patch rollout(%s/%s) statefulset failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return false, err - } - klog.Infof("resume rollout(%s/%s) statefulset(partition=0) success", r.rollout.Namespace, r.rollout.Name) - return true, nil - } -} - -func (r *innerBatchRelease) Finalize() (bool, error) { - batch := &rolloutv1alpha1.BatchRelease{} - err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch) - if err != nil && errors.IsNotFound(err) { - klog.Infof("rollout(%s/%s) delete BatchRelease success", r.rollout.Namespace, r.rollout.Name) - return true, nil - } else if err != nil { - klog.Errorf("rollout(%s/%s) fetch BatchRelease failed: %s", r.rollout.Namespace, r.rollout.Name) - return false, err - } - if !batch.DeletionTimestamp.IsZero() { - klog.Infof("rollout(%s/%s) BatchRelease is terminating, and wait a moment", r.rollout.Namespace, r.rollout.Name) - return false, nil - } - - //delete batchRelease - err = r.Delete(context.TODO(), batch) - if err != nil { - klog.Errorf("rollout(%s/%s) delete BatchRelease failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return false, err - } - klog.Infof("rollout(%s/%s) delete BatchRelease, and wait a moment", r.rollout.Namespace, r.rollout.Name) - return false, nil -} - -func createBatchRelease(rollout *rolloutv1alpha1.Rollout, batchName, rolloutID string) *rolloutv1alpha1.BatchRelease { - var batches []rolloutv1alpha1.ReleaseBatch - for _, step := range rollout.Spec.Strategy.Canary.Steps { - if step.Replicas == nil { - batches = append(batches, rolloutv1alpha1.ReleaseBatch{CanaryReplicas: intstr.FromString(strconv.Itoa(int(*step.Weight)) + "%")}) - } else { - batches = append(batches, rolloutv1alpha1.ReleaseBatch{CanaryReplicas: *step.Replicas}) - } - } - - br := &rolloutv1alpha1.BatchRelease{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: rollout.Namespace, - Name: batchName, - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(rollout, schema.GroupVersionKind{ - Group: rolloutv1alpha1.SchemeGroupVersion.Group, - Version: rolloutv1alpha1.SchemeGroupVersion.Version, - Kind: "Rollout", - }), - }, - Labels: map[string]string{ - BatchReleaseOwnerRefLabel: rollout.Name, - }, - }, - Spec: rolloutv1alpha1.BatchReleaseSpec{ - TargetRef: rolloutv1alpha1.ObjectRef{ - WorkloadRef: &rolloutv1alpha1.WorkloadRef{ - APIVersion: rollout.Spec.ObjectRef.WorkloadRef.APIVersion, - Kind: rollout.Spec.ObjectRef.WorkloadRef.Kind, - Name: rollout.Spec.ObjectRef.WorkloadRef.Name, - }, - }, - ReleasePlan: rolloutv1alpha1.ReleasePlan{ - Batches: batches, - RolloutID: rolloutID, - BatchPartition: utilpointer.Int32Ptr(0), - }, - }, - } - return br -} - -// {workload.name}-batch -func rolloutBatchName(rollout *rolloutv1alpha1.Rollout) string { - return rollout.Name -} - -// IsPromoted return true if the current batch has been promoted: -// - 1. BatchRelease BatchPartition == Rollout currentStepIndex-1; -// - 2. Rollback annotation has been patched to BatchRelease when rolling back. -func IsPromoted(rollout *rolloutv1alpha1.Rollout, batch *rolloutv1alpha1.BatchRelease, isRollback bool) bool { - currentBatch := int32(0) - if rollout.Status.CanaryStatus != nil { - currentBatch = rollout.Status.CanaryStatus.CurrentStepIndex - 1 - } - - if batch.Spec.ReleasePlan.BatchPartition == nil || *batch.Spec.ReleasePlan.BatchPartition != currentBatch { - return false - } - - if isRollback && batch.Annotations[util.RollbackInBatchAnnotation] != rollout.Annotations[util.RollbackInBatchAnnotation] { - return false - } - return true -} - -func batchPlanDeepEqual(old, new *rolloutv1alpha1.BatchRelease, currentBatch int32) bool { - if old.Spec.ReleasePlan.BatchPartition == nil || *old.Spec.ReleasePlan.BatchPartition != currentBatch { - return false - } - if old.Spec.ReleasePlan.RolloutID != new.Spec.ReleasePlan.RolloutID { - return false - } - return reflect.DeepEqual(old.Spec.ReleasePlan.Batches, new.Spec.ReleasePlan.Batches) -} diff --git a/pkg/controller/rollout/canary.go b/pkg/controller/rollout/canary.go deleted file mode 100644 index 802d30d6..00000000 --- a/pkg/controller/rollout/canary.go +++ /dev/null @@ -1,303 +0,0 @@ -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rollout - -import ( - "context" - "fmt" - "time" - - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/controller/rollout/batchrelease" - "github.com/openkruise/rollouts/pkg/util" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" - "k8s.io/klog/v2" -) - -func (r *rolloutContext) runCanary() error { - canaryStatus := r.newStatus.CanaryStatus - // init canary status - if canaryStatus.CanaryRevision == "" { - canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateUpgrade - canaryStatus.CanaryRevision = r.workload.CanaryRevision - canaryStatus.ObservedRolloutID = getRolloutID(r.workload, r.rollout) - canaryStatus.CurrentStepIndex = 1 - canaryStatus.RolloutHash = r.rollout.Annotations[util.RolloutHashAnnotation] - } - - // update canary status - batch, err := r.batchControl.FetchBatchRelease() - if err != nil { - canaryStatus.Message = "BatchRelease not found" - canaryStatus.CanaryReplicas = r.workload.CanaryReplicas - canaryStatus.CanaryReadyReplicas = r.workload.CanaryReadyReplicas - } else { - canaryStatus.Message = fmt.Sprintf("BatchRelease at state %s, id %s, step %d", - batch.Status.CanaryStatus.CurrentBatchState, batch.Status.ObservedRolloutID, batch.Status.CanaryStatus.CurrentBatch+1) - canaryStatus.CanaryReplicas = batch.Status.CanaryStatus.UpdatedReplicas - canaryStatus.CanaryReadyReplicas = batch.Status.CanaryStatus.UpdatedReadyReplicas - } - - // sync rollout-id to batchRelease if we need - if err := r.batchControl.SyncRolloutID(r.newStatus.CanaryStatus.ObservedRolloutID); err != nil { - return err - } - - switch canaryStatus.CurrentStepState { - case rolloutv1alpha1.CanaryStepStateUpgrade: - klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, rolloutv1alpha1.CanaryStepStateUpgrade) - done, err := r.doCanaryUpgrade() - if err != nil { - return err - } else if done { - canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateTrafficRouting - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name, - canaryStatus.CurrentStepIndex, rolloutv1alpha1.CanaryStepStateUpgrade, canaryStatus.CurrentStepState) - } - - case rolloutv1alpha1.CanaryStepStateTrafficRouting: - klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, rolloutv1alpha1.CanaryStepStateTrafficRouting) - done, err := r.doCanaryTrafficRouting() - if err != nil { - return err - } else if done { - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateMetricsAnalysis - klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name, - canaryStatus.CurrentStepIndex, rolloutv1alpha1.CanaryStepStateTrafficRouting, canaryStatus.CurrentStepState) - } - expectedTime := time.Now().Add(time.Duration(defaultGracePeriodSeconds) * time.Second) - r.recheckTime = &expectedTime - - case rolloutv1alpha1.CanaryStepStateMetricsAnalysis: - klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, rolloutv1alpha1.CanaryStepStateMetricsAnalysis) - done, err := r.doCanaryMetricsAnalysis() - if err != nil { - return err - } else if done { - canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStatePaused - klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name, - canaryStatus.CurrentStepIndex, rolloutv1alpha1.CanaryStepStateMetricsAnalysis, canaryStatus.CurrentStepState) - } - - case rolloutv1alpha1.CanaryStepStatePaused: - klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, rolloutv1alpha1.CanaryStepStatePaused) - done, err := r.doCanaryPaused() - if err != nil { - return err - } else if done { - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateReady - klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name, - canaryStatus.CurrentStepIndex, rolloutv1alpha1.CanaryStepStatePaused, canaryStatus.CurrentStepState) - } - - case rolloutv1alpha1.CanaryStepStateReady: - klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, rolloutv1alpha1.CanaryStepStateReady) - // run next step - if len(r.rollout.Spec.Strategy.Canary.Steps) > int(canaryStatus.CurrentStepIndex) { - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - canaryStatus.CurrentStepIndex++ - canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateUpgrade - klog.Infof("rollout(%s/%s) canary step from(%d) -> to(%d)", r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex-1, canaryStatus.CurrentStepIndex) - } else { - klog.Infof("rollout(%s/%s) canary run all steps, and completed", r.rollout.Namespace, r.rollout.Name) - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateCompleted - } - klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name, - canaryStatus.CurrentStepIndex, rolloutv1alpha1.CanaryStepStateReady, canaryStatus.CurrentStepState) - // canary completed - case rolloutv1alpha1.CanaryStepStateCompleted: - klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, rolloutv1alpha1.CanaryStepStateCompleted) - } - - return nil -} - -func (r *rolloutContext) doCanaryUpgrade() (bool, error) { - // only traffic routing - /*if len(r.rollout.Spec.Strategy.Canary.Steps) == 0 { - if r.workload.CanaryReadyReplicas > 0 { - klog.Infof("rollout(%s/%s) workload(%s) canaryAvailable(%d), and go to the next stage", - r.rollout.Namespace, r.rollout.Name, r.workload.Name, r.workload.CanaryReadyReplicas) - return true, nil - } - klog.Infof("rollout(%s/%s) workload(%s) canaryAvailable(%d), and wait a moment", - r.rollout.Namespace, r.rollout.Name, r.workload.Name, r.workload.CanaryReadyReplicas) - return false, nil - }*/ - - // verify whether batchRelease configuration is the latest - steps := len(r.rollout.Spec.Strategy.Canary.Steps) - canaryStatus := r.newStatus.CanaryStatus - isLatest, err := r.batchControl.Verify(canaryStatus.CurrentStepIndex) - if err != nil { - return false, err - } else if !isLatest { - return false, nil - } - - // fetch batchRelease - batch, err := r.batchControl.FetchBatchRelease() - if err != nil { - return false, err - } else if batch.Status.ObservedReleasePlanHash != util.HashReleasePlanBatches(&batch.Spec.ReleasePlan) || - batch.Generation != batch.Status.ObservedGeneration { - klog.Infof("rollout(%s/%s) batchReleasePlan is not consistent, and wait a moment", r.rollout.Namespace, r.rollout.Name) - return false, nil - } - batchData := util.DumpJSON(batch.Status) - cond := util.GetRolloutCondition(*r.newStatus, rolloutv1alpha1.RolloutConditionProgressing) - cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and upgrade workload new versions", canaryStatus.CurrentStepIndex, steps) - r.newStatus.Message = cond.Message - // promote workload next batch release - if !batchrelease.IsPromoted(r.rollout, batch, r.workload.IsInRollback) { - r.recorder.Eventf(r.rollout, corev1.EventTypeNormal, "Progressing", fmt.Sprintf("start upgrade step(%d) canary pods with new versions", canaryStatus.CurrentStepIndex)) - klog.Infof("rollout(%s/%s) will promote batch from(%d) -> to(%d)", r.rollout.Namespace, r.rollout.Name, *batch.Spec.ReleasePlan.BatchPartition+1, canaryStatus.CurrentStepIndex) - return r.batchControl.Promote(canaryStatus.CurrentStepIndex, r.workload.IsInRollback, false) - } - - // check whether batchRelease is ready - if batch.Status.CanaryStatus.CurrentBatchState != rolloutv1alpha1.ReadyBatchState || - batch.Status.CanaryStatus.CurrentBatch+1 < canaryStatus.CurrentStepIndex { - klog.Infof("rollout(%s/%s) batch(%s) state(%s), and wait a moment", - r.rollout.Namespace, r.rollout.Name, batchData, batch.Status.CanaryStatus.CurrentBatchState) - return false, nil - } - r.recorder.Eventf(r.rollout, corev1.EventTypeNormal, "Progressing", fmt.Sprintf("upgrade step(%d) canary pods with new versions done", canaryStatus.CurrentStepIndex)) - klog.Infof("rollout(%s/%s) batch(%s) state(%s), and success", - r.rollout.Namespace, r.rollout.Name, batchData, batch.Status.CanaryStatus.CurrentBatchState) - return true, nil -} - -func (r *rolloutContext) doCanaryMetricsAnalysis() (bool, error) { - // todo - return true, nil -} - -func (r *rolloutContext) doCanaryPaused() (bool, error) { - // No step set, need manual confirmation - if len(r.rollout.Spec.Strategy.Canary.Steps) == 0 { - klog.Infof("rollout(%s/%s) don't contains steps, and need manual confirmation", r.rollout.Namespace, r.rollout.Name) - return false, nil - } - canaryStatus := r.newStatus.CanaryStatus - currentStep := r.rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] - steps := len(r.rollout.Spec.Strategy.Canary.Steps) - cond := util.GetRolloutCondition(*r.newStatus, rolloutv1alpha1.RolloutConditionProgressing) - // need manual confirmation - if currentStep.Pause.Duration == nil { - klog.Infof("rollout(%s/%s) don't set pause duration, and need manual confirmation", r.rollout.Namespace, r.rollout.Name) - cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and you need manually confirm to enter the next step", canaryStatus.CurrentStepIndex, steps) - r.newStatus.Message = cond.Message - return false, nil - } - cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and wait duration(%d seconds) to enter the next step", canaryStatus.CurrentStepIndex, steps, *currentStep.Pause.Duration) - r.newStatus.Message = cond.Message - // wait duration time, then go to next step - duration := time.Second * time.Duration(*currentStep.Pause.Duration) - expectedTime := canaryStatus.LastUpdateTime.Add(duration) - if expectedTime.Before(time.Now()) { - klog.Infof("rollout(%s/%s) canary step(%d) paused duration(%d seconds), and go to the next step", - r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex, *currentStep.Pause.Duration) - return true, nil - } - r.recheckTime = &expectedTime - return false, nil -} - -// cleanup after rollout is completed or finished -func (r *rolloutContext) doCanaryFinalising() (bool, error) { - // when CanaryStatus is nil, which means canary action hasn't started yet, don't need doing cleanup - if r.newStatus.CanaryStatus == nil { - return true, nil - } - klog.Infof("rollout(%s/%s) in finalizing: remove rollout state in workload", r.rollout.Namespace, r.rollout.Name) - // 1. rollout progressing complete, allow workload paused=false in webhook - err := r.removeRolloutStateInWorkload() - if err != nil { - return false, err - } - klog.Infof("rollout(%s/%s) in finalizing: restore stable service", r.rollout.Namespace, r.rollout.Name) - // 2. restore stable service, remove podRevision selector - done, err := r.restoreStableService() - if err != nil || !done { - return done, err - } - // 3. upgrade stable deployment, set paused=false - // isComplete indicates whether rollout progressing complete, and wait for all pods are ready - // else indicates rollout is canceled - klog.Infof("rollout(%s/%s) in finalizing: upgrade stable workload", r.rollout.Namespace, r.rollout.Name) - done, err = r.batchControl.Promote(-1, false, r.isComplete) - if err != nil || !done { - return done, err - } - // 4. route all traffic to stable service - klog.Infof("rollout(%s/%s) in finalizing: restore traffic routing", r.rollout.Namespace, r.rollout.Name) - done, err = r.doFinalisingTrafficRouting() - if err != nil || !done { - return done, err - } - // 5. delete batchRelease crd - klog.Infof("rollout(%s/%s) in finalizing: remove batchRelease crd", r.rollout.Namespace, r.rollout.Name) - done, err = r.batchControl.Finalize() - if err != nil { - klog.Errorf("rollout(%s/%s) Finalize batchRelease failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return false, err - } else if !done { - return false, nil - } - klog.Infof("rollout(%s/%s) do finalize success", r.rollout.Namespace, r.rollout.Name) - return true, nil -} - -func (r *rolloutContext) removeRolloutStateInWorkload() error { - if r.workload == nil || r.rollout.Spec.ObjectRef.WorkloadRef == nil { - return nil - } - if _, ok := r.workload.Annotations[util.InRolloutProgressingAnnotation]; !ok { - return nil - } - workloadRef := r.rollout.Spec.ObjectRef.WorkloadRef - workloadGVK := schema.FromAPIVersionAndKind(workloadRef.APIVersion, workloadRef.Kind) - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - obj := util.GetEmptyWorkloadObject(workloadGVK) - if obj == nil { - return nil - } - if err := r.Get(context.TODO(), types.NamespacedName{Name: r.workload.Name, Namespace: r.workload.Namespace}, obj); err != nil { - klog.Errorf("getting updated workload(%s.%s) failed: %s", r.workload.Namespace, r.workload.Name, err.Error()) - return err - } - annotations := obj.GetAnnotations() - delete(annotations, util.InRolloutProgressingAnnotation) - obj.SetAnnotations(annotations) - return r.Update(context.TODO(), obj) - }) - if err != nil { - klog.Errorf("update rollout(%s/%s) workload(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.workload.Name, err.Error()) - return err - } - klog.Infof("remove rollout(%s/%s) workload(%s) annotation[%s] success", r.rollout.Namespace, r.rollout.Name, r.workload.Name, util.InRolloutProgressingAnnotation) - return nil -} diff --git a/pkg/controller/rollout/context.go b/pkg/controller/rollout/context.go deleted file mode 100644 index d8342056..00000000 --- a/pkg/controller/rollout/context.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rollout - -import ( - "fmt" - "time" - - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/controller/rollout/batchrelease" - "github.com/openkruise/rollouts/pkg/util" - "k8s.io/client-go/tools/record" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type rolloutContext struct { - client.Client - - rollout *rolloutv1alpha1.Rollout - - newStatus *rolloutv1alpha1.RolloutStatus - - isComplete bool - - stableService string - - canaryService string - - workload *util.Workload - - batchControl batchrelease.BatchRelease - - recheckTime *time.Time - - recorder record.EventRecorder -} - -func newRolloutContext(client client.Client, recorder record.EventRecorder, rollout *rolloutv1alpha1.Rollout, newStatus *rolloutv1alpha1.RolloutStatus, workload *util.Workload) *rolloutContext { - rolloutCon := &rolloutContext{ - Client: client, - rollout: rollout, - newStatus: newStatus, - batchControl: batchrelease.NewInnerBatchController(client, rollout, getRolloutID(workload, rollout)), - workload: workload, - recorder: recorder, - } - if len(rolloutCon.rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { - rolloutCon.stableService = rolloutCon.rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service - rolloutCon.canaryService = fmt.Sprintf("%s-canary", rolloutCon.stableService) - } - return rolloutCon -} - -func (r *rolloutContext) reconcile() error { - // canary strategy - if r.rollout.Spec.Strategy.Canary != nil { - klog.Infof("rollout(%s/%s) run Canary action...", r.rollout.Namespace, r.rollout.Name) - return r.runCanary() - } - return nil -} - -func (r *rolloutContext) finalising() (bool, error) { - // canary strategy - if r.rollout.Spec.Strategy.Canary != nil { - done, err := r.doCanaryFinalising() - if err == nil && !done { - // The finalizer is not finished, wait one second - expectedTime := time.Now().Add(time.Duration(defaultGracePeriodSeconds) * time.Second) - r.recheckTime = &expectedTime - } - return done, err - } - return false, nil -} - -func (r *rolloutContext) podRevisionLabelKey() string { - if r.workload == nil { - return "" - } - return r.workload.RevisionLabelKey -} - -func getRolloutID(workload *util.Workload, rollout *rolloutv1alpha1.Rollout) string { - if workload != nil { - firstChoice := workload.Labels[util.RolloutIDLabel] - if firstChoice != "" { - return firstChoice - } - } - if rollout != nil { - return rollout.Spec.RolloutID - } - return "" -} diff --git a/pkg/controller/rollout/finalizer.go b/pkg/controller/rollout/finalizer.go deleted file mode 100644 index df0eb818..00000000 --- a/pkg/controller/rollout/finalizer.go +++ /dev/null @@ -1,105 +0,0 @@ -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rollout - -import ( - "time" - - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/util" - corev1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" -) - -func (r *RolloutReconciler) reconcileRolloutTerminating(rollout *rolloutv1alpha1.Rollout) (*time.Time, error) { - cond := util.GetRolloutCondition(rollout.Status, rolloutv1alpha1.RolloutConditionTerminating) - if cond.Reason == rolloutv1alpha1.TerminatingReasonCompleted { - return nil, nil - } - newStatus := rollout.Status.DeepCopy() - done, recheckTime, err := r.doFinalising(rollout, newStatus, false) - if err != nil { - return nil, err - } else if done { - klog.Infof("rollout(%s/%s) is terminating, and state from(%s) -> to(%s)", rollout.Namespace, rollout.Name, cond.Reason, rolloutv1alpha1.TerminatingReasonCompleted) - cond.Reason = rolloutv1alpha1.TerminatingReasonCompleted - cond.Status = corev1.ConditionTrue - util.SetRolloutCondition(newStatus, *cond) - } - err = r.updateRolloutStatusInternal(rollout, *newStatus) - if err != nil { - klog.Errorf("update rollout(%s/%s) status failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return nil, err - } - return recheckTime, nil -} - -func (r *RolloutReconciler) doFinalising(rollout *rolloutv1alpha1.Rollout, newStatus *rolloutv1alpha1.RolloutStatus, isComplete bool) (bool, *time.Time, error) { - klog.Infof("reconcile rollout(%s/%s) doFinalising", rollout.Namespace, rollout.Name) - // fetch target workload - workload, err := r.Finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) - if err != nil { - klog.Errorf("rollout(%s/%s) GetWorkloadForRef failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return false, nil, err - } - rolloutCon := newRolloutContext(r.Client, r.Recorder, rollout, newStatus, workload) - rolloutCon.isComplete = isComplete - done, err := rolloutCon.finalising() - if err != nil { - klog.Errorf("rollout(%s/%s) Progressing failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return false, nil, err - } else if !done { - klog.Infof("rollout(%s/%s) finalizer is not finished, and retry reconcile", rollout.Namespace, rollout.Name) - return false, rolloutCon.recheckTime, nil - } - //newStatus.CanaryStatus = nil - klog.Infof("run rollout(%s/%s) Progressing Finalising done", rollout.Namespace, rollout.Name) - return true, nil, nil -} - -// handle adding and handle finalizer logic, it turns if we should continue to reconcile -func (r *RolloutReconciler) handleFinalizer(rollout *rolloutv1alpha1.Rollout) error { - // delete rollout crd, remove finalizer - if !rollout.DeletionTimestamp.IsZero() { - cond := util.GetRolloutCondition(rollout.Status, rolloutv1alpha1.RolloutConditionTerminating) - if cond != nil && cond.Reason == rolloutv1alpha1.TerminatingReasonCompleted { - // Completed - if controllerutil.ContainsFinalizer(rollout, util.KruiseRolloutFinalizer) { - err := util.UpdateFinalizer(r.Client, rollout, util.RemoveFinalizerOpType, util.KruiseRolloutFinalizer) - if err != nil { - klog.Errorf("remove rollout(%s/%s) finalizer failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return err - } - klog.Infof("remove rollout(%s/%s) finalizer success", rollout.Namespace, rollout.Name) - } - return nil - } - return nil - } - - // create rollout crd, add finalizer - if !controllerutil.ContainsFinalizer(rollout, util.KruiseRolloutFinalizer) { - err := util.UpdateFinalizer(r.Client, rollout, util.AddFinalizerOpType, util.KruiseRolloutFinalizer) - if err != nil { - klog.Errorf("register rollout(%s/%s) finalizer failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return err - } - klog.Infof("register rollout(%s/%s) finalizer success", rollout.Namespace, rollout.Name) - } - return nil -} diff --git a/pkg/controller/rollout/progressing.go b/pkg/controller/rollout/progressing.go deleted file mode 100644 index 2e5de267..00000000 --- a/pkg/controller/rollout/progressing.go +++ /dev/null @@ -1,248 +0,0 @@ -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rollout - -import ( - "context" - "fmt" - "strconv" - "time" - - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/controller/rollout/batchrelease" - "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting" - "github.com/openkruise/rollouts/pkg/util" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/klog/v2" -) - -var defaultGracePeriodSeconds int32 = 3 - -// parameter1 retryReconcile, parameter2 error -func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *rolloutv1alpha1.Rollout) (*time.Time, error) { - cond := util.GetRolloutCondition(rollout.Status, rolloutv1alpha1.RolloutConditionProgressing) - klog.Infof("reconcile rollout(%s/%s) progressing action", rollout.Namespace, rollout.Name) - workload, err := r.Finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) - if err != nil { - klog.Errorf("rollout(%s/%s) get workload failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return nil, err - } else if workload == nil { - klog.Errorf("rollout(%s/%s) workload Not Found", rollout.Namespace, rollout.Name) - return nil, nil - } else if !workload.IsStatusConsistent { - klog.Infof("rollout(%s/%s) workload status isn't consistent, then wait a moment", rollout.Namespace, rollout.Name) - return nil, nil - } - - var recheckTime *time.Time - newStatus := rollout.Status.DeepCopy() - switch cond.Reason { - case rolloutv1alpha1.ProgressingReasonInitializing: - klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) - // new canaryStatus - newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{} - done, _, err := r.doProgressingInitializing(rollout, newStatus) - if err != nil { - klog.Errorf("rollout(%s/%s) doProgressingInitializing error(%s)", rollout.Namespace, rollout.Name, err.Error()) - return nil, err - } else if done { - progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonInRolling, "Rollout is in Progressing") - } else { - // Incomplete, recheck - expectedTime := time.Now().Add(time.Duration(defaultGracePeriodSeconds) * time.Second) - recheckTime = &expectedTime - klog.Infof("rollout(%s/%s) doProgressingInitializing is incomplete, and recheck(%s)", rollout.Namespace, rollout.Name, expectedTime.String()) - } - - case rolloutv1alpha1.ProgressingReasonInRolling: - klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) - recheckTime, err = r.doProgressingInRolling(rollout, workload, newStatus) - if err != nil { - return nil, err - } - - case rolloutv1alpha1.ProgressingReasonFinalising: - klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) - var done bool - done, recheckTime, err = r.doFinalising(rollout, newStatus, true) - if err != nil { - return nil, err - // finalizer is finished - } else if done { - progressingStateTransition(newStatus, corev1.ConditionTrue, rolloutv1alpha1.ProgressingReasonSucceeded, "Rollout has been completed, and succeed") - } - - case rolloutv1alpha1.ProgressingReasonPaused: - if workload.IsInRollback { - newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision - r.Recorder.Eventf(rollout, corev1.EventTypeNormal, "Progressing", "workload has been rollback, then rollout is canceled") - klog.Infof("rollout(%s/%s) workload has been rollback, then rollout canceled", rollout.Namespace, rollout.Name) - progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonCancelling, "The workload has been rolled back and the rollout process will be cancelled") - // from paused to inRolling - } else if !rollout.Spec.Strategy.Paused { - klog.Infof("rollout(%s/%s) is Progressing, but paused", rollout.Namespace, rollout.Name) - progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonInRolling, "") - } - - case rolloutv1alpha1.ProgressingReasonCancelling: - klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) - var done bool - done, recheckTime, err = r.doFinalising(rollout, newStatus, false) - if err != nil { - return nil, err - // finalizer is finished - } else if done { - progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonCanceled, "") - } - - case rolloutv1alpha1.ProgressingReasonSucceeded, rolloutv1alpha1.ProgressingReasonCanceled: - klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) - } - - err = r.updateRolloutStatusInternal(rollout, *newStatus) - if err != nil { - klog.Errorf("update rollout(%s/%s) status failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return nil, err - } - return recheckTime, nil -} - -func progressingStateTransition(status *rolloutv1alpha1.RolloutStatus, condStatus corev1.ConditionStatus, reason, message string) { - cond := util.GetRolloutCondition(*status, rolloutv1alpha1.RolloutConditionProgressing) - if cond == nil { - cond = util.NewRolloutCondition(rolloutv1alpha1.RolloutConditionProgressing, condStatus, reason, message) - } else { - cond.Status = condStatus - cond.Reason = reason - if message != "" { - cond.Message = message - } - } - util.SetRolloutCondition(status, *cond) - status.Message = cond.Message -} - -func (r *RolloutReconciler) doProgressingInitializing(rollout *rolloutv1alpha1.Rollout, newStatus *rolloutv1alpha1.RolloutStatus) (bool, string, error) { - // canary release - return r.verifyCanaryStrategy(rollout, newStatus) -} - -func (r *RolloutReconciler) doProgressingReset(rollout *rolloutv1alpha1.Rollout, newStatus *rolloutv1alpha1.RolloutStatus) (bool, error) { - rolloutCon := newRolloutContext(r.Client, r.Recorder, rollout, newStatus, nil) - if rolloutCon.rollout.Spec.Strategy.Canary.TrafficRoutings != nil { - // 1. remove stable service podRevision selector - done, err := rolloutCon.restoreStableService() - if err != nil || !done { - return done, err - } - // 2. route all traffic to stable service - done, err = rolloutCon.doFinalisingTrafficRouting() - if err != nil || !done { - return done, err - } - } - - // 3. delete batchRelease CRD - done, err := rolloutCon.batchControl.Finalize() - if err != nil { - klog.Errorf("rollout(%s/%s) DoFinalising batchRelease failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return false, err - } else if !done { - return false, nil - } - return true, nil -} - -func (r *RolloutReconciler) verifyCanaryStrategy(rollout *rolloutv1alpha1.Rollout, newStatus *rolloutv1alpha1.RolloutStatus) (bool, string, error) { - canary := rollout.Spec.Strategy.Canary - // Traffic routing - if canary.TrafficRoutings != nil && len(canary.TrafficRoutings) > 0 { - rolloutCon := newRolloutContext(r.Client, r.Recorder, rollout, newStatus, nil) - trController, err := rolloutCon.newTrafficRoutingController(rolloutCon) - if err != nil { - return false, "", err - } - if ok, msg, err := r.verifyTrafficRouting(rollout.Namespace, canary.TrafficRoutings[0], trController); !ok { - return ok, msg, err - } - } - - // It is not allowed to modify the rollout.spec in progressing phase (validate webhook rollout), - // but in many scenarios the user may modify the workload and rollout spec at the same time, - // and there is a possibility that the workload is released first, and due to some network or other reasons the rollout spec is delayed by a few seconds, - // so this is mainly compatible with this scenario. - cond := util.GetRolloutCondition(*newStatus, rolloutv1alpha1.RolloutConditionProgressing) - if verifyTime := cond.LastUpdateTime.Add(time.Second * time.Duration(defaultGracePeriodSeconds)); verifyTime.After(time.Now()) { - klog.Infof("verify rollout(%s/%s) TrafficRouting done, and wait a moment", rollout.Namespace, rollout.Name) - return false, "", nil - } - return true, "", nil -} - -func (r *RolloutReconciler) verifyTrafficRouting(ns string, tr *rolloutv1alpha1.TrafficRouting, c trafficrouting.Controller) (bool, string, error) { - // check service - service := &corev1.Service{} - err := r.Get(context.TODO(), types.NamespacedName{Namespace: ns, Name: tr.Service}, service) - if err != nil { - if errors.IsNotFound(err) { - return false, fmt.Sprintf("Service(%s/%s) is Not Found", ns, tr.Service), nil - } - return false, "", err - } - - // check the traffic routing configuration - err = c.Initialize(context.TODO()) - if err != nil { - return false, "", err - } - return true, "", nil -} - -func (r *RolloutReconciler) reCalculateCanaryStepIndex(rollout *rolloutv1alpha1.Rollout, batchControl batchrelease.BatchRelease) (int32, error) { - batch, err := batchControl.FetchBatchRelease() - if errors.IsNotFound(err) { - return 1, nil - } else if err != nil { - return 0, err - } - workload, err := r.Finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) - if err != nil { - klog.Errorf("rollout(%s/%s) get workload failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return 0, err - } - currentReplicas, _ := intstr.GetScaledValueFromIntOrPercent(&batch.Spec.ReleasePlan.Batches[*batch.Spec.ReleasePlan.BatchPartition].CanaryReplicas, int(workload.Replicas), true) - - var stepIndex int32 - for i := range rollout.Spec.Strategy.Canary.Steps { - step := rollout.Spec.Strategy.Canary.Steps[i] - var desiredReplicas int - if step.Replicas != nil { - desiredReplicas, _ = intstr.GetScaledValueFromIntOrPercent(step.Replicas, int(workload.Replicas), true) - } else { - replicas := intstr.FromString(strconv.Itoa(int(*step.Weight)) + "%") - desiredReplicas, _ = intstr.GetScaledValueFromIntOrPercent(&replicas, int(workload.Replicas), true) - } - stepIndex = int32(i + 1) - if currentReplicas <= desiredReplicas { - break - } - } - return stepIndex, nil -} diff --git a/pkg/controller/rollout/progressing_test.go b/pkg/controller/rollout/progressing_test.go deleted file mode 100644 index d90d10fd..00000000 --- a/pkg/controller/rollout/progressing_test.go +++ /dev/null @@ -1,266 +0,0 @@ -/* -Copyright 2021. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rollout - -import ( - "context" - "testing" - - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/controller/rollout/batchrelease" - "github.com/openkruise/rollouts/pkg/util" - apps "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/util/intstr" - utilpointer "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestReCalculateCanaryStepIndex(t *testing.T) { - cases := []struct { - name string - getObj func() (*apps.Deployment, *apps.ReplicaSet) - getRollout func() *rolloutv1alpha1.Rollout - getBatchRelease func() *rolloutv1alpha1.BatchRelease - expectStepIndex int32 - }{ - { - name: "steps changed v1", - getObj: func() (*apps.Deployment, *apps.ReplicaSet) { - obj := deploymentDemo.DeepCopy() - return obj, rsDemo.DeepCopy() - }, - getRollout: func() *rolloutv1alpha1.Rollout { - obj := rolloutDemo.DeepCopy() - obj.Spec.Strategy.Canary.Steps = []rolloutv1alpha1.CanaryStep{ - { - Weight: utilpointer.Int32(20), - }, - { - Weight: utilpointer.Int32(50), - }, - { - Weight: utilpointer.Int32(100), - }, - } - return obj - }, - getBatchRelease: func() *rolloutv1alpha1.BatchRelease { - obj := batchDemo.DeepCopy() - obj.Spec.ReleasePlan.Batches = []rolloutv1alpha1.ReleaseBatch{ - { - CanaryReplicas: intstr.FromString("40%"), - }, - { - CanaryReplicas: intstr.FromString("60%"), - }, - { - CanaryReplicas: intstr.FromString("100%"), - }, - } - obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) - return obj - }, - expectStepIndex: 2, - }, - { - name: "steps changed v2", - getObj: func() (*apps.Deployment, *apps.ReplicaSet) { - obj := deploymentDemo.DeepCopy() - return obj, rsDemo.DeepCopy() - }, - getRollout: func() *rolloutv1alpha1.Rollout { - obj := rolloutDemo.DeepCopy() - obj.Spec.Strategy.Canary.Steps = []rolloutv1alpha1.CanaryStep{ - { - Weight: utilpointer.Int32(20), - }, - { - Weight: utilpointer.Int32(40), - }, - { - Weight: utilpointer.Int32(100), - }, - } - return obj - }, - getBatchRelease: func() *rolloutv1alpha1.BatchRelease { - obj := batchDemo.DeepCopy() - obj.Spec.ReleasePlan.Batches = []rolloutv1alpha1.ReleaseBatch{ - { - CanaryReplicas: intstr.FromString("40%"), - }, - { - CanaryReplicas: intstr.FromString("60%"), - }, - { - CanaryReplicas: intstr.FromString("100%"), - }, - } - obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) - return obj - }, - expectStepIndex: 2, - }, - { - name: "steps changed v3", - getObj: func() (*apps.Deployment, *apps.ReplicaSet) { - obj := deploymentDemo.DeepCopy() - return obj, rsDemo.DeepCopy() - }, - getRollout: func() *rolloutv1alpha1.Rollout { - obj := rolloutDemo.DeepCopy() - obj.Spec.Strategy.Canary.Steps = []rolloutv1alpha1.CanaryStep{ - { - Weight: utilpointer.Int32(40), - }, - { - Weight: utilpointer.Int32(60), - }, - { - Weight: utilpointer.Int32(100), - }, - } - return obj - }, - getBatchRelease: func() *rolloutv1alpha1.BatchRelease { - obj := batchDemo.DeepCopy() - obj.Spec.ReleasePlan.Batches = []rolloutv1alpha1.ReleaseBatch{ - { - CanaryReplicas: intstr.FromString("20%"), - }, - { - CanaryReplicas: intstr.FromString("40%"), - }, - { - CanaryReplicas: intstr.FromString("100%"), - }, - } - obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(1) - return obj - }, - expectStepIndex: 1, - }, - { - name: "steps changed v4", - getObj: func() (*apps.Deployment, *apps.ReplicaSet) { - obj := deploymentDemo.DeepCopy() - return obj, rsDemo.DeepCopy() - }, - getRollout: func() *rolloutv1alpha1.Rollout { - obj := rolloutDemo.DeepCopy() - obj.Spec.Strategy.Canary.Steps = []rolloutv1alpha1.CanaryStep{ - { - Weight: utilpointer.Int32(10), - }, - { - Weight: utilpointer.Int32(30), - }, - { - Weight: utilpointer.Int32(100), - }, - } - return obj - }, - getBatchRelease: func() *rolloutv1alpha1.BatchRelease { - obj := batchDemo.DeepCopy() - obj.Spec.ReleasePlan.Batches = []rolloutv1alpha1.ReleaseBatch{ - { - CanaryReplicas: intstr.FromString("20%"), - }, - { - CanaryReplicas: intstr.FromString("40%"), - }, - { - CanaryReplicas: intstr.FromString("100%"), - }, - } - obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) - return obj - }, - expectStepIndex: 2, - }, - { - name: "steps changed v5", - getObj: func() (*apps.Deployment, *apps.ReplicaSet) { - obj := deploymentDemo.DeepCopy() - return obj, rsDemo.DeepCopy() - }, - getRollout: func() *rolloutv1alpha1.Rollout { - obj := rolloutDemo.DeepCopy() - obj.Spec.Strategy.Canary.Steps = []rolloutv1alpha1.CanaryStep{ - { - Weight: utilpointer.Int32(2), - Replicas: &intstr.IntOrString{ - Type: intstr.String, - StrVal: "10%", - }, - }, - { - Weight: utilpointer.Int32(3), - Replicas: &intstr.IntOrString{ - Type: intstr.String, - StrVal: "10%", - }, - }, - } - return obj - }, - getBatchRelease: func() *rolloutv1alpha1.BatchRelease { - obj := batchDemo.DeepCopy() - obj.Spec.ReleasePlan.Batches = []rolloutv1alpha1.ReleaseBatch{ - { - CanaryReplicas: intstr.FromString("10%"), - }, - { - CanaryReplicas: intstr.FromString("20%"), - }, - { - CanaryReplicas: intstr.FromString("30%"), - }, - } - obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) - return obj - }, - expectStepIndex: 1, - }, - } - - for _, cs := range cases { - t.Run(cs.name, func(t *testing.T) { - client := fake.NewClientBuilder().WithScheme(scheme).Build() - client.Create(context.TODO(), cs.getBatchRelease()) - dep, rs := cs.getObj() - client.Create(context.TODO(), dep) - client.Create(context.TODO(), rs) - client.Create(context.TODO(), cs.getRollout()) - - reconciler := &RolloutReconciler{ - Client: client, - Scheme: scheme, - Finder: util.NewControllerFinder(client), - } - batchControl := batchrelease.NewInnerBatchController(client, cs.getRollout(), "") - newStepIndex, err := reconciler.reCalculateCanaryStepIndex(cs.getRollout(), batchControl) - if err != nil { - t.Fatalf(err.Error()) - } - if cs.expectStepIndex != newStepIndex { - t.Fatalf("expect %d, but %d", cs.expectStepIndex, newStepIndex) - } - }) - } -} diff --git a/pkg/controller/rollout/rolling.go b/pkg/controller/rollout/rolling.go deleted file mode 100644 index 0863caf1..00000000 --- a/pkg/controller/rollout/rolling.go +++ /dev/null @@ -1,159 +0,0 @@ -package rollout - -import ( - "time" - - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/controller/rollout/batchrelease" - "github.com/openkruise/rollouts/pkg/util" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" -) - -func (r *RolloutReconciler) doProgressingInRolling(rollout *rolloutv1alpha1.Rollout, workload *util.Workload, newStatus *rolloutv1alpha1.RolloutStatus) (*time.Time, error) { - // Handle the 5 special cases firstly, and we had better keep the order of following cases: - - switch { - // 1. In case of rollback in a quick way, un-paused and just use workload rolling strategy - case isRollingBackDirectly(rollout, workload): - return r.handleRollbackDirectly(rollout, workload, newStatus) - - // 2. In case of rollout paused, just stop reconcile - case isRolloutPaused(rollout): - return r.handleRolloutPaused(rollout, newStatus) - - // 3. In case of rollback in a batch way, use rollout step strategy - case isRollingBackInBatches(rollout, workload): - return r.handleRollbackInBatches(rollout, workload, newStatus) - - // 4. In case of continuous publishing(v1 -> v2 -> v3), restart publishing - case isContinuousRelease(rollout, workload): - return r.handleContinuousRelease(rollout, workload, newStatus) - - // 5. In case of rollout plan changed, recalculate and publishing - case isRolloutPlanChanged(rollout): - return r.handleRolloutPlanChanged(rollout, workload, newStatus) - } - - return r.handleNormalRolling(rollout, workload, newStatus) -} - -func (r *RolloutReconciler) handleRolloutPaused(rollout *rolloutv1alpha1.Rollout, newStatus *rolloutv1alpha1.RolloutStatus) (*time.Time, error) { - klog.Infof("rollout(%s/%s) is Progressing, but paused", rollout.Namespace, rollout.Name) - progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonPaused, "Rollout has been paused, you can resume it by kube-cli") - return nil, nil -} - -func (r *RolloutReconciler) handleContinuousRelease(rollout *rolloutv1alpha1.Rollout, workload *util.Workload, newStatus *rolloutv1alpha1.RolloutStatus) (*time.Time, error) { - r.Recorder.Eventf(rollout, corev1.EventTypeNormal, "Progressing", "workload continuous publishing canaryRevision, then restart publishing") - klog.Infof("rollout(%s/%s) workload continuous publishing canaryRevision from(%s) -> to(%s), then restart publishing", - rollout.Namespace, rollout.Name, newStatus.CanaryStatus.CanaryRevision, workload.CanaryRevision) - - var recheckTime *time.Time - done, err := r.doProgressingReset(rollout, newStatus) - if err != nil { - klog.Errorf("rollout(%s/%s) doProgressingReset failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return nil, err - } else if done { - progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonInitializing, "Workload is continuous release") - klog.Infof("rollout(%s/%s) workload is continuous publishing, reset complete", rollout.Namespace, rollout.Name) - } else { - // Incomplete, recheck - expectedTime := time.Now().Add(time.Duration(defaultGracePeriodSeconds) * time.Second) - recheckTime = &expectedTime - klog.Infof("rollout(%s/%s) workload is continuous publishing, reset incomplete, and recheck(%s)", rollout.Namespace, rollout.Name, expectedTime.String()) - } - return recheckTime, nil -} - -func (r *RolloutReconciler) handleRollbackDirectly(rollout *rolloutv1alpha1.Rollout, workload *util.Workload, newStatus *rolloutv1alpha1.RolloutStatus) (*time.Time, error) { - newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision - r.Recorder.Eventf(rollout, corev1.EventTypeNormal, "Progressing", "workload has been rollback, then rollout is canceled") - klog.Infof("rollout(%s/%s) workload has been rollback directly, then rollout canceled", rollout.Namespace, rollout.Name) - progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonCancelling, "The workload has been rolled back and the rollout process will be cancelled") - return nil, nil -} - -func (r *RolloutReconciler) handleRollbackInBatches(rollout *rolloutv1alpha1.Rollout, workload *util.Workload, newStatus *rolloutv1alpha1.RolloutStatus) (*time.Time, error) { - // restart from the beginning - newStatus.CanaryStatus.CurrentStepIndex = 1 - newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision - newStatus.CanaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateUpgrade - newStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - newStatus.CanaryStatus.RolloutHash = rollout.Annotations[util.RolloutHashAnnotation] - klog.Infof("rollout(%s/%s) workload has been rollback in batches, then restart from beginning", rollout.Namespace, rollout.Name) - return nil, nil -} - -func (r *RolloutReconciler) handleRolloutPlanChanged(rollout *rolloutv1alpha1.Rollout, workload *util.Workload, newStatus *rolloutv1alpha1.RolloutStatus) (*time.Time, error) { - batchControl := batchrelease.NewInnerBatchController(r.Client, rollout, getRolloutID(workload, rollout)) - newStepIndex, err := r.reCalculateCanaryStepIndex(rollout, batchControl) - if err != nil { - klog.Errorf("rollout(%s/%s) reCalculate Canary StepIndex failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return nil, err - } - // canary step configuration change causes current step index change - newStatus.CanaryStatus.CurrentStepIndex = newStepIndex - newStatus.CanaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateUpgrade - newStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - newStatus.CanaryStatus.RolloutHash = rollout.Annotations[util.RolloutHashAnnotation] - klog.Infof("rollout(%s/%s) canary step configuration change, and stepIndex(%d) state(%s)", - rollout.Namespace, rollout.Name, newStatus.CanaryStatus.CurrentStepIndex, newStatus.CanaryStatus.CurrentStepState) - return nil, nil -} - -func (r *RolloutReconciler) handleNormalRolling(rollout *rolloutv1alpha1.Rollout, workload *util.Workload, newStatus *rolloutv1alpha1.RolloutStatus) (*time.Time, error) { - //check if canary is done - if newStatus.CanaryStatus.CurrentStepState == rolloutv1alpha1.CanaryStepStateCompleted { - klog.Infof("rollout(%s/%s) progressing rolling done", rollout.Namespace, rollout.Name) - progressingStateTransition(newStatus, corev1.ConditionTrue, rolloutv1alpha1.ProgressingReasonFinalising, "Rollout has been completed and some closing work is being done") - } else { // rollout is in rolling - newStatus.CanaryStatus.PodTemplateHash = workload.PodTemplateHash - return r.doNormalRolling(rollout, workload, newStatus) - } - return nil, nil -} - -func (r *RolloutReconciler) doNormalRolling(rollout *rolloutv1alpha1.Rollout, workload *util.Workload, newStatus *rolloutv1alpha1.RolloutStatus) (*time.Time, error) { - rolloutCon := newRolloutContext(r.Client, r.Recorder, rollout, newStatus, workload) - err := rolloutCon.reconcile() - if err != nil { - klog.Errorf("rollout(%s/%s) Progressing failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return nil, err - } - return rolloutCon.recheckTime, nil -} - -/* ********************************************************************** - help functions -*********************************************************************** */ -func isRolloutPaused(rollout *rolloutv1alpha1.Rollout) bool { - return rollout.Spec.Strategy.Paused -} - -func isRolloutPlanChanged(rollout *rolloutv1alpha1.Rollout) bool { - status := &rollout.Status - return status.CanaryStatus.RolloutHash != "" && status.CanaryStatus.RolloutHash != rollout.Annotations[util.RolloutHashAnnotation] -} - -func isContinuousRelease(rollout *rolloutv1alpha1.Rollout, workload *util.Workload) bool { - status := &rollout.Status - return status.CanaryStatus.CanaryRevision != "" && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && !workload.IsInRollback -} - -func isRollingBackDirectly(rollout *rolloutv1alpha1.Rollout, workload *util.Workload) bool { - status := &rollout.Status - inBatch := util.IsRollbackInBatchPolicy(rollout, workload.Labels) - return workload.IsInRollback && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && !inBatch -} - -func isRollingBackInBatches(rollout *rolloutv1alpha1.Rollout, workload *util.Workload) bool { - // currently, only support the case of no traffic routing - if len(rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { - return false - } - status := &rollout.Status - inBatch := util.IsRollbackInBatchPolicy(rollout, workload.Labels) - return workload.IsInRollback && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && inBatch -} diff --git a/pkg/controller/rollout/rollout_canary.go b/pkg/controller/rollout/rollout_canary.go new file mode 100644 index 00000000..cc39231f --- /dev/null +++ b/pkg/controller/rollout/rollout_canary.go @@ -0,0 +1,476 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollout + +import ( + "context" + "fmt" + "reflect" + "strconv" + "time" + + appsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting" + "github.com/openkruise/rollouts/pkg/util" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type canaryReleaseManager struct { + client.Client + trafficRoutingManager *trafficrouting.Manager + recorder record.EventRecorder +} + +func (m *canaryReleaseManager) runCanary(c *util.RolloutContext) error { + canaryStatus := c.NewStatus.CanaryStatus + // update canary status + if br, _ := m.fetchBatchRelease(c.Rollout.Namespace, c.Rollout.Name); br != nil { + canaryStatus.CanaryReplicas = br.Status.CanaryStatus.UpdatedReplicas + canaryStatus.CanaryReadyReplicas = br.Status.CanaryStatus.UpdatedReadyReplicas + } + // update podTemplateHash, Why is this position assigned? + // Because If workload is deployment, only after canary pod already was created, + // we can get the podTemplateHash from pod.annotations[pod-template-hash] + if canaryStatus.PodTemplateHash == "" { + canaryStatus.PodTemplateHash = c.Workload.PodTemplateHash + } + + switch canaryStatus.CurrentStepState { + case v1alpha1.CanaryStepStateUpgrade: + klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", c.Rollout.Namespace, c.Rollout.Name, v1alpha1.CanaryStepStateUpgrade) + done, err := m.doCanaryUpgrade(c) + if err != nil { + return err + } else if done { + canaryStatus.CurrentStepState = v1alpha1.CanaryStepStateTrafficRouting + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", c.Rollout.Namespace, c.Rollout.Name, + canaryStatus.CurrentStepIndex, v1alpha1.CanaryStepStateUpgrade, canaryStatus.CurrentStepState) + } + + case v1alpha1.CanaryStepStateTrafficRouting: + klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", c.Rollout.Namespace, c.Rollout.Name, v1alpha1.CanaryStepStateTrafficRouting) + done, err := m.trafficRoutingManager.DoTrafficRouting(c) + if err != nil { + return err + } else if done { + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + canaryStatus.CurrentStepState = v1alpha1.CanaryStepStateMetricsAnalysis + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", c.Rollout.Namespace, c.Rollout.Name, + canaryStatus.CurrentStepIndex, v1alpha1.CanaryStepStateTrafficRouting, canaryStatus.CurrentStepState) + } + expectedTime := time.Now().Add(time.Duration(defaultGracePeriodSeconds) * time.Second) + c.RecheckTime = &expectedTime + + case v1alpha1.CanaryStepStateMetricsAnalysis: + klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", c.Rollout.Namespace, c.Rollout.Name, v1alpha1.CanaryStepStateMetricsAnalysis) + done, err := m.doCanaryMetricsAnalysis(c) + if err != nil { + return err + } else if done { + canaryStatus.CurrentStepState = v1alpha1.CanaryStepStatePaused + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", c.Rollout.Namespace, c.Rollout.Name, + canaryStatus.CurrentStepIndex, v1alpha1.CanaryStepStateMetricsAnalysis, canaryStatus.CurrentStepState) + } + + case v1alpha1.CanaryStepStatePaused: + klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", c.Rollout.Namespace, c.Rollout.Name, v1alpha1.CanaryStepStatePaused) + done, err := m.doCanaryPaused(c) + if err != nil { + return err + } else if done { + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + canaryStatus.CurrentStepState = v1alpha1.CanaryStepStateReady + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", c.Rollout.Namespace, c.Rollout.Name, + canaryStatus.CurrentStepIndex, v1alpha1.CanaryStepStatePaused, canaryStatus.CurrentStepState) + } + + case v1alpha1.CanaryStepStateReady: + klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", c.Rollout.Namespace, c.Rollout.Name, v1alpha1.CanaryStepStateReady) + // run next step + if len(c.Rollout.Spec.Strategy.Canary.Steps) > int(canaryStatus.CurrentStepIndex) { + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + canaryStatus.CurrentStepIndex++ + canaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + klog.Infof("rollout(%s/%s) canary step from(%d) -> to(%d)", c.Rollout.Namespace, c.Rollout.Name, canaryStatus.CurrentStepIndex-1, canaryStatus.CurrentStepIndex) + } else { + klog.Infof("rollout(%s/%s) canary run all steps, and completed", c.Rollout.Namespace, c.Rollout.Name) + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + canaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + } + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", c.Rollout.Namespace, c.Rollout.Name, + canaryStatus.CurrentStepIndex, v1alpha1.CanaryStepStateReady, canaryStatus.CurrentStepState) + // canary completed + case v1alpha1.CanaryStepStateCompleted: + klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", c.Rollout.Namespace, c.Rollout.Name, v1alpha1.CanaryStepStateCompleted) + } + + return nil +} + +func (m *canaryReleaseManager) doCanaryUpgrade(c *util.RolloutContext) (bool, error) { + // verify whether batchRelease configuration is the latest + steps := len(c.Rollout.Spec.Strategy.Canary.Steps) + canaryStatus := c.NewStatus.CanaryStatus + cond := util.GetRolloutCondition(*c.NewStatus, v1alpha1.RolloutConditionProgressing) + cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and upgrade workload to new version", canaryStatus.CurrentStepIndex, steps) + c.NewStatus.Message = cond.Message + // run batch release to upgrade the workloads + done, br, err := m.runBatchRelease(c.Rollout, getRolloutID(c.Workload), canaryStatus.CurrentStepIndex, c.Workload.IsInRollback) + if err != nil { + return false, err + } else if !done { + return false, nil + } + if br.Status.ObservedReleasePlanHash != util.HashReleasePlanBatches(&br.Spec.ReleasePlan) || + br.Generation != br.Status.ObservedGeneration { + klog.Infof("rollout(%s/%s) batchRelease status is inconsistent, and wait a moment", c.Rollout.Namespace, c.Rollout.Name) + return false, nil + } + // check whether batchRelease is ready(whether new pods is ready.) + if br.Status.CanaryStatus.CurrentBatchState != v1alpha1.ReadyBatchState || + br.Status.CanaryStatus.CurrentBatch+1 < canaryStatus.CurrentStepIndex { + klog.Infof("rollout(%s/%s) batchRelease status(%s) is not ready, and wait a moment", c.Rollout.Namespace, c.Rollout.Name, util.DumpJSON(br.Status)) + return false, nil + } + m.recorder.Eventf(c.Rollout, corev1.EventTypeNormal, "Progressing", fmt.Sprintf("upgrade step(%d) canary pods with new versions done", canaryStatus.CurrentStepIndex)) + klog.Infof("rollout(%s/%s) batch(%s) state(%s), and success", + c.Rollout.Namespace, c.Rollout.Name, util.DumpJSON(br.Status), br.Status.CanaryStatus.CurrentBatchState) + return true, nil +} + +func (m *canaryReleaseManager) doCanaryMetricsAnalysis(c *util.RolloutContext) (bool, error) { + // todo + return true, nil +} + +func (m *canaryReleaseManager) doCanaryPaused(c *util.RolloutContext) (bool, error) { + canaryStatus := c.NewStatus.CanaryStatus + currentStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] + steps := len(c.Rollout.Spec.Strategy.Canary.Steps) + cond := util.GetRolloutCondition(*c.NewStatus, v1alpha1.RolloutConditionProgressing) + // need manual confirmation + if currentStep.Pause.Duration == nil { + klog.Infof("rollout(%s/%s) don't set pause duration, and need manual confirmation", c.Rollout.Namespace, c.Rollout.Name) + cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and you need manually confirm to enter the next step", canaryStatus.CurrentStepIndex, steps) + c.NewStatus.Message = cond.Message + return false, nil + } + cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and wait duration(%d seconds) to enter the next step", canaryStatus.CurrentStepIndex, steps, *currentStep.Pause.Duration) + c.NewStatus.Message = cond.Message + // wait duration time, then go to next step + duration := time.Second * time.Duration(*currentStep.Pause.Duration) + expectedTime := canaryStatus.LastUpdateTime.Add(duration) + if expectedTime.Before(time.Now()) { + klog.Infof("rollout(%s/%s) canary step(%d) paused duration(%d seconds), and go to the next step", + c.Rollout.Namespace, c.Rollout.Name, canaryStatus.CurrentStepIndex, *currentStep.Pause.Duration) + return true, nil + } + c.RecheckTime = &expectedTime + return false, nil +} + +// cleanup after rollout is completed or finished +func (m *canaryReleaseManager) doCanaryFinalising(c *util.RolloutContext) (bool, error) { + // when CanaryStatus is nil, which means canary action hasn't started yet, don't need doing cleanup + if c.NewStatus.CanaryStatus == nil { + return true, nil + } + // 1. rollout progressing complete, remove rollout progressing annotation in workload + err := m.removeRolloutProgressingAnnotation(c) + if err != nil { + return false, err + } + // 2. remove stable service the pod revision selector, so stable service will be selector all version pods. + done, err := m.trafficRoutingManager.FinalisingTrafficRouting(c, true) + if err != nil || !done { + return done, err + } + // 3. set workload.pause=false; set workload.partition=0 + done, err = m.restoreWorkload(c) + if err != nil || !done { + return done, err + } + // 4. modify network api(ingress or gateway api) configuration, and route 100% traffic to stable pods. + done, err = m.trafficRoutingManager.FinalisingTrafficRouting(c, false) + if err != nil || !done { + return done, err + } + // 5. delete batchRelease crd + done, err = m.removeBatchRelease(c) + if err != nil { + klog.Errorf("rollout(%s/%s) Finalize batchRelease failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return false, err + } else if !done { + return false, nil + } + klog.Infof("rollout(%s/%s) doCanaryFinalising success", c.Rollout.Namespace, c.Rollout.Name) + return true, nil +} + +func (m *canaryReleaseManager) removeRolloutProgressingAnnotation(c *util.RolloutContext) error { + if c.Workload == nil { + return nil + } + if _, ok := c.Workload.Annotations[util.InRolloutProgressingAnnotation]; !ok { + return nil + } + workloadRef := c.Rollout.Spec.ObjectRef.WorkloadRef + workloadGVK := schema.FromAPIVersionAndKind(workloadRef.APIVersion, workloadRef.Kind) + obj := util.GetEmptyWorkloadObject(workloadGVK) + if err := m.Get(context.TODO(), types.NamespacedName{Name: c.Workload.Name, Namespace: c.Workload.Namespace}, obj); err != nil { + klog.Errorf("getting updated workload(%s.%s) failed: %s", c.Workload.Namespace, c.Workload.Name, err.Error()) + return err + } + body := fmt.Sprintf(`{"metadata":{"annotations":{"%s":null}}}`, util.InRolloutProgressingAnnotation) + if err := m.Patch(context.TODO(), obj, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { + klog.Errorf("rollout(%s/%s) patch workload(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, c.Workload.Name, err.Error()) + return err + } + klog.Infof("remove rollout(%s/%s) workload(%s) annotation[%s] success", c.Rollout.Namespace, c.Rollout.Name, c.Workload.Name, util.InRolloutProgressingAnnotation) + return nil +} + +func (m *canaryReleaseManager) runBatchRelease(rollout *v1alpha1.Rollout, rolloutId string, batch int32, isRollback bool) (bool, *v1alpha1.BatchRelease, error) { + batch = batch - 1 + br, err := m.fetchBatchRelease(rollout.Namespace, rollout.Name) + if errors.IsNotFound(err) { + // create new BatchRelease Crd + br = createBatchRelease(rollout, rolloutId, batch, isRollback) + if err = m.Create(context.TODO(), br); err != nil && !errors.IsAlreadyExists(err) { + klog.Errorf("rollout(%s/%s) create BatchRelease failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return false, nil, err + } + klog.Infof("rollout(%s/%s) create BatchRelease(%s) success", rollout.Namespace, rollout.Name, util.DumpJSON(br)) + return false, br, nil + } else if err != nil { + klog.Errorf("rollout(%s/%s) fetch BatchRelease failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return false, nil, err + } + + // check whether batchRelease configuration is the latest + newBr := createBatchRelease(rollout, rolloutId, batch, isRollback) + if reflect.DeepEqual(br.Spec, newBr.Spec) && reflect.DeepEqual(br.Annotations, newBr.Annotations) { + klog.Infof("rollout(%s/%s) do batchRelease batch(%d) success", rollout.Namespace, rollout.Name, batch+1) + return true, br, nil + } + + // update batchRelease to the latest version + if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err = m.Get(context.TODO(), client.ObjectKey{Namespace: newBr.Namespace, Name: newBr.Name}, br); err != nil { + klog.Errorf("error getting BatchRelease(%s/%s) from client", newBr.Namespace, newBr.Name) + return err + } + br.Spec = newBr.Spec + br.Annotations = newBr.Annotations + return m.Client.Update(context.TODO(), br) + }); err != nil { + klog.Errorf("rollout(%s/%s) update batchRelease failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return false, nil, err + } + klog.Infof("rollout(%s/%s) update batchRelease(%s) configuration to latest", rollout.Namespace, rollout.Name, util.DumpJSON(br)) + return false, br, nil +} + +func (m *canaryReleaseManager) fetchBatchRelease(ns, name string) (*v1alpha1.BatchRelease, error) { + br := &v1alpha1.BatchRelease{} + // batchRelease.name is equal related rollout.name + err := m.Get(context.TODO(), client.ObjectKey{Namespace: ns, Name: name}, br) + return br, err +} + +func createBatchRelease(rollout *v1alpha1.Rollout, rolloutID string, batch int32, isRollback bool) *v1alpha1.BatchRelease { + var batches []v1alpha1.ReleaseBatch + for _, step := range rollout.Spec.Strategy.Canary.Steps { + if step.Replicas == nil { + batches = append(batches, v1alpha1.ReleaseBatch{CanaryReplicas: intstr.FromString(strconv.Itoa(int(*step.Weight)) + "%")}) + } else { + batches = append(batches, v1alpha1.ReleaseBatch{CanaryReplicas: *step.Replicas}) + } + } + br := &v1alpha1.BatchRelease{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: rollout.Namespace, + Name: rollout.Name, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(rollout, rolloutControllerKind)}, + }, + Spec: v1alpha1.BatchReleaseSpec{ + TargetRef: v1alpha1.ObjectRef{ + WorkloadRef: &v1alpha1.WorkloadRef{ + APIVersion: rollout.Spec.ObjectRef.WorkloadRef.APIVersion, + Kind: rollout.Spec.ObjectRef.WorkloadRef.Kind, + Name: rollout.Spec.ObjectRef.WorkloadRef.Name, + }, + }, + ReleasePlan: v1alpha1.ReleasePlan{ + Batches: batches, + RolloutID: rolloutID, + BatchPartition: utilpointer.Int32Ptr(batch), + }, + }, + } + if isRollback { + if br.Annotations == nil { + br.Annotations = map[string]string{} + } + br.Annotations[v1alpha1.RollbackInBatchAnnotation] = "true" + } + return br +} + +func (m *canaryReleaseManager) removeBatchRelease(c *util.RolloutContext) (bool, error) { + batch := &v1alpha1.BatchRelease{} + err := m.Get(context.TODO(), client.ObjectKey{Namespace: c.Rollout.Namespace, Name: c.Rollout.Name}, batch) + if err != nil && errors.IsNotFound(err) { + return true, nil + } else if err != nil { + klog.Errorf("rollout(%s/%s) fetch BatchRelease failed: %s", c.Rollout.Namespace, c.Rollout.Name) + return false, err + } + if !batch.DeletionTimestamp.IsZero() { + klog.Infof("rollout(%s/%s) BatchRelease is terminating, and wait a moment", c.Rollout.Namespace, c.Rollout.Name) + return false, nil + } + + //delete batchRelease + err = m.Delete(context.TODO(), batch) + if err != nil { + klog.Errorf("rollout(%s/%s) delete BatchRelease failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return false, err + } + klog.Infof("rollout(%s/%s) deleting BatchRelease, and wait a moment", c.Rollout.Namespace, c.Rollout.Name) + return false, nil +} + +// todo, remove the section, and batchRelease will implement the process +// 1. set workload.pause = false +// 2. set workload.partition = 0 +func (m *canaryReleaseManager) restoreWorkload(c *util.RolloutContext) (bool, error) { + // cloneSet + ref := c.Rollout.Spec.ObjectRef.WorkloadRef + switch ref.Kind { + case util.ControllerKruiseKindCS.Kind: + obj := &appsv1alpha1.CloneSet{} + err := m.Get(context.TODO(), types.NamespacedName{Namespace: c.Rollout.Namespace, Name: ref.Name}, obj) + if err != nil { + if errors.IsNotFound(err) { + klog.Warningf("rollout(%s/%s) cloneSet(%s) not found, and return true", c.Rollout.Namespace, c.Rollout.Name, ref.Name) + return true, nil + } + return false, err + } + // default partition.IntVal=0 + if !obj.Spec.UpdateStrategy.Paused && obj.Spec.UpdateStrategy.Partition.IntVal == 0 && obj.Spec.UpdateStrategy.Partition.Type == intstr.Int { + return true, nil + } + err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err = m.Get(context.TODO(), types.NamespacedName{Namespace: c.Rollout.Namespace, Name: ref.Name}, obj); err != nil { + return err + } + obj.Spec.UpdateStrategy.Paused = false + obj.Spec.UpdateStrategy.Partition = nil + return m.Update(context.TODO(), obj) + }) + if err != nil { + klog.Errorf("update rollout(%s/%s) cloneSet failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return false, err + } + klog.Infof("resume rollout(%s/%s) cloneSet(paused=false,partition=nil) success", c.Rollout.Namespace, c.Rollout.Name) + return true, nil + + case util.ControllerKindDep.Kind: + // deployment + dName := ref.Name + obj := &apps.Deployment{} + err := m.Get(context.TODO(), types.NamespacedName{Namespace: c.Rollout.Namespace, Name: dName}, obj) + if err != nil { + if errors.IsNotFound(err) { + klog.Warningf("rollout(%s/%s) stable deployment(%s) not found, and return true", c.Rollout.Namespace, c.Rollout.Name, dName) + return true, nil + } + return false, err + } + // set deployment paused=false + if obj.Spec.Paused { + err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err = m.Get(context.TODO(), types.NamespacedName{Namespace: c.Rollout.Namespace, Name: dName}, obj); err != nil { + return err + } + obj.Spec.Paused = false + return m.Update(context.TODO(), obj) + }) + if err != nil { + klog.Errorf("update rollout(%s/%s) stable deployment failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return false, err + } + klog.Infof("resume rollout(%s/%s) stable deployment(paused=false) success", c.Rollout.Namespace, c.Rollout.Name) + } + // wait stable deployment pods are ready + data := util.DumpJSON(obj.Status) + maxUnavailable, _ := intstr.GetScaledValueFromIntOrPercent(obj.Spec.Strategy.RollingUpdate.MaxUnavailable, int(*obj.Spec.Replicas), true) + if obj.Status.ObservedGeneration != obj.Generation || obj.Status.UpdatedReplicas != *obj.Spec.Replicas || + obj.Status.Replicas != *obj.Spec.Replicas || *obj.Spec.Replicas-obj.Status.AvailableReplicas > int32(maxUnavailable) { + klog.Infof("rollout(%s/%s) stable deployment status(%s), and wait a moment", c.Rollout.Namespace, c.Rollout.Name, data) + return false, nil + } + klog.Infof("resume rollout(%s/%s) stable deployment(paused=false) status(%s) success", c.Rollout.Namespace, c.Rollout.Name, data) + return true, nil + + default: + // statefulset-like workloads + workloadRef := ref + workloadNsn := types.NamespacedName{Namespace: c.Rollout.Namespace, Name: workloadRef.Name} + workloadGVK := schema.FromAPIVersionAndKind(workloadRef.APIVersion, workloadRef.Kind) + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(workloadGVK) + err := m.Get(context.TODO(), workloadNsn, obj) + if err != nil { + if errors.IsNotFound(err) { + klog.Warningf("rollout(%s/%s) statefulset(%s) not found, and return true", c.Rollout.Namespace, c.Rollout.Name, workloadNsn.Name) + return true, nil + } + return false, err + } + + if util.GetStatefulSetPartition(obj) == 0 { + return true, nil + } + + cloneObj := obj.DeepCopy() + body := `{"spec":{"updateStrategy":{"rollingUpdate":{"partition":0}}}}` + err = m.Patch(context.TODO(), cloneObj, client.RawPatch(types.MergePatchType, []byte(body))) + if err != nil { + klog.Errorf("patch rollout(%s/%s) statefulset failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return false, err + } + klog.Infof("resume rollout(%s/%s) statefulset(partition=0) success", c.Rollout.Namespace, c.Rollout.Name) + return true, nil + } +} diff --git a/pkg/controller/rollout/rollout_canary_test.go b/pkg/controller/rollout/rollout_canary_test.go new file mode 100644 index 00000000..2a456e7d --- /dev/null +++ b/pkg/controller/rollout/rollout_canary_test.go @@ -0,0 +1,273 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollout + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting" + "github.com/openkruise/rollouts/pkg/util" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestRunCanary(t *testing.T) { + cases := []struct { + name string + getObj func() ([]*apps.Deployment, []*apps.ReplicaSet) + getNetwork func() ([]*corev1.Service, []*netv1.Ingress) + getRollout func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) + expectStatus func() *v1alpha1.RolloutStatus + expectBr func() *v1alpha1.BatchRelease + }{ + { + name: "run canary upgrade1", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + rs1 := rsDemo.DeepCopy() + return []*apps.Deployment{dep1}, []*apps.ReplicaSet{rs1} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + obj := rolloutDemo.DeepCopy() + obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 + obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" + obj.Status.CanaryStatus.CanaryRevision = "56855c89f9" + obj.Status.CanaryStatus.CurrentStepIndex = 1 + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(&obj.Status, *cond) + return obj, nil + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus.ObservedWorkloadGeneration = 2 + s.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + s.CanaryStatus.StableRevision = "pod-template-hash-v1" + s.CanaryStatus.CanaryRevision = "56855c89f9" + s.CanaryStatus.CurrentStepIndex = 1 + s.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(*s, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(s, *cond) + return s + }, + expectBr: func() *v1alpha1.BatchRelease { + br := batchDemo.DeepCopy() + br.Spec.ReleasePlan.Batches = []v1alpha1.ReleaseBatch{ + { + CanaryReplicas: intstr.FromInt(1), + }, + { + CanaryReplicas: intstr.FromInt(2), + }, + { + CanaryReplicas: intstr.FromInt(6), + }, + { + CanaryReplicas: intstr.FromInt(10), + }, + } + br.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) + return br + }, + }, + { + name: "run canary traffic routing", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + dep2 := deploymentDemo.DeepCopy() + dep2.UID = "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180" + dep2.Name = dep1.Name + "-canary" + dep2.Labels[util.CanaryDeploymentLabel] = dep1.Name + rs1 := rsDemo.DeepCopy() + rs2 := rsDemo.DeepCopy() + rs2.Name = "echoserver-canary-2" + rs2.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "Deployment", + Name: dep2.Name, + UID: "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180", + Controller: utilpointer.BoolPtr(true), + }, + } + rs2.Labels["pod-template-hash"] = "pod-template-hash-v2" + rs2.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + return []*apps.Deployment{dep1, dep2}, []*apps.ReplicaSet{rs1, rs2} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + obj := rolloutDemo.DeepCopy() + obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 + obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" + obj.Status.CanaryStatus.CanaryRevision = "56855c89f9" + obj.Status.CanaryStatus.CurrentStepIndex = 1 + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(&obj.Status, *cond) + br := batchDemo.DeepCopy() + br.Spec.ReleasePlan.Batches = []v1alpha1.ReleaseBatch{ + { + CanaryReplicas: intstr.FromInt(1), + }, + { + CanaryReplicas: intstr.FromInt(2), + }, + { + CanaryReplicas: intstr.FromInt(6), + }, + { + CanaryReplicas: intstr.FromInt(10), + }, + } + br.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) + br.Status = v1alpha1.BatchReleaseStatus{ + ObservedGeneration: 1, + ObservedReleasePlanHash: "6d6a40791161e88ec0483688e951b589a4cbd0bf351974827706b79f99378fd5", + CanaryStatus: v1alpha1.BatchReleaseCanaryStatus{ + CurrentBatchState: v1alpha1.ReadyBatchState, + CurrentBatch: 0, + UpdatedReplicas: 1, + UpdatedReadyReplicas: 1, + }, + } + return obj, br + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus.ObservedWorkloadGeneration = 2 + s.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + s.CanaryStatus.StableRevision = "pod-template-hash-v1" + s.CanaryStatus.CanaryRevision = "56855c89f9" + s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + s.CanaryStatus.CanaryReplicas = 1 + s.CanaryStatus.CanaryReadyReplicas = 1 + s.CanaryStatus.CurrentStepIndex = 1 + s.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateTrafficRouting + cond := util.GetRolloutCondition(*s, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(s, *cond) + return s + }, + expectBr: func() *v1alpha1.BatchRelease { + br := batchDemo.DeepCopy() + br.Spec.ReleasePlan.Batches = []v1alpha1.ReleaseBatch{ + { + CanaryReplicas: intstr.FromInt(1), + }, + { + CanaryReplicas: intstr.FromInt(2), + }, + { + CanaryReplicas: intstr.FromInt(6), + }, + { + CanaryReplicas: intstr.FromInt(10), + }, + } + br.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) + return br + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fmt.Println(cs.name) + deps, rss := cs.getObj() + rollout, br := cs.getRollout() + fc := fake.NewClientBuilder().WithScheme(scheme).WithObjects(rollout).Build() + for _, rs := range rss { + _ = fc.Create(context.TODO(), rs) + } + for _, dep := range deps { + _ = fc.Create(context.TODO(), dep) + } + if br != nil { + _ = fc.Create(context.TODO(), br) + } + ss, in := cs.getNetwork() + _ = fc.Create(context.TODO(), ss[0]) + _ = fc.Create(context.TODO(), in[0]) + r := &RolloutReconciler{ + Client: fc, + Scheme: scheme, + Recorder: record.NewFakeRecorder(10), + finder: util.NewControllerFinder(fc), + trafficRoutingManager: trafficrouting.NewTrafficRoutingManager(fc), + } + r.canaryManager = &canaryReleaseManager{ + Client: fc, + trafficRoutingManager: r.trafficRoutingManager, + recorder: r.Recorder, + } + workload, _ := r.finder.GetWorkloadForRef("", rollout.Spec.ObjectRef.WorkloadRef) + c := &util.RolloutContext{ + Rollout: rollout, + NewStatus: rollout.Status.DeepCopy(), + Workload: workload, + } + err := r.canaryManager.runCanary(c) + if err != nil { + t.Fatalf("reconcileRolloutProgressing failed: %s", err.Error()) + } + checkBatchReleaseEqual(fc, t, client.ObjectKey{Name: rollout.Name}, cs.expectBr()) + cStatus := c.NewStatus.DeepCopy() + cStatus.Message = "" + if cStatus.CanaryStatus != nil { + cStatus.CanaryStatus.LastUpdateTime = nil + } + cond := util.GetRolloutCondition(*cStatus, v1alpha1.RolloutConditionProgressing) + cond.Message = "" + util.SetRolloutCondition(cStatus, *cond) + if !reflect.DeepEqual(cs.expectStatus(), cStatus) { + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(cs.expectStatus()), util.DumpJSON(cStatus)) + } + }) + } +} + +func checkBatchReleaseEqual(c client.WithWatch, t *testing.T, key client.ObjectKey, expect *v1alpha1.BatchRelease) { + obj := &v1alpha1.BatchRelease{} + err := c.Get(context.TODO(), key, obj) + if err != nil { + t.Fatalf("get object failed: %s", err.Error()) + } + if !reflect.DeepEqual(expect.Spec, obj.Spec) { + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.Spec), util.DumpJSON(obj.Spec)) + } +} diff --git a/pkg/controller/rollout/rollout_controller.go b/pkg/controller/rollout/rollout_controller.go index 97519889..b7cc635d 100644 --- a/pkg/controller/rollout/rollout_controller.go +++ b/pkg/controller/rollout/rollout_controller.go @@ -22,7 +22,8 @@ import ( "sync" "time" - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting" "github.com/openkruise/rollouts/pkg/util" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -40,6 +41,8 @@ var ( runtimeController controller.Controller workloadHandler handler.EventHandler watchedWorkload sync.Map + + rolloutControllerKind = v1alpha1.SchemeGroupVersion.WithKind("Rollout") ) func init() { @@ -56,10 +59,12 @@ func init() { // RolloutReconciler reconciles a Rollout object type RolloutReconciler struct { client.Client - Scheme *runtime.Scheme - + Scheme *runtime.Scheme Recorder record.EventRecorder - Finder *util.ControllerFinder + + finder *util.ControllerFinder + trafficRoutingManager *trafficrouting.Manager + canaryManager *canaryReleaseManager } //+kubebuilder:rbac:groups=rollouts.kruise.io,resources=rollouts,verbs=get;list;watch;create;update;patch;delete @@ -86,7 +91,7 @@ type RolloutReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile func (r *RolloutReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Fetch the Rollout instance - rollout := &rolloutv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} err := r.Get(context.TODO(), req.NamespacedName, rollout) if err != nil { if errors.IsNotFound(err) { @@ -111,7 +116,6 @@ func (r *RolloutReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } else if succeeded { watchedWorkload.LoadOrStore(workloadGVK.String(), struct{}{}) klog.Infof("Rollout controller begin to watch workload type: %s", workloadGVK.String()) - // return, and wait informer cache to be synced return ctrl.Result{}, nil } @@ -123,18 +127,16 @@ func (r *RolloutReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, err } // update rollout status - done, err := r.updateRolloutStatus(rollout) - if err != nil { + retry, err := r.updateRolloutStatus(rollout) + if err != nil || retry { return ctrl.Result{}, err - } else if !done { - return ctrl.Result{}, nil } var recheckTime *time.Time switch rollout.Status.Phase { - case rolloutv1alpha1.RolloutPhaseProgressing: + case v1alpha1.RolloutPhaseProgressing: recheckTime, err = r.reconcileRolloutProgressing(rollout) - case rolloutv1alpha1.RolloutPhaseTerminating: + case v1alpha1.RolloutPhaseTerminating: recheckTime, err = r.reconcileRolloutTerminating(rollout) } if err != nil { @@ -153,20 +155,25 @@ func (r *RolloutReconciler) SetupWithManager(mgr ctrl.Manager) error { if err != nil { return err } - // Watch for changes to rollout - if err = c.Watch(&source.Kind{Type: &rolloutv1alpha1.Rollout{}}, &handler.EnqueueRequestForObject{}); err != nil { + if err = c.Watch(&source.Kind{Type: &v1alpha1.Rollout{}}, &handler.EnqueueRequestForObject{}); err != nil { return err } // Watch for changes to batchRelease - if err = c.Watch(&source.Kind{Type: &rolloutv1alpha1.BatchRelease{}}, &enqueueRequestForBatchRelease{reader: mgr.GetCache()}); err != nil { + if err = c.Watch(&source.Kind{Type: &v1alpha1.BatchRelease{}}, &enqueueRequestForBatchRelease{reader: mgr.GetCache()}); err != nil { return err } - runtimeController = c workloadHandler = &enqueueRequestForWorkload{reader: mgr.GetCache(), scheme: r.Scheme} if err = util.AddWorkloadWatcher(c, workloadHandler); err != nil { return err } + r.finder = util.NewControllerFinder(mgr.GetClient()) + r.trafficRoutingManager = trafficrouting.NewTrafficRoutingManager(mgr.GetClient()) + r.canaryManager = &canaryReleaseManager{ + Client: mgr.GetClient(), + trafficRoutingManager: r.trafficRoutingManager, + recorder: r.Recorder, + } return nil } diff --git a/pkg/controller/rollout/rollout_controller_test.go b/pkg/controller/rollout/rollout_controller_test.go index e0831946..722bbbb9 100644 --- a/pkg/controller/rollout/rollout_controller_test.go +++ b/pkg/controller/rollout/rollout_controller_test.go @@ -18,11 +18,15 @@ package rollout import ( kruisev1aplphal "github.com/openkruise/kruise-api/apps/v1alpha1" - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" clientgoscheme "k8s.io/client-go/kubernetes/scheme" utilpointer "k8s.io/utils/pointer" ) @@ -30,46 +34,110 @@ import ( var ( scheme *runtime.Scheme - rolloutDemo = &rolloutv1alpha1.Rollout{ + rolloutDemo = &v1alpha1.Rollout{ ObjectMeta: metav1.ObjectMeta{ Name: "rollout-demo", Labels: map[string]string{}, + Annotations: map[string]string{ + util.RolloutHashAnnotation: "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd", + }, }, - Spec: rolloutv1alpha1.RolloutSpec{ - ObjectRef: rolloutv1alpha1.ObjectRef{ - WorkloadRef: &rolloutv1alpha1.WorkloadRef{ + Spec: v1alpha1.RolloutSpec{ + ObjectRef: v1alpha1.ObjectRef{ + WorkloadRef: &v1alpha1.WorkloadRef{ APIVersion: "apps/v1", Kind: "Deployment", Name: "echoserver", }, }, - Strategy: rolloutv1alpha1.RolloutStrategy{ - Canary: &rolloutv1alpha1.CanaryStrategy{}, + Strategy: v1alpha1.RolloutStrategy{ + Canary: &v1alpha1.CanaryStrategy{ + Steps: []v1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(5), + Replicas: &intstr.IntOrString{IntVal: 1}, + }, + { + Weight: utilpointer.Int32(20), + Replicas: &intstr.IntOrString{IntVal: 2}, + }, + { + Weight: utilpointer.Int32(60), + Replicas: &intstr.IntOrString{IntVal: 6}, + }, + { + Weight: utilpointer.Int32(100), + Replicas: &intstr.IntOrString{IntVal: 10}, + }, + }, + TrafficRoutings: []*v1alpha1.TrafficRouting{ + { + Service: "echoserver", + Ingress: &v1alpha1.IngressTrafficRouting{ + Name: "echoserver", + }, + }, + }, + }, + }, + }, + Status: v1alpha1.RolloutStatus{ + Phase: v1alpha1.RolloutPhaseProgressing, + CanaryStatus: &v1alpha1.CanaryStatus{}, + Conditions: []v1alpha1.RolloutCondition{ + { + Type: v1alpha1.RolloutConditionProgressing, + Reason: v1alpha1.ProgressingReasonInitializing, + Status: corev1.ConditionFalse, + }, }, }, } - + maxUnavailable = intstr.FromString("20%") deploymentDemo = &apps.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", Kind: "Deployment", }, ObjectMeta: metav1.ObjectMeta{ - Name: "echoserver", - Labels: map[string]string{}, - Generation: 1, + Name: "echoserver", + Labels: map[string]string{}, + Annotations: map[string]string{ + util.InRolloutProgressingAnnotation: "rollout-demo", + }, + Generation: 2, UID: types.UID("606132e0-85ef-460a-8cf5-cd8f915a8cc3"), }, Spec: apps.DeploymentSpec{ - Replicas: utilpointer.Int32(100), + Replicas: utilpointer.Int32(10), Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "echoserver", }, }, + Strategy: apps.DeploymentStrategy{ + RollingUpdate: &apps.RollingUpdateDeployment{ + MaxUnavailable: &maxUnavailable, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "echoserver", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + Image: "echoserver:v2", + }, + }, + }, + }, }, Status: apps.DeploymentStatus{ - ObservedGeneration: 1, + ObservedGeneration: 2, }, } @@ -79,9 +147,10 @@ var ( Kind: "ReplicaSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: "echoserver-xxx", + Name: "echoserver-1", Labels: map[string]string{ - "app": "echoserver", + "app": "echoserver", + "pod-template-hash": "pod-template-hash-v1", }, OwnerReferences: []metav1.OwnerReference{ { @@ -93,22 +162,106 @@ var ( }, }, }, + Spec: apps.ReplicaSetSpec{ + Replicas: utilpointer.Int32(10), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "echoserver", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "echoserver", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + Image: "echoserver:v1", + }, + }, + }, + }, + }, } - batchDemo = &rolloutv1alpha1.BatchRelease{ + batchDemo = &v1alpha1.BatchRelease{ ObjectMeta: metav1.ObjectMeta{ - Name: "rollout-demo", - Labels: map[string]string{}, + Name: "rollout-demo", + Labels: map[string]string{}, + Generation: 1, }, - Spec: rolloutv1alpha1.BatchReleaseSpec{ - TargetRef: rolloutv1alpha1.ObjectRef{ - WorkloadRef: &rolloutv1alpha1.WorkloadRef{ + Spec: v1alpha1.BatchReleaseSpec{ + TargetRef: v1alpha1.ObjectRef{ + WorkloadRef: &v1alpha1.WorkloadRef{ APIVersion: "apps/v1", Kind: "Deployment", Name: "echoserver", }, }, }, + Status: v1alpha1.BatchReleaseStatus{}, + } + + demoService = corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "echoserver", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(8080), + }, + }, + Selector: map[string]string{ + "app": "echoserver", + }, + }, + } + + demoIngress = netv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "networking.k8s.io/v1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "echoserver", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + }, + Spec: netv1.IngressSpec{ + Rules: []netv1.IngressRule{ + { + Host: "echoserver.example.com", + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{ + Paths: []netv1.HTTPIngressPath{ + { + Path: "/apis/echo", + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: "echoserver", + Port: netv1.ServiceBackendPort{ + Name: "http", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, } ) @@ -116,5 +269,5 @@ func init() { scheme = runtime.NewScheme() _ = clientgoscheme.AddToScheme(scheme) _ = kruisev1aplphal.AddToScheme(scheme) - _ = rolloutv1alpha1.AddToScheme(scheme) + _ = v1alpha1.AddToScheme(scheme) } diff --git a/pkg/controller/rollout/rollouts_workload_event_handler.go b/pkg/controller/rollout/rollout_event_handler.go similarity index 95% rename from pkg/controller/rollout/rollouts_workload_event_handler.go rename to pkg/controller/rollout/rollout_event_handler.go index 60a5dcbe..6dc2930d 100644 --- a/pkg/controller/rollout/rollouts_workload_event_handler.go +++ b/pkg/controller/rollout/rollout_event_handler.go @@ -20,7 +20,6 @@ import ( "context" rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/controller/rollout/batchrelease" utilclient "github.com/openkruise/rollouts/pkg/util/client" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -121,11 +120,7 @@ func (w *enqueueRequestForBatchRelease) Update(evt event.UpdateEvent, q workqueu } func (w *enqueueRequestForBatchRelease) handleEvent(q workqueue.RateLimitingInterface, obj client.Object) { - rollout := obj.GetLabels()[batchrelease.BatchReleaseOwnerRefLabel] - if rollout == "" { - return - } - klog.Infof("BatchRelease(%s/%s) and reconcile Rollout (%s)", obj.GetNamespace(), obj.GetName(), rollout) - nsn := types.NamespacedName{Namespace: obj.GetNamespace(), Name: rollout} + klog.Infof("BatchRelease(%s/%s) and reconcile Rollout (%s)", obj.GetNamespace(), obj.GetName(), obj.GetName()) + nsn := types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()} q.Add(reconcile.Request{NamespacedName: nsn}) } diff --git a/pkg/controller/rollout/rollouts_workload_event_handler_test.go b/pkg/controller/rollout/rollout_event_handler_test.go similarity index 100% rename from pkg/controller/rollout/rollouts_workload_event_handler_test.go rename to pkg/controller/rollout/rollout_event_handler_test.go diff --git a/pkg/controller/rollout/rollout_progressing.go b/pkg/controller/rollout/rollout_progressing.go new file mode 100644 index 00000000..2a4c2a38 --- /dev/null +++ b/pkg/controller/rollout/rollout_progressing.go @@ -0,0 +1,358 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollout + +import ( + "strconv" + "time" + + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/klog/v2" +) + +var defaultGracePeriodSeconds int32 = 3 + +// parameter1 retryReconcile, parameter2 error +func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *v1alpha1.Rollout) (*time.Time, error) { + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + klog.Infof("reconcile rollout(%s/%s) progressing action...", rollout.Namespace, rollout.Name) + workload, err := r.finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) + if err != nil { + klog.Errorf("rollout(%s/%s) get workload failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return nil, err + } else if workload == nil { + klog.Errorf("rollout(%s/%s) workload Not Found", rollout.Namespace, rollout.Name) + return nil, nil + } else if !workload.IsStatusConsistent { + klog.Infof("rollout(%s/%s) workload status is inconsistent, then wait a moment", rollout.Namespace, rollout.Name) + return nil, nil + } + newStatus := rollout.Status.DeepCopy() + rolloutContext := &util.RolloutContext{Rollout: rollout, NewStatus: newStatus, Workload: workload} + switch cond.Reason { + case v1alpha1.ProgressingReasonInitializing: + klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) + // new canaryStatus + newStatus.CanaryStatus = &v1alpha1.CanaryStatus{ + ObservedWorkloadGeneration: rolloutContext.Workload.Generation, + RolloutHash: rolloutContext.Rollout.Annotations[util.RolloutHashAnnotation], + ObservedRolloutID: getRolloutID(rolloutContext.Workload), + StableRevision: rolloutContext.Workload.StableRevision, + CanaryRevision: rolloutContext.Workload.CanaryRevision, + CurrentStepIndex: 1, + CurrentStepState: v1alpha1.CanaryStepStateUpgrade, + LastUpdateTime: &metav1.Time{Time: time.Now()}, + } + done, _, err := r.doProgressingInitializing(rolloutContext) + if err != nil { + klog.Errorf("rollout(%s/%s) doProgressingInitializing error(%s)", rollout.Namespace, rollout.Name, err.Error()) + return nil, err + } else if done { + progressingStateTransition(newStatus, corev1.ConditionFalse, v1alpha1.ProgressingReasonInRolling, "Rollout is in Progressing") + } else { + // Incomplete, recheck + expectedTime := time.Now().Add(time.Duration(defaultGracePeriodSeconds) * time.Second) + rolloutContext.RecheckTime = &expectedTime + klog.Infof("rollout(%s/%s) doProgressingInitializing is incomplete, and recheck(%s)", rollout.Namespace, rollout.Name, expectedTime.String()) + } + + case v1alpha1.ProgressingReasonInRolling: + klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) + err = r.doProgressingInRolling(rolloutContext) + if err != nil { + return nil, err + } + + case v1alpha1.ProgressingReasonFinalising: + klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) + var done bool + done, err = r.doFinalising(rolloutContext) + if err != nil { + return nil, err + // finalizer is finished + } else if done { + progressingStateTransition(newStatus, corev1.ConditionTrue, v1alpha1.ProgressingReasonSucceeded, "Rollout has been completed, and succeed") + } else { + // Incomplete, recheck + expectedTime := time.Now().Add(time.Duration(defaultGracePeriodSeconds) * time.Second) + rolloutContext.RecheckTime = &expectedTime + klog.Infof("rollout(%s/%s) doProgressingFinalising is incomplete, and recheck(%s)", rollout.Namespace, rollout.Name, expectedTime.String()) + } + + case v1alpha1.ProgressingReasonPaused: + if workload.IsInRollback { + newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision + r.Recorder.Eventf(rollout, corev1.EventTypeNormal, "Progressing", "workload has been rollback, then rollout is canceled") + klog.Infof("rollout(%s/%s) workload has been rollback, then rollout canceled", rollout.Namespace, rollout.Name) + progressingStateTransition(newStatus, corev1.ConditionFalse, v1alpha1.ProgressingReasonCancelling, "The workload has been rolled back and the rollout process will be cancelled") + // from paused to inRolling + } else if !rollout.Spec.Strategy.Paused { + klog.Infof("rollout(%s/%s) is Progressing, but paused", rollout.Namespace, rollout.Name) + progressingStateTransition(newStatus, corev1.ConditionFalse, v1alpha1.ProgressingReasonInRolling, "") + } + + case v1alpha1.ProgressingReasonCancelling: + klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) + var done bool + done, err = r.doFinalising(rolloutContext) + if err != nil { + return nil, err + // finalizer is finished + } else if done { + progressingStateTransition(newStatus, corev1.ConditionFalse, v1alpha1.ProgressingReasonCanceled, "") + } + + case v1alpha1.ProgressingReasonSucceeded, v1alpha1.ProgressingReasonCanceled: + klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason) + } + + err = r.updateRolloutStatusInternal(rollout, *newStatus) + if err != nil { + return nil, err + } + return rolloutContext.RecheckTime, nil +} + +func (r *RolloutReconciler) doProgressingInitializing(c *util.RolloutContext) (bool, string, error) { + // Traffic routing + if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { + if err := r.trafficRoutingManager.InitializeTrafficRouting(c); err != nil { + return false, "", err + } + } + + // It is not allowed to modify the rollout.spec in progressing phase (validate webhook rollout), + // but in many scenarios the user may modify the workload and rollout spec at the same time, + // and there is a possibility that the workload is released first, and due to some network or other reasons the rollout spec is delayed by a few seconds, + // so this is mainly compatible with this scenario. + cond := util.GetRolloutCondition(*c.NewStatus, v1alpha1.RolloutConditionProgressing) + if verifyTime := cond.LastUpdateTime.Add(time.Second * time.Duration(defaultGracePeriodSeconds)); verifyTime.After(time.Now()) { + klog.Infof("verify rollout(%s/%s) TrafficRouting, and wait a moment", c.Rollout.Namespace, c.Rollout.Name) + return false, "", nil + } + return true, "", nil +} + +func (r *RolloutReconciler) doProgressingInRolling(c *util.RolloutContext) error { + // Handle the 5 special cases firstly, and we had better keep the order of following cases: + + switch { + // 1. In case of rollback in a quick way, un-paused and just use workload rolling strategy + case isRollingBackDirectly(c.Rollout, c.Workload): + return r.handleRollbackDirectly(c.Rollout, c.Workload, c.NewStatus) + + // 2. In case of rollout paused, just stop reconcile + case isRolloutPaused(c.Rollout): + return r.handleRolloutPaused(c.Rollout, c.NewStatus) + + // 3. In case of rollback in a batch way, use rollout step strategy + case isRollingBackInBatches(c.Rollout, c.Workload): + return r.handleRollbackInBatches(c.Rollout, c.Workload, c.NewStatus) + + // 4. In case of continuous publishing(v1 -> v2 -> v3), restart publishing + case isContinuousRelease(c.Rollout, c.Workload): + return r.handleContinuousRelease(c) + + // 5. In case of rollout plan changed, recalculate and publishing + case isRolloutPlanChanged(c.Rollout): + return r.handleRolloutPlanChanged(c) + } + return r.handleNormalRolling(c) +} + +func (r *RolloutReconciler) handleRolloutPaused(rollout *v1alpha1.Rollout, newStatus *v1alpha1.RolloutStatus) error { + klog.Infof("rollout(%s/%s) is Progressing, but paused", rollout.Namespace, rollout.Name) + progressingStateTransition(newStatus, corev1.ConditionFalse, v1alpha1.ProgressingReasonPaused, "Rollout has been paused, you can resume it by kube-cli") + return nil +} + +func (r *RolloutReconciler) handleContinuousRelease(c *util.RolloutContext) error { + r.Recorder.Eventf(c.Rollout, corev1.EventTypeNormal, "Progressing", "workload continuous publishing canaryRevision, then restart publishing") + klog.Infof("rollout(%s/%s) workload continuous publishing canaryRevision from(%s) -> to(%s), then restart publishing", + c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.CanaryStatus.CanaryRevision, c.Workload.CanaryRevision) + + done, err := r.doProgressingReset(c) + if err != nil { + klog.Errorf("rollout(%s/%s) doProgressingReset failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return err + } else if done { + c.NewStatus.CanaryStatus = nil + progressingStateTransition(c.NewStatus, corev1.ConditionFalse, v1alpha1.ProgressingReasonInitializing, "Workload is continuous release") + klog.Infof("rollout(%s/%s) workload is continuous publishing, reset complete", c.Rollout.Namespace, c.Rollout.Name) + } else { + // Incomplete, recheck + expectedTime := time.Now().Add(time.Duration(defaultGracePeriodSeconds) * time.Second) + c.RecheckTime = &expectedTime + klog.Infof("rollout(%s/%s) workload is continuous publishing, reset incomplete, and recheck(%s)", c.Rollout.Namespace, c.Rollout.Name, expectedTime.String()) + } + return nil +} + +func (r *RolloutReconciler) handleRollbackDirectly(rollout *v1alpha1.Rollout, workload *util.Workload, newStatus *v1alpha1.RolloutStatus) error { + newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision + r.Recorder.Eventf(rollout, corev1.EventTypeNormal, "Progressing", "workload has been rollback, then rollout is canceled") + klog.Infof("rollout(%s/%s) workload has been rollback directly, then rollout canceled", rollout.Namespace, rollout.Name) + progressingStateTransition(newStatus, corev1.ConditionFalse, v1alpha1.ProgressingReasonCancelling, "The workload has been rolled back and the rollout process will be cancelled") + return nil +} + +func (r *RolloutReconciler) handleRollbackInBatches(rollout *v1alpha1.Rollout, workload *util.Workload, newStatus *v1alpha1.RolloutStatus) error { + // restart from the beginning + newStatus.CanaryStatus.CurrentStepIndex = 1 + newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision + newStatus.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + newStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + newStatus.CanaryStatus.RolloutHash = rollout.Annotations[util.RolloutHashAnnotation] + klog.Infof("rollout(%s/%s) workload has been rollback in batches, then restart from beginning", rollout.Namespace, rollout.Name) + return nil +} + +func (r *RolloutReconciler) handleRolloutPlanChanged(c *util.RolloutContext) error { + newStepIndex, err := r.recalculateCanaryStep(c) + if err != nil { + klog.Errorf("rollout(%s/%s) reCalculate Canary StepIndex failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return err + } + // canary step configuration change causes current step index change + c.NewStatus.CanaryStatus.CurrentStepIndex = newStepIndex + c.NewStatus.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + c.NewStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + c.NewStatus.CanaryStatus.RolloutHash = c.Rollout.Annotations[util.RolloutHashAnnotation] + klog.Infof("rollout(%s/%s) canary step configuration change, and stepIndex(%d) state(%s)", + c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.CanaryStatus.CurrentStepIndex, c.NewStatus.CanaryStatus.CurrentStepState) + return nil +} + +func (r *RolloutReconciler) handleNormalRolling(c *util.RolloutContext) error { + //check if canary is done + if c.NewStatus.CanaryStatus.CurrentStepState == v1alpha1.CanaryStepStateCompleted { + klog.Infof("rollout(%s/%s) progressing rolling done", c.Rollout.Namespace, c.Rollout.Name) + progressingStateTransition(c.NewStatus, corev1.ConditionTrue, v1alpha1.ProgressingReasonFinalising, "Rollout has been completed and some closing work is being done") + } else { // rollout is in rolling + return r.canaryManager.runCanary(c) + } + return nil +} + +/* ********************************************************************** + help functions +*********************************************************************** */ +func isRolloutPaused(rollout *v1alpha1.Rollout) bool { + return rollout.Spec.Strategy.Paused +} + +func isRolloutPlanChanged(rollout *v1alpha1.Rollout) bool { + status := &rollout.Status + return status.CanaryStatus.RolloutHash != "" && status.CanaryStatus.RolloutHash != rollout.Annotations[util.RolloutHashAnnotation] +} + +func isContinuousRelease(rollout *v1alpha1.Rollout, workload *util.Workload) bool { + status := &rollout.Status + return status.CanaryStatus.CanaryRevision != "" && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && !workload.IsInRollback +} + +func isRollingBackDirectly(rollout *v1alpha1.Rollout, workload *util.Workload) bool { + status := &rollout.Status + inBatch := util.IsRollbackInBatchPolicy(rollout, workload.Labels) + return workload.IsInRollback && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && !inBatch +} + +func isRollingBackInBatches(rollout *v1alpha1.Rollout, workload *util.Workload) bool { + status := &rollout.Status + inBatch := util.IsRollbackInBatchPolicy(rollout, workload.Labels) + return workload.IsInRollback && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && inBatch +} + +// 1. modify network api(ingress or gateway api) configuration, and route 100% traffic to stable pods +// 2. remove batchRelease CR. +func (r *RolloutReconciler) doProgressingReset(c *util.RolloutContext) (bool, error) { + if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { + // modify network api(ingress or gateway api) configuration, and route 100% traffic to stable pods + done, err := r.trafficRoutingManager.FinalisingTrafficRouting(c, false) + if err != nil || !done { + return done, err + } + } + done, err := r.canaryManager.removeBatchRelease(c) + if err != nil { + klog.Errorf("rollout(%s/%s) DoFinalising batchRelease failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return false, err + } else if !done { + return false, nil + } + return true, nil +} + +func (r *RolloutReconciler) recalculateCanaryStep(c *util.RolloutContext) (int32, error) { + batch, err := r.canaryManager.fetchBatchRelease(c.Rollout.Namespace, c.Rollout.Name) + if errors.IsNotFound(err) { + return 1, nil + } else if err != nil { + return 0, err + } + currentReplicas, _ := intstr.GetScaledValueFromIntOrPercent(&batch.Spec.ReleasePlan.Batches[*batch.Spec.ReleasePlan.BatchPartition].CanaryReplicas, int(c.Workload.Replicas), true) + var stepIndex int32 + for i := range c.Rollout.Spec.Strategy.Canary.Steps { + step := c.Rollout.Spec.Strategy.Canary.Steps[i] + var desiredReplicas int + if step.Replicas != nil { + desiredReplicas, _ = intstr.GetScaledValueFromIntOrPercent(step.Replicas, int(c.Workload.Replicas), true) + } else { + replicas := intstr.FromString(strconv.Itoa(int(*step.Weight)) + "%") + desiredReplicas, _ = intstr.GetScaledValueFromIntOrPercent(&replicas, int(c.Workload.Replicas), true) + } + stepIndex = int32(i + 1) + if currentReplicas <= desiredReplicas { + break + } + } + return stepIndex, nil +} + +func (r *RolloutReconciler) doFinalising(c *util.RolloutContext) (bool, error) { + klog.Infof("reconcile rollout(%s/%s) doFinalising", c.Rollout.Namespace, c.Rollout.Name) + done, err := r.canaryManager.doCanaryFinalising(c) + if err != nil { + klog.Errorf("rollout(%s/%s) Progressing failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return false, err + } else if !done { + klog.Infof("rollout(%s/%s) finalizer is not finished, and retry reconcile", c.Rollout.Namespace, c.Rollout.Name) + return false, nil + } + klog.Infof("run rollout(%s/%s) Progressing Finalising done", c.Rollout.Namespace, c.Rollout.Name) + return true, nil +} + +func progressingStateTransition(status *v1alpha1.RolloutStatus, condStatus corev1.ConditionStatus, reason, message string) { + cond := util.GetRolloutCondition(*status, v1alpha1.RolloutConditionProgressing) + if cond == nil { + cond = util.NewRolloutCondition(v1alpha1.RolloutConditionProgressing, condStatus, reason, message) + } else { + cond.Status = condStatus + cond.Reason = reason + if message != "" { + cond.Message = message + } + } + util.SetRolloutCondition(status, *cond) + status.Message = cond.Message +} diff --git a/pkg/controller/rollout/rollout_progressing_test.go b/pkg/controller/rollout/rollout_progressing_test.go new file mode 100644 index 00000000..dbf90a44 --- /dev/null +++ b/pkg/controller/rollout/rollout_progressing_test.go @@ -0,0 +1,753 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollout + +import ( + "context" + "reflect" + "testing" + + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting" + "github.com/openkruise/rollouts/pkg/util" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestReconcileRolloutProgressing(t *testing.T) { + cases := []struct { + name string + getObj func() ([]*apps.Deployment, []*apps.ReplicaSet) + getNetwork func() ([]*corev1.Service, []*netv1.Ingress) + getRollout func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) + expectStatus func() *v1alpha1.RolloutStatus + }{ + { + name: "ReconcileRolloutProgressing init -> rolling", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + rs1 := rsDemo.DeepCopy() + return []*apps.Deployment{dep1}, []*apps.ReplicaSet{rs1} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + obj := rolloutDemo.DeepCopy() + return obj, nil + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus.ObservedWorkloadGeneration = 2 + s.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + s.CanaryStatus.StableRevision = "pod-template-hash-v1" + s.CanaryStatus.CanaryRevision = "56855c89f9" + s.CanaryStatus.CurrentStepIndex = 1 + s.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(*s, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(s, *cond) + return s + }, + }, + { + name: "ReconcileRolloutProgressing rolling1", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + dep2 := deploymentDemo.DeepCopy() + dep2.UID = "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180" + dep2.Name = dep1.Name + "-canary" + dep2.Labels[util.CanaryDeploymentLabel] = dep1.Name + rs1 := rsDemo.DeepCopy() + rs2 := rsDemo.DeepCopy() + rs2.Name = "echoserver-canary-2" + rs2.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "Deployment", + Name: dep2.Name, + UID: "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180", + Controller: utilpointer.BoolPtr(true), + }, + } + rs2.Labels["pod-template-hash"] = "pod-template-hash-v2" + rs2.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + return []*apps.Deployment{dep1, dep2}, []*apps.ReplicaSet{rs1, rs2} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + obj := rolloutDemo.DeepCopy() + obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 + obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" + obj.Status.CanaryStatus.CanaryRevision = "56855c89f9" + obj.Status.CanaryStatus.CurrentStepIndex = 1 + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(&obj.Status, *cond) + return obj, nil + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus.ObservedWorkloadGeneration = 2 + s.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + s.CanaryStatus.StableRevision = "pod-template-hash-v1" + s.CanaryStatus.CanaryRevision = "56855c89f9" + s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + s.CanaryStatus.CurrentStepIndex = 1 + s.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(*s, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(s, *cond) + return s + }, + }, + { + name: "ReconcileRolloutProgressing rolling -> finalizing", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + dep2 := deploymentDemo.DeepCopy() + dep2.UID = "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180" + dep2.Name = dep1.Name + "-canary" + dep2.Labels[util.CanaryDeploymentLabel] = dep1.Name + rs1 := rsDemo.DeepCopy() + rs2 := rsDemo.DeepCopy() + rs2.Name = "echoserver-canary-2" + rs2.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "Deployment", + Name: dep2.Name, + UID: "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180", + Controller: utilpointer.BoolPtr(true), + }, + } + rs2.Labels["pod-template-hash"] = "pod-template-hash-v2" + rs2.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + return []*apps.Deployment{dep1, dep2}, []*apps.ReplicaSet{rs1, rs2} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + obj := rolloutDemo.DeepCopy() + obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 + obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" + obj.Status.CanaryStatus.CanaryRevision = "56855c89f9" + obj.Status.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + obj.Status.CanaryStatus.CurrentStepIndex = 4 + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(&obj.Status, *cond) + return obj, nil + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus.ObservedWorkloadGeneration = 2 + s.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + s.CanaryStatus.StableRevision = "pod-template-hash-v1" + s.CanaryStatus.CanaryRevision = "56855c89f9" + s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + cond := util.GetRolloutCondition(*s, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonFinalising + cond.Status = corev1.ConditionTrue + util.SetRolloutCondition(s, *cond) + return s + }, + }, + { + name: "ReconcileRolloutProgressing finalizing", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + delete(dep1.Annotations, util.InRolloutProgressingAnnotation) + dep1.Status = apps.DeploymentStatus{ + ObservedGeneration: 2, + Replicas: 10, + UpdatedReplicas: 5, + ReadyReplicas: 10, + AvailableReplicas: 10, + } + rs1 := rsDemo.DeepCopy() + return []*apps.Deployment{dep1}, []*apps.ReplicaSet{rs1} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + obj := rolloutDemo.DeepCopy() + obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 + obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" + obj.Status.CanaryStatus.CanaryRevision = "56855c89f9" + obj.Status.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + obj.Status.CanaryStatus.CurrentStepIndex = 4 + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonFinalising + cond.Status = corev1.ConditionTrue + util.SetRolloutCondition(&obj.Status, *cond) + return obj, nil + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus.ObservedWorkloadGeneration = 2 + s.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + s.CanaryStatus.StableRevision = "pod-template-hash-v1" + s.CanaryStatus.CanaryRevision = "56855c89f9" + s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + cond := util.GetRolloutCondition(*s, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonFinalising + cond.Status = corev1.ConditionTrue + util.SetRolloutCondition(s, *cond) + return s + }, + }, + { + name: "ReconcileRolloutProgressing finalizing -> succeeded", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + delete(dep1.Annotations, util.InRolloutProgressingAnnotation) + dep1.Status = apps.DeploymentStatus{ + ObservedGeneration: 2, + Replicas: 10, + UpdatedReplicas: 10, + ReadyReplicas: 10, + AvailableReplicas: 10, + } + rs1 := rsDemo.DeepCopy() + return []*apps.Deployment{dep1}, []*apps.ReplicaSet{rs1} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + obj := rolloutDemo.DeepCopy() + obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 + obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" + obj.Status.CanaryStatus.CanaryRevision = "56855c89f9" + obj.Status.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + obj.Status.CanaryStatus.CurrentStepIndex = 4 + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonFinalising + cond.Status = corev1.ConditionTrue + util.SetRolloutCondition(&obj.Status, *cond) + return obj, nil + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus.ObservedWorkloadGeneration = 2 + s.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + s.CanaryStatus.StableRevision = "pod-template-hash-v1" + s.CanaryStatus.CanaryRevision = "56855c89f9" + s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + cond := util.GetRolloutCondition(*s, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonSucceeded + cond.Status = corev1.ConditionTrue + util.SetRolloutCondition(s, *cond) + return s + }, + }, + { + name: "ReconcileRolloutProgressing rolling -> rollback", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + dep1.Spec.Template.Spec.Containers[0].Image = "echoserver:v1" + dep2 := deploymentDemo.DeepCopy() + dep2.UID = "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180" + dep2.Name = dep1.Name + "-canary" + dep2.Labels[util.CanaryDeploymentLabel] = dep1.Name + rs1 := rsDemo.DeepCopy() + rs2 := rsDemo.DeepCopy() + rs2.Name = "echoserver-canary-2" + rs2.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "Deployment", + Name: dep2.Name, + UID: "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180", + Controller: utilpointer.BoolPtr(true), + }, + } + rs2.Labels["pod-template-hash"] = "pod-template-hash-v2" + rs2.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + return []*apps.Deployment{dep1, dep2}, []*apps.ReplicaSet{rs1, rs2} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + obj := rolloutDemo.DeepCopy() + obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 + obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" + obj.Status.CanaryStatus.CanaryRevision = "56855c89f9" + obj.Status.CanaryStatus.CurrentStepIndex = 1 + obj.Status.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(&obj.Status, *cond) + return obj, nil + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus.ObservedWorkloadGeneration = 2 + s.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + s.CanaryStatus.StableRevision = "pod-template-hash-v1" + s.CanaryStatus.CanaryRevision = "5d48f79ff8" + s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + s.CanaryStatus.CurrentStepIndex = 1 + s.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(*s, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonCancelling + util.SetRolloutCondition(s, *cond) + return s + }, + }, + { + name: "ReconcileRolloutProgressing rolling -> rollback", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + dep1.Spec.Template.Spec.Containers[0].Image = "echoserver:v1" + dep2 := deploymentDemo.DeepCopy() + dep2.UID = "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180" + dep2.Name = dep1.Name + "-canary" + dep2.Labels[util.CanaryDeploymentLabel] = dep1.Name + rs1 := rsDemo.DeepCopy() + rs2 := rsDemo.DeepCopy() + rs2.Name = "echoserver-canary-2" + rs2.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "Deployment", + Name: dep2.Name, + UID: "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180", + Controller: utilpointer.BoolPtr(true), + }, + } + rs2.Labels["pod-template-hash"] = "pod-template-hash-v2" + rs2.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + return []*apps.Deployment{dep1, dep2}, []*apps.ReplicaSet{rs1, rs2} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + obj := rolloutDemo.DeepCopy() + obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 + obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" + obj.Status.CanaryStatus.CanaryRevision = "56855c89f9" + obj.Status.CanaryStatus.CurrentStepIndex = 1 + obj.Status.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(&obj.Status, *cond) + return obj, nil + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus.ObservedWorkloadGeneration = 2 + s.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + s.CanaryStatus.StableRevision = "pod-template-hash-v1" + s.CanaryStatus.CanaryRevision = "5d48f79ff8" + s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + s.CanaryStatus.CurrentStepIndex = 1 + s.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(*s, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonCancelling + util.SetRolloutCondition(s, *cond) + return s + }, + }, + { + name: "ReconcileRolloutProgressing rolling -> continueRelease", + getObj: func() ([]*apps.Deployment, []*apps.ReplicaSet) { + dep1 := deploymentDemo.DeepCopy() + dep1.Spec.Template.Spec.Containers[0].Image = "echoserver:v3" + dep2 := deploymentDemo.DeepCopy() + dep2.UID = "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180" + dep2.Name = dep1.Name + "-canary" + dep2.Labels[util.CanaryDeploymentLabel] = dep1.Name + rs1 := rsDemo.DeepCopy() + rs2 := rsDemo.DeepCopy() + rs2.Name = "echoserver-canary-2" + rs2.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "Deployment", + Name: dep2.Name, + UID: "1ca4d850-9ec3-48bd-84cb-19f2e8cf4180", + Controller: utilpointer.BoolPtr(true), + }, + } + rs2.Labels["pod-template-hash"] = "pod-template-hash-v2" + rs2.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + return []*apps.Deployment{dep1, dep2}, []*apps.ReplicaSet{rs1, rs2} + }, + getNetwork: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *v1alpha1.BatchRelease) { + obj := rolloutDemo.DeepCopy() + obj.Status.CanaryStatus.ObservedWorkloadGeneration = 2 + obj.Status.CanaryStatus.RolloutHash = "f55bvd874d5f2fzvw46bv966x4bwbdv4wx6bd9f7b46ww788954b8z8w29b7wxfd" + obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" + obj.Status.CanaryStatus.CanaryRevision = "56855c89f9" + obj.Status.CanaryStatus.CurrentStepIndex = 3 + obj.Status.CanaryStatus.CanaryReplicas = 5 + obj.Status.CanaryStatus.CanaryReadyReplicas = 3 + obj.Status.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateUpgrade + cond := util.GetRolloutCondition(obj.Status, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInRolling + util.SetRolloutCondition(&obj.Status, *cond) + return obj, nil + }, + expectStatus: func() *v1alpha1.RolloutStatus { + s := rolloutDemo.Status.DeepCopy() + s.CanaryStatus = nil + cond := util.GetRolloutCondition(*s, v1alpha1.RolloutConditionProgressing) + cond.Reason = v1alpha1.ProgressingReasonInitializing + util.SetRolloutCondition(s, *cond) + return s + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + deps, rss := cs.getObj() + rollout, br := cs.getRollout() + fc := fake.NewClientBuilder().WithScheme(scheme).WithObjects(rollout).Build() + for _, rs := range rss { + _ = fc.Create(context.TODO(), rs) + } + for _, dep := range deps { + _ = fc.Create(context.TODO(), dep) + } + if br != nil { + _ = fc.Create(context.TODO(), br) + } + ss, in := cs.getNetwork() + _ = fc.Create(context.TODO(), ss[0]) + _ = fc.Create(context.TODO(), in[0]) + r := &RolloutReconciler{ + Client: fc, + Scheme: scheme, + Recorder: record.NewFakeRecorder(10), + finder: util.NewControllerFinder(fc), + trafficRoutingManager: trafficrouting.NewTrafficRoutingManager(fc), + } + r.canaryManager = &canaryReleaseManager{ + Client: fc, + trafficRoutingManager: r.trafficRoutingManager, + recorder: r.Recorder, + } + _, err := r.reconcileRolloutProgressing(rollout) + if err != nil { + t.Fatalf("reconcileRolloutProgressing failed: %s", err.Error()) + } + checkRolloutEqual(fc, t, client.ObjectKey{Name: rollout.Name}, cs.expectStatus()) + }) + } +} + +func checkRolloutEqual(c client.WithWatch, t *testing.T, key client.ObjectKey, expect *v1alpha1.RolloutStatus) { + obj := &v1alpha1.Rollout{} + err := c.Get(context.TODO(), key, obj) + if err != nil { + t.Fatalf("get object failed: %s", err.Error()) + } + cStatus := obj.Status.DeepCopy() + cStatus.Message = "" + if cStatus.CanaryStatus != nil { + cStatus.CanaryStatus.LastUpdateTime = nil + } + cond := util.GetRolloutCondition(*cStatus, v1alpha1.RolloutConditionProgressing) + cond.Message = "" + util.SetRolloutCondition(cStatus, *cond) + if !reflect.DeepEqual(expect, cStatus) { + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(cStatus)) + } +} + +func TestReCalculateCanaryStepIndex(t *testing.T) { + cases := []struct { + name string + getObj func() (*apps.Deployment, *apps.ReplicaSet) + getRollout func() *v1alpha1.Rollout + getBatchRelease func() *v1alpha1.BatchRelease + expectStepIndex int32 + }{ + { + name: "steps changed v1", + getObj: func() (*apps.Deployment, *apps.ReplicaSet) { + obj := deploymentDemo.DeepCopy() + return obj, rsDemo.DeepCopy() + }, + getRollout: func() *v1alpha1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(20), + }, + { + Weight: utilpointer.Int32(50), + }, + { + Weight: utilpointer.Int32(100), + }, + } + return obj + }, + getBatchRelease: func() *v1alpha1.BatchRelease { + obj := batchDemo.DeepCopy() + obj.Spec.ReleasePlan.Batches = []v1alpha1.ReleaseBatch{ + { + CanaryReplicas: intstr.FromString("40%"), + }, + { + CanaryReplicas: intstr.FromString("60%"), + }, + { + CanaryReplicas: intstr.FromString("100%"), + }, + } + obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) + return obj + }, + expectStepIndex: 2, + }, + { + name: "steps changed v2", + getObj: func() (*apps.Deployment, *apps.ReplicaSet) { + obj := deploymentDemo.DeepCopy() + return obj, rsDemo.DeepCopy() + }, + getRollout: func() *v1alpha1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(20), + }, + { + Weight: utilpointer.Int32(40), + }, + { + Weight: utilpointer.Int32(100), + }, + } + return obj + }, + getBatchRelease: func() *v1alpha1.BatchRelease { + obj := batchDemo.DeepCopy() + obj.Spec.ReleasePlan.Batches = []v1alpha1.ReleaseBatch{ + { + CanaryReplicas: intstr.FromString("40%"), + }, + { + CanaryReplicas: intstr.FromString("60%"), + }, + { + CanaryReplicas: intstr.FromString("100%"), + }, + } + obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) + return obj + }, + expectStepIndex: 2, + }, + { + name: "steps changed v3", + getObj: func() (*apps.Deployment, *apps.ReplicaSet) { + obj := deploymentDemo.DeepCopy() + return obj, rsDemo.DeepCopy() + }, + getRollout: func() *v1alpha1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(40), + }, + { + Weight: utilpointer.Int32(60), + }, + { + Weight: utilpointer.Int32(100), + }, + } + return obj + }, + getBatchRelease: func() *v1alpha1.BatchRelease { + obj := batchDemo.DeepCopy() + obj.Spec.ReleasePlan.Batches = []v1alpha1.ReleaseBatch{ + { + CanaryReplicas: intstr.FromString("20%"), + }, + { + CanaryReplicas: intstr.FromString("40%"), + }, + { + CanaryReplicas: intstr.FromString("100%"), + }, + } + obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(1) + return obj + }, + expectStepIndex: 1, + }, + { + name: "steps changed v4", + getObj: func() (*apps.Deployment, *apps.ReplicaSet) { + obj := deploymentDemo.DeepCopy() + return obj, rsDemo.DeepCopy() + }, + getRollout: func() *v1alpha1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(10), + }, + { + Weight: utilpointer.Int32(30), + }, + { + Weight: utilpointer.Int32(100), + }, + } + return obj + }, + getBatchRelease: func() *v1alpha1.BatchRelease { + obj := batchDemo.DeepCopy() + obj.Spec.ReleasePlan.Batches = []v1alpha1.ReleaseBatch{ + { + CanaryReplicas: intstr.FromString("20%"), + }, + { + CanaryReplicas: intstr.FromString("40%"), + }, + { + CanaryReplicas: intstr.FromString("100%"), + }, + } + obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) + return obj + }, + expectStepIndex: 2, + }, + { + name: "steps changed v5", + getObj: func() (*apps.Deployment, *apps.ReplicaSet) { + obj := deploymentDemo.DeepCopy() + return obj, rsDemo.DeepCopy() + }, + getRollout: func() *v1alpha1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(2), + Replicas: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "10%", + }, + }, + { + Weight: utilpointer.Int32(3), + Replicas: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "10%", + }, + }, + } + return obj + }, + getBatchRelease: func() *v1alpha1.BatchRelease { + obj := batchDemo.DeepCopy() + obj.Spec.ReleasePlan.Batches = []v1alpha1.ReleaseBatch{ + { + CanaryReplicas: intstr.FromString("10%"), + }, + { + CanaryReplicas: intstr.FromString("20%"), + }, + { + CanaryReplicas: intstr.FromString("30%"), + }, + } + obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) + return obj + }, + expectStepIndex: 1, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + client := fake.NewClientBuilder().WithScheme(scheme).Build() + client.Create(context.TODO(), cs.getBatchRelease()) + dep, rs := cs.getObj() + client.Create(context.TODO(), dep) + client.Create(context.TODO(), rs) + client.Create(context.TODO(), cs.getRollout()) + reconciler := &RolloutReconciler{ + Client: client, + Scheme: scheme, + finder: util.NewControllerFinder(client), + } + reconciler.canaryManager = &canaryReleaseManager{ + Client: client, + trafficRoutingManager: reconciler.trafficRoutingManager, + recorder: reconciler.Recorder, + } + rollout := cs.getRollout() + workload, err := reconciler.finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) + if err != nil { + t.Fatalf(err.Error()) + } + c := &util.RolloutContext{Rollout: rollout, Workload: workload} + newStepIndex, err := reconciler.recalculateCanaryStep(c) + if err != nil { + t.Fatalf(err.Error()) + } + if cs.expectStepIndex != newStepIndex { + t.Fatalf("expect %d, but %d", cs.expectStepIndex, newStepIndex) + } + }) + } +} diff --git a/pkg/controller/rollout/rollout_status.go b/pkg/controller/rollout/rollout_status.go new file mode 100644 index 00000000..fde40928 --- /dev/null +++ b/pkg/controller/rollout/rollout_status.go @@ -0,0 +1,228 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollout + +import ( + "context" + "reflect" + "time" + + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func (r *RolloutReconciler) updateRolloutStatus(rollout *v1alpha1.Rollout) (retry bool, err error) { + newStatus := *rollout.Status.DeepCopy() + newStatus.ObservedGeneration = rollout.GetGeneration() + defer func() { + err = r.updateRolloutStatusInternal(rollout, newStatus) + }() + + // delete rollout CRD + if !rollout.DeletionTimestamp.IsZero() { + if newStatus.Phase != v1alpha1.RolloutPhaseTerminating { + newStatus.Phase = v1alpha1.RolloutPhaseTerminating + cond := util.NewRolloutCondition(v1alpha1.RolloutConditionTerminating, corev1.ConditionFalse, v1alpha1.TerminatingReasonInTerminating, "Rollout is in terminating") + util.SetRolloutCondition(&newStatus, *cond) + } + return false, nil + } + if newStatus.Phase == "" { + newStatus.Phase = v1alpha1.RolloutPhaseInitial + } + // get ref workload + workload, err := r.finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) + if err != nil { + klog.Errorf("rollout(%s/%s) get workload failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return false, err + } else if workload == nil { + newStatus = v1alpha1.RolloutStatus{ + ObservedGeneration: rollout.Generation, + Phase: v1alpha1.RolloutPhaseInitial, + Message: "Workload Not Found", + } + klog.Infof("rollout(%s/%s) workload not found, and reset status be Initial", rollout.Namespace, rollout.Name) + return true, nil + } + // workload status generation is not equal to workload.generation + if !workload.IsStatusConsistent { + klog.Infof("rollout(%s/%s) workload status is inconsistent, then wait a moment", rollout.Namespace, rollout.Name) + return true, nil + } + + // update workload generation to canaryStatus.ObservedWorkloadGeneration + // rollout is a target ref bypass, so there needs to be a field to identify the rollout execution process or results, + // which version of deployment is targeted, ObservedWorkloadGeneration that is to compare with the workload generation + if newStatus.CanaryStatus != nil && newStatus.CanaryStatus.CanaryRevision != "" && + newStatus.CanaryStatus.CanaryRevision == workload.CanaryRevision { + newStatus.CanaryStatus.ObservedRolloutID = getRolloutID(workload) + newStatus.CanaryStatus.ObservedWorkloadGeneration = workload.Generation + } + + switch newStatus.Phase { + case v1alpha1.RolloutPhaseInitial: + klog.Infof("rollout(%s/%s) status phase from(%s) -> to(%s)", rollout.Namespace, rollout.Name, v1alpha1.RolloutPhaseInitial, v1alpha1.RolloutPhaseHealthy) + newStatus.Phase = v1alpha1.RolloutPhaseHealthy + newStatus.Message = "rollout is healthy" + case v1alpha1.RolloutPhaseHealthy: + // workload released, entering the rollout progressing phase + if workload.InRolloutProgressing { + klog.Infof("rollout(%s/%s) status phase from(%s) -> to(%s)", rollout.Namespace, rollout.Name, v1alpha1.RolloutPhaseHealthy, v1alpha1.RolloutPhaseProgressing) + newStatus.Phase = v1alpha1.RolloutPhaseProgressing + cond := util.NewRolloutCondition(v1alpha1.RolloutConditionProgressing, corev1.ConditionFalse, v1alpha1.ProgressingReasonInitializing, "Rollout is in Progressing") + util.SetRolloutCondition(&newStatus, *cond) + } else if newStatus.CanaryStatus == nil { + // The following logic is to make PaaS be able to judge whether the rollout is ready + // at the first deployment of the Rollout/Workload. For example: generally, a PaaS + // platform can use the following code to judge whether the rollout progression is completed: + // ``` + // if getRolloutID(workload, rollout) == rollout.Status.CanaryStatus.ObservedRolloutID && + // rollout.Status.CanaryStatus.CurrentStepState == "Completed" { + // // do something after rollout + // } + //``` + // But at the first deployment of Rollout/Workload, CanaryStatus isn't set due to no rollout progression, + // and PaaS platform cannot judge whether the deployment is completed base on the code above. So we have + // to update the status just like the rollout was completed. + + newStatus.CanaryStatus = &v1alpha1.CanaryStatus{ + ObservedRolloutID: getRolloutID(workload), + ObservedWorkloadGeneration: workload.Generation, + PodTemplateHash: workload.PodTemplateHash, + CanaryRevision: workload.CanaryRevision, + StableRevision: workload.StableRevision, + CurrentStepIndex: int32(len(rollout.Spec.Strategy.Canary.Steps)), + CurrentStepState: v1alpha1.CanaryStepStateCompleted, + } + newStatus.Message = "workload deployment is completed" + } + case v1alpha1.RolloutPhaseProgressing: + cond := util.GetRolloutCondition(newStatus, v1alpha1.RolloutConditionProgressing) + if cond == nil || cond.Reason == v1alpha1.ProgressingReasonSucceeded || cond.Reason == v1alpha1.ProgressingReasonCanceled { + newStatus.Phase = v1alpha1.RolloutPhaseHealthy + } + } + return false, nil +} + +func (r *RolloutReconciler) updateRolloutStatusInternal(rollout *v1alpha1.Rollout, newStatus v1alpha1.RolloutStatus) error { + spec := rollout.Spec.DeepCopy() + // ignore paused filed + spec.Strategy.Paused = false + data := util.DumpJSON(spec) + // rollout.spec hash + rolloutHash := rand.SafeEncodeString(util.EncodeHash(data)) + if reflect.DeepEqual(rollout.Status, newStatus) && rollout.Annotations[util.RolloutHashAnnotation] == rolloutHash { + return nil + } + rolloutClone := rollout.DeepCopy() + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err := r.Client.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: rollout.Name}, rolloutClone); err != nil { + klog.Errorf("error getting updated rollout(%s/%s) from client", rollout.Namespace, rollout.Name) + return err + } + rolloutClone.Status = newStatus + if rolloutClone.Annotations == nil { + rolloutClone.Annotations = map[string]string{} + } + rolloutClone.Annotations[util.RolloutHashAnnotation] = rolloutHash + return r.Client.Status().Update(context.TODO(), rolloutClone) + }); err != nil { + klog.Errorf("update rollout(%s/%s) status failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return err + } + rollout.Status = newStatus + if rollout.Annotations == nil { + rollout.Annotations = map[string]string{} + } + rollout.Annotations[util.RolloutHashAnnotation] = rolloutHash + klog.Infof("rollout(%s/%s) status from(%s) -> to(%s) success", rollout.Namespace, rollout.Name, util.DumpJSON(rollout.Status), util.DumpJSON(newStatus)) + return nil +} + +func (r *RolloutReconciler) reconcileRolloutTerminating(rollout *v1alpha1.Rollout) (*time.Time, error) { + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionTerminating) + if cond.Reason == v1alpha1.TerminatingReasonCompleted { + return nil, nil + } + newStatus := rollout.Status.DeepCopy() + workload, err := r.finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) + if err != nil { + klog.Errorf("rollout(%s/%s) get workload failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return nil, err + } + c := &util.RolloutContext{Rollout: rollout, NewStatus: newStatus, Workload: workload} + done, err := r.doFinalising(c) + if err != nil { + return nil, err + } else if done { + klog.Infof("rollout(%s/%s) is terminating, and state from(%s) -> to(%s)", rollout.Namespace, rollout.Name, cond.Reason, v1alpha1.TerminatingReasonCompleted) + cond.Reason = v1alpha1.TerminatingReasonCompleted + cond.Status = corev1.ConditionTrue + util.SetRolloutCondition(newStatus, *cond) + } + err = r.updateRolloutStatusInternal(rollout, *newStatus) + if err != nil { + return nil, err + } + return c.RecheckTime, nil +} + +// handle adding and handle finalizer logic, it turns if we should continue to reconcile +func (r *RolloutReconciler) handleFinalizer(rollout *v1alpha1.Rollout) error { + // delete rollout crd, remove finalizer + if !rollout.DeletionTimestamp.IsZero() { + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionTerminating) + if cond != nil && cond.Reason == v1alpha1.TerminatingReasonCompleted { + // Completed + if controllerutil.ContainsFinalizer(rollout, util.KruiseRolloutFinalizer) { + err := util.UpdateFinalizer(r.Client, rollout, util.RemoveFinalizerOpType, util.KruiseRolloutFinalizer) + if err != nil { + klog.Errorf("remove rollout(%s/%s) finalizer failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return err + } + klog.Infof("remove rollout(%s/%s) finalizer success", rollout.Namespace, rollout.Name) + } + return nil + } + return nil + } + + // create rollout crd, add finalizer + if !controllerutil.ContainsFinalizer(rollout, util.KruiseRolloutFinalizer) { + err := util.UpdateFinalizer(r.Client, rollout, util.AddFinalizerOpType, util.KruiseRolloutFinalizer) + if err != nil { + klog.Errorf("register rollout(%s/%s) finalizer failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return err + } + klog.Infof("register rollout(%s/%s) finalizer success", rollout.Namespace, rollout.Name) + } + return nil +} + +func getRolloutID(workload *util.Workload) string { + if workload != nil { + return workload.Labels[v1alpha1.RolloutIDLabel] + } + return "" +} diff --git a/pkg/controller/rollout/status.go b/pkg/controller/rollout/status.go deleted file mode 100644 index c74ce129..00000000 --- a/pkg/controller/rollout/status.go +++ /dev/null @@ -1,198 +0,0 @@ -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rollout - -import ( - "context" - "crypto/sha256" - "encoding/json" - "fmt" - "reflect" - - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/util" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/rand" - "k8s.io/client-go/util/retry" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func (r *RolloutReconciler) updateRolloutStatus(rollout *rolloutv1alpha1.Rollout) (done bool, err error) { - newStatus := *rollout.Status.DeepCopy() - newStatus.ObservedGeneration = rollout.GetGeneration() - defer func() { - err = r.updateRolloutStatusInternal(rollout, newStatus) - if err != nil { - klog.Errorf("update rollout(%s/%s) status failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return - } - err = r.calculateRolloutHash(rollout) - if err != nil { - return - } - rollout.Status = newStatus - }() - - // delete rollout CRD - if !rollout.DeletionTimestamp.IsZero() && newStatus.Phase != rolloutv1alpha1.RolloutPhaseTerminating { - newStatus.Phase = rolloutv1alpha1.RolloutPhaseTerminating - cond := util.NewRolloutCondition(rolloutv1alpha1.RolloutConditionTerminating, corev1.ConditionFalse, rolloutv1alpha1.TerminatingReasonInTerminating, "Rollout is in terminating") - util.SetRolloutCondition(&newStatus, *cond) - } else if newStatus.Phase == "" { - newStatus.Phase = rolloutv1alpha1.RolloutPhaseInitial - } - // get ref workload - workload, err := r.Finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) - if err != nil { - klog.Errorf("rollout(%s/%s) get workload failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return - } else if workload == nil { - if rollout.DeletionTimestamp.IsZero() { - resetStatus(&newStatus) - klog.Infof("rollout(%s/%s) workload not found, and reset status be Initial", rollout.Namespace, rollout.Name) - } - done = true - return - } - - // workload status is not consistent - if !workload.IsStatusConsistent { - klog.Infof("rollout(%s/%s) workload status isn't consistent, then wait a moment", rollout.Namespace, rollout.Name) - done = false - return - } - newStatus.StableRevision = workload.StableRevision - // update workload generation to canaryStatus.ObservedWorkloadGeneration - // rollout is a target ref bypass, so there needs to be a field to identify the rollout execution process or results, - // which version of deployment is targeted, ObservedWorkloadGeneration that is to compare with the workload generation - if newStatus.CanaryStatus != nil && newStatus.CanaryStatus.CanaryRevision != "" && - newStatus.CanaryStatus.CanaryRevision == workload.CanaryRevision { - newStatus.CanaryStatus.ObservedRolloutID = getRolloutID(workload, rollout) - newStatus.CanaryStatus.ObservedWorkloadGeneration = workload.Generation - } - - switch newStatus.Phase { - case rolloutv1alpha1.RolloutPhaseInitial: - klog.Infof("rollout(%s/%s) status phase from(%s) -> to(%s)", rollout.Namespace, rollout.Name, rolloutv1alpha1.RolloutPhaseInitial, rolloutv1alpha1.RolloutPhaseHealthy) - newStatus.Phase = rolloutv1alpha1.RolloutPhaseHealthy - newStatus.Message = "rollout is healthy" - case rolloutv1alpha1.RolloutPhaseHealthy: - if workload.InRolloutProgressing { - // from healthy to progressing - klog.Infof("rollout(%s/%s) status phase from(%s) -> to(%s)", rollout.Namespace, rollout.Name, rolloutv1alpha1.RolloutPhaseHealthy, rolloutv1alpha1.RolloutPhaseProgressing) - newStatus.Phase = rolloutv1alpha1.RolloutPhaseProgressing - cond := util.NewRolloutCondition(rolloutv1alpha1.RolloutConditionProgressing, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonInitializing, "Rollout is in Progressing") - util.SetRolloutCondition(&newStatus, *cond) - } else if newStatus.CanaryStatus == nil { - // The following logic is to make PaaS be able to judge whether the rollout is ready - // at the first deployment of the Rollout/Workload. For example: generally, a PaaS - // platform can use the following code to judge whether the rollout progression is completed: - // ``` - // if getRolloutID(workload, rollout) == rollout.Status.CanaryStatus.ObservedRolloutID && - // rollout.Status.CanaryStatus.CurrentStepState == "Completed" { - // // do something after rollout - // } - //``` - // But at the first deployment of Rollout/Workload, CanaryStatus isn't set due to no rollout progression, - // and PaaS platform cannot judge whether the deployment is completed base on the code above. So we have - // to update the status just like the rollout was completed. - - newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{ - CanaryReplicas: workload.CanaryReplicas, - CanaryReadyReplicas: workload.CanaryReadyReplicas, - ObservedRolloutID: getRolloutID(workload, rollout), - ObservedWorkloadGeneration: workload.Generation, - PodTemplateHash: workload.PodTemplateHash, - CanaryRevision: workload.CanaryRevision, - CurrentStepIndex: int32(len(rollout.Spec.Strategy.Canary.Steps)), - CurrentStepState: rolloutv1alpha1.CanaryStepStateCompleted, - } - newStatus.Message = "workload deployment is completed" - } - case rolloutv1alpha1.RolloutPhaseProgressing: - cond := util.GetRolloutCondition(newStatus, rolloutv1alpha1.RolloutConditionProgressing) - if cond == nil || cond.Reason == rolloutv1alpha1.ProgressingReasonSucceeded || cond.Reason == rolloutv1alpha1.ProgressingReasonCanceled { - newStatus.Phase = rolloutv1alpha1.RolloutPhaseHealthy - } - } - done = true - return -} - -func (r *RolloutReconciler) updateRolloutStatusInternal(rollout *rolloutv1alpha1.Rollout, newStatus rolloutv1alpha1.RolloutStatus) error { - if reflect.DeepEqual(rollout.Status, newStatus) { - return nil - } - rolloutClone := rollout.DeepCopy() - if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if err := r.Client.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: rollout.Name}, rolloutClone); err != nil { - klog.Errorf("error getting updated rollout(%s/%s) from client", rollout.Namespace, rollout.Name) - return err - } - rolloutClone.Status = newStatus - if err := r.Client.Status().Update(context.TODO(), rolloutClone); err != nil { - return err - } - return nil - }); err != nil { - return err - } - oldBy, _ := json.Marshal(rollout.Status) - newBy, _ := json.Marshal(newStatus) - klog.Infof("rollout(%s/%s) status from(%s) -> to(%s)", rollout.Namespace, rollout.Name, string(oldBy), string(newBy)) - return nil -} - -// ResetStatus resets the status of the rollout to start from beginning -func resetStatus(status *rolloutv1alpha1.RolloutStatus) { - status.StableRevision = "" - //util.RemoveRolloutCondition(status, rolloutv1alpha1.RolloutConditionProgressing) - status.Phase = rolloutv1alpha1.RolloutPhaseInitial - status.Message = "workload not found" -} - -func (r *RolloutReconciler) calculateRolloutHash(rollout *rolloutv1alpha1.Rollout) error { - spec := rollout.Spec.DeepCopy() - // ignore paused filed - spec.Strategy.Paused = false - data := util.DumpJSON(spec) - hash := rand.SafeEncodeString(hash(data)) - if rollout.Annotations[util.RolloutHashAnnotation] == hash { - return nil - } - // update rollout hash in annotation - cloneObj := rollout.DeepCopy() - body := fmt.Sprintf(`{"metadata":{"annotations":{"%s":"%s"}}}`, util.RolloutHashAnnotation, hash) - err := r.Patch(context.TODO(), cloneObj, client.RawPatch(types.MergePatchType, []byte(body))) - if err != nil { - klog.Errorf("rollout(%s/%s) patch(%s) failed: %s", rollout.Namespace, rollout.Name, body, err.Error()) - return err - } - if rollout.Annotations == nil { - rollout.Annotations = map[string]string{} - } - rollout.Annotations[util.RolloutHashAnnotation] = hash - klog.Infof("rollout(%s/%s) patch annotation(%s=%s) success", rollout.Namespace, rollout.Name, util.RolloutHashAnnotation, hash) - return nil -} - -// hash hashes `data` with sha256 and returns the hex string -func hash(data string) string { - return fmt.Sprintf("%x", sha256.Sum256([]byte(data))) -} diff --git a/pkg/controller/rollout/trafficrouting.go b/pkg/controller/rollout/trafficrouting.go deleted file mode 100644 index f1919fe9..00000000 --- a/pkg/controller/rollout/trafficrouting.go +++ /dev/null @@ -1,286 +0,0 @@ -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rollout - -import ( - "context" - "fmt" - "time" - - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting" - "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting/gateway" - "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting/nginx" - "github.com/openkruise/rollouts/pkg/util" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/klog/v2" - utilpointer "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) { - if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { - return true, nil - } - - if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { - r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds - } - canaryStatus := r.newStatus.CanaryStatus - if r.newStatus.StableRevision == "" || canaryStatus.PodTemplateHash == "" { - klog.Warningf("rollout(%s/%s) stableRevision or podTemplateHash can't be empty, and wait a moment", r.rollout.Namespace, r.rollout.Name) - return false, nil - } - canaryStatus.CanaryService = r.canaryService - //fetch stable service - stableService := &corev1.Service{} - err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.stableService}, stableService) - if err != nil { - klog.Errorf("rollout(%s/%s) get stable service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.stableService, err.Error()) - // not found, wait a moment, retry - if errors.IsNotFound(err) { - return false, nil - } - return false, err - } - // fetch canary service - // todo for the time being, we do not consider the scenario where the user only changes the stable service definition during rollout progressing - canaryService := &corev1.Service{} - err = r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.canaryService}, canaryService) - if err != nil && !errors.IsNotFound(err) { - klog.Errorf("rollout(%s/%s) get canary service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.canaryService, err.Error()) - return false, err - } else if errors.IsNotFound(err) { - klog.Infof("rollout(%s/%s) canary service(%s) Not Found, and create it", r.rollout.Namespace, r.rollout.Name, r.canaryService) - return false, r.createCanaryService(stableService) - } - - // update service selector - // update service selector specific revision pods - if canaryService.Spec.Selector[r.podRevisionLabelKey()] != canaryStatus.PodTemplateHash { - cloneObj := canaryService.DeepCopy() - body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, r.podRevisionLabelKey(), canaryStatus.PodTemplateHash) - if err = r.Patch(context.TODO(), cloneObj, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { - klog.Errorf("rollout(%s/%s) patch canary service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.canaryService, err.Error()) - return false, err - } - // update canary service time, and wait 3 seconds, just to be safe - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - klog.Infof("add rollout(%s/%s) canary service(%s) selector(%s=%s) success", - r.rollout.Namespace, r.rollout.Name, r.canaryService, r.podRevisionLabelKey(), canaryStatus.PodTemplateHash) - } - if stableService.Spec.Selector[r.podRevisionLabelKey()] != r.newStatus.StableRevision { - cloneObj := stableService.DeepCopy() - body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, r.podRevisionLabelKey(), r.newStatus.StableRevision) - if err = r.Patch(context.TODO(), cloneObj, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { - klog.Errorf("rollout(%s/%s) patch stable service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.stableService, err.Error()) - return false, err - } - // update stable service time, and wait 3 seconds, just to be safe - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - klog.Infof("add rollout(%s/%s) stable service(%s) selector(%s=%s) success", - r.rollout.Namespace, r.rollout.Name, r.stableService, r.podRevisionLabelKey(), r.newStatus.StableRevision) - return false, nil - } - - // After restore stable service configuration, give the ingress provider 3 seconds to take effect - if verifyTime := canaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { - klog.Infof("update rollout(%s/%s) stable service(%s) done, and wait 3 seconds", r.rollout.Namespace, r.rollout.Name, r.stableService) - return false, nil - } - - // route traffic configuration - trController, err := r.newTrafficRoutingController(r) - if err != nil { - klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return false, err - } - var desiredWeight int32 - if len(r.rollout.Spec.Strategy.Canary.Steps) > 0 { - desiredWeight = *r.rollout.Spec.Strategy.Canary.Steps[r.newStatus.CanaryStatus.CurrentStepIndex-1].Weight - } - steps := len(r.rollout.Spec.Strategy.Canary.Steps) - cond := util.GetRolloutCondition(*r.newStatus, rolloutv1alpha1.RolloutConditionProgressing) - cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and route traffic weight(%d)", canaryStatus.CurrentStepIndex, steps, desiredWeight) - verify, err := trController.EnsureRoutes(context.TODO(), desiredWeight) - if err != nil { - return false, err - } else if !verify { - klog.Infof("rollout(%s/%s) is doing step(%d) trafficRouting(%d)", r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex, desiredWeight) - return false, nil - } - klog.Infof("rollout(%s/%s) do step(%d) trafficRouting(%d) success", r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex, desiredWeight) - return true, nil -} - -func (r *rolloutContext) restoreStableService() (bool, error) { - if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { - return true, nil - } - - if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { - r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds - } - //fetch stable service - stableService := &corev1.Service{} - err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.stableService}, stableService) - if err != nil { - if errors.IsNotFound(err) { - return true, nil - } - klog.Errorf("rollout(%s/%s) get stable service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.stableService, err.Error()) - return false, err - } - - if r.newStatus.CanaryStatus == nil { - r.newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{} - } - //restore stable service configuration，remove hash revision selector - if stableService.Spec.Selector != nil && stableService.Spec.Selector[r.podRevisionLabelKey()] != "" { - cloneObj := stableService.DeepCopy() - body := fmt.Sprintf(`{"spec":{"selector":{"%s":null}}}`, r.podRevisionLabelKey()) - if err = r.Patch(context.TODO(), cloneObj, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { - klog.Errorf("rollout(%s/%s) patch stable service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.stableService, err.Error()) - return false, err - } - klog.Infof("remove rollout(%s/%s) stable service(%s) pod revision selector success, and retry later", r.rollout.Namespace, r.rollout.Name, r.stableService) - r.newStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - return false, nil - } - // After restore stable service configuration, give the ingress provider 3 seconds to take effect - if r.newStatus.CanaryStatus.LastUpdateTime != nil { - if verifyTime := r.newStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { - klog.Infof("restore rollout(%s/%s) stable service(%s) done, and wait a moment", r.rollout.Namespace, r.rollout.Name, r.stableService) - return false, nil - } - } - klog.Infof("rollout(%s/%s) doFinalising restore stable service(%s) success", r.rollout.Namespace, r.rollout.Name, r.stableService) - return true, nil -} - -func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) { - if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { - return true, nil - } - klog.Infof("rollout(%s/%s) start finalising traffic routing", r.rollout.Namespace, r.rollout.Name) - if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { - r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds - } - if r.newStatus.CanaryStatus == nil { - r.newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{} - } - // 1. restore ingress and route traffic to stable service - trController, err := r.newTrafficRoutingController(r) - if err != nil { - klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return false, err - } - verify, err := trController.EnsureRoutes(context.TODO(), 0) - if err != nil { - return false, err - } else if !verify { - klog.Infof("rollout(%s/%s) do finalising: ensure canary routes(weight:0)", r.rollout.Namespace, r.rollout.Name) - return false, nil - } - - // DoFinalising, such as delete nginx canary ingress - if err = trController.Finalise(context.TODO()); err != nil { - return false, err - } - - // 2. remove canary service - if r.newStatus.CanaryStatus.CanaryService == "" { - return true, nil - } - cService := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: r.rollout.Namespace, - Name: r.newStatus.CanaryStatus.CanaryService, - }, - } - err = r.Delete(context.TODO(), cService) - if err != nil && !errors.IsNotFound(err) { - klog.Errorf("rollout(%s/%s) remove canary service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, cService.Name, err.Error()) - return false, err - } - klog.Infof("rollout(%s/%s) remove canary service(%s) success", r.rollout.Namespace, r.rollout.Name, cService.Name) - return true, nil -} - -func (r *rolloutContext) newTrafficRoutingController(roCtx *rolloutContext) (trafficrouting.Controller, error) { - canary := roCtx.rollout.Spec.Strategy.Canary - if canary.TrafficRoutings[0].Ingress != nil { - gvk := schema.GroupVersionKind{Group: rolloutv1alpha1.GroupVersion.Group, Version: rolloutv1alpha1.GroupVersion.Version, Kind: "Rollout"} - return nginx.NewNginxTrafficRouting(r.Client, r.newStatus, nginx.Config{ - RolloutName: r.rollout.Name, - RolloutNs: r.rollout.Namespace, - CanaryService: r.canaryService, - StableService: r.stableService, - TrafficConf: r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress, - OwnerRef: *metav1.NewControllerRef(r.rollout, gvk), - }) - } - if canary.TrafficRoutings[0].Gateway != nil { - return gateway.NewGatewayTrafficRouting(r.Client, r.newStatus, gateway.Config{ - RolloutName: r.rollout.Name, - RolloutNs: r.rollout.Namespace, - CanaryService: r.canaryService, - StableService: r.stableService, - TrafficConf: r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].Gateway, - }) - } - return nil, fmt.Errorf("TrafficRouting only support Ingress or Gateway") -} - -func (r *rolloutContext) createCanaryService(stableService *corev1.Service) error { - canaryService := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: r.rollout.Namespace, - Name: r.canaryService, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: r.rollout.APIVersion, - Kind: r.rollout.Kind, - Name: r.rollout.Name, - UID: r.rollout.UID, - Controller: utilpointer.BoolPtr(true), - BlockOwnerDeletion: utilpointer.BoolPtr(true), - }, - }, - }, - Spec: *stableService.Spec.DeepCopy(), - } - // set field nil - canaryService.Spec.ClusterIP = "" - canaryService.Spec.ClusterIPs = nil - canaryService.Spec.ExternalIPs = nil - canaryService.Spec.IPFamilyPolicy = nil - canaryService.Spec.IPFamilies = nil - canaryService.Spec.LoadBalancerIP = "" - canaryService.Spec.Selector[r.podRevisionLabelKey()] = r.newStatus.CanaryStatus.PodTemplateHash - err := r.Create(context.TODO(), canaryService) - if err != nil && !errors.IsAlreadyExists(err) { - klog.Errorf("create rollout(%s/%s) canary service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.canaryService, err.Error()) - return err - } - klog.Infof("create rollout(%s/%s) canary service(%s) success", r.rollout.Namespace, r.rollout.Name, util.DumpJSON(canaryService)) - return nil -} diff --git a/pkg/controller/rollouthistory/rollouthistory_controller.go b/pkg/controller/rollouthistory/rollouthistory_controller.go index be09159c..a9601972 100644 --- a/pkg/controller/rollouthistory/rollouthistory_controller.go +++ b/pkg/controller/rollouthistory/rollouthistory_controller.go @@ -26,7 +26,6 @@ import ( rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" "github.com/openkruise/rollouts/pkg/feature" - "github.com/openkruise/rollouts/pkg/util" utilfeature "github.com/openkruise/rollouts/pkg/util/feature" corev1 "k8s.io/api/core/v1" @@ -359,7 +358,7 @@ func (r *RolloutHistoryReconciler) recordStatusCanarySteps(rollout *rolloutv1alp return err } // get extra labelSelector including rolloutBathID, rolloutID and workload selector - lableSelectorString := fmt.Sprintf("%v=%v,%v=%v,%v", util.RolloutBatchIDLabel, len(rolloutHistory.Status.CanarySteps)+1, util.RolloutIDLabel, rolloutHistory.Spec.Rollout.RolloutID, selector.String()) + lableSelectorString := fmt.Sprintf("%v=%v,%v=%v,%v", rolloutv1alpha1.RolloutBatchIDLabel, len(rolloutHistory.Status.CanarySteps)+1, rolloutv1alpha1.RolloutIDLabel, rolloutHistory.Spec.Rollout.RolloutID, selector.String()) extraSelector, err = labels.Parse(lableSelectorString) if err != nil { return err diff --git a/pkg/controller/rollouthistory/rollouthistory_controller_test.go b/pkg/controller/rollouthistory/rollouthistory_controller_test.go index eaad1815..efaea08f 100644 --- a/pkg/controller/rollouthistory/rollouthistory_controller_test.go +++ b/pkg/controller/rollouthistory/rollouthistory_controller_test.go @@ -37,7 +37,6 @@ import ( "github.com/openkruise/kruise-api/apps/pub" kruisev1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/util" ) func init() { @@ -133,7 +132,7 @@ var ( Name: "workload-demo", }, }, - RolloutID: "1", + DeprecatedRolloutID: "1", Strategy: rolloutv1alpha1.RolloutStrategy{ Canary: &rolloutv1alpha1.CanaryStrategy{ Steps: []rolloutv1alpha1.CanaryStep{ @@ -321,8 +320,8 @@ var ( Name: "pod-demo", Namespace: "default", Labels: map[string]string{ - util.RolloutBatchIDLabel: "1", - util.RolloutIDLabel: "1", + rolloutv1alpha1.RolloutBatchIDLabel: "1", + rolloutv1alpha1.RolloutIDLabel: "1", }, }, Spec: corev1.PodSpec{ @@ -409,9 +408,9 @@ func TestReconcile(t *testing.T) { PodIP: "1.2.3.1", } pod1.Labels = map[string]string{ - util.RolloutBatchIDLabel: "1", - util.RolloutIDLabel: "2", - "app": "echoserver", + rolloutv1alpha1.RolloutBatchIDLabel: "1", + rolloutv1alpha1.RolloutIDLabel: "2", + "app": "echoserver", } pod2 := podDemo.DeepCopy() @@ -421,9 +420,9 @@ func TestReconcile(t *testing.T) { PodIP: "1.2.3.2", } pod2.Labels = map[string]string{ - util.RolloutBatchIDLabel: "2", - util.RolloutIDLabel: "2", - "app": "echoserver", + rolloutv1alpha1.RolloutBatchIDLabel: "2", + rolloutv1alpha1.RolloutIDLabel: "2", + "app": "echoserver", } pod3 := podDemo.DeepCopy() @@ -433,9 +432,9 @@ func TestReconcile(t *testing.T) { PodIP: "1.2.3.3", } pod3.Labels = map[string]string{ - util.RolloutBatchIDLabel: "3", - util.RolloutIDLabel: "2", - "app": "echoserver", + rolloutv1alpha1.RolloutBatchIDLabel: "3", + rolloutv1alpha1.RolloutIDLabel: "2", + "app": "echoserver", } pod4 := podDemo.DeepCopy() @@ -445,9 +444,9 @@ func TestReconcile(t *testing.T) { PodIP: "1.2.3.4", } pod4.Labels = map[string]string{ - util.RolloutBatchIDLabel: "3", - util.RolloutIDLabel: "2", - "app": "echoserver", + rolloutv1alpha1.RolloutBatchIDLabel: "3", + rolloutv1alpha1.RolloutIDLabel: "2", + "app": "echoserver", } pod5 := podDemo.DeepCopy() @@ -457,9 +456,9 @@ func TestReconcile(t *testing.T) { PodIP: "1.2.3.5", } pod5.Labels = map[string]string{ - util.RolloutBatchIDLabel: "3", - util.RolloutIDLabel: "2", - "app": "echoserver", + rolloutv1alpha1.RolloutBatchIDLabel: "3", + rolloutv1alpha1.RolloutIDLabel: "2", + "app": "echoserver", } return []*corev1.Pod{pod1, pod2, pod3, pod4, pod5} @@ -482,7 +481,7 @@ func TestReconcile(t *testing.T) { }, getRollout: func() []*rolloutv1alpha1.Rollout { rollout := rolloutDemo1.DeepCopy() - rollout.Spec.RolloutID = "2" + rollout.Spec.DeprecatedRolloutID = "2" rollout.Status = rolloutv1alpha1.RolloutStatus{ CanaryStatus: &rolloutv1alpha1.CanaryStatus{ ObservedRolloutID: "2", @@ -615,7 +614,7 @@ func TestReconcile(t *testing.T) { }, getRollout: func() []*rolloutv1alpha1.Rollout { rollout := rolloutDemo1.DeepCopy() - rollout.Spec.RolloutID = "" + rollout.Spec.DeprecatedRolloutID = "" rollout.Status = rolloutv1alpha1.RolloutStatus{ CanaryStatus: &rolloutv1alpha1.CanaryStatus{ ObservedRolloutID: "", @@ -663,7 +662,7 @@ func TestReconcile(t *testing.T) { }, getRollout: func() []*rolloutv1alpha1.Rollout { rollout := rolloutDemo1.DeepCopy() - rollout.Spec.RolloutID = "4" + rollout.Spec.DeprecatedRolloutID = "4" rollout.Status = rolloutv1alpha1.RolloutStatus{ CanaryStatus: &rolloutv1alpha1.CanaryStatus{ ObservedRolloutID: "4", @@ -869,7 +868,7 @@ func TestReconcile(t *testing.T) { }, getRollout: func() []*rolloutv1alpha1.Rollout { rollout := rolloutDemo1.DeepCopy() - rollout.Spec.RolloutID = "5" + rollout.Spec.DeprecatedRolloutID = "5" rollout.Status = rolloutv1alpha1.RolloutStatus{ CanaryStatus: &rolloutv1alpha1.CanaryStatus{ ObservedRolloutID: "5", diff --git a/pkg/trafficrouting/manager.go b/pkg/trafficrouting/manager.go new file mode 100644 index 00000000..2bd00106 --- /dev/null +++ b/pkg/trafficrouting/manager.go @@ -0,0 +1,309 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trafficrouting + +import ( + "context" + "fmt" + "time" + + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting/network" + "github.com/openkruise/rollouts/pkg/trafficrouting/network/gateway" + "github.com/openkruise/rollouts/pkg/trafficrouting/network/nginx" + "github.com/openkruise/rollouts/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + defaultGracePeriodSeconds int32 = 3 + rolloutControllerKind = v1alpha1.SchemeGroupVersion.WithKind("Rollout") +) + +// Manager responsible for adjusting network resources +// such as Service, Ingress, Gateway API, etc., to achieve traffic grayscale. +type Manager struct { + client.Client +} + +func NewTrafficRoutingManager(c client.Client) *Manager { + return &Manager{c} +} + +// InitializeTrafficRouting determine if the network resources(service & ingress & gateway api) exist. +// If it is Ingress, init method will create the canary ingress resources, and set weight=0. +func (m *Manager) InitializeTrafficRouting(c *util.RolloutContext) error { + if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { + return nil + } + sService := c.Rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service + // check service + service := &corev1.Service{} + if err := m.Get(context.TODO(), types.NamespacedName{Namespace: c.Rollout.Namespace, Name: sService}, service); err != nil { + return err + } + cService := fmt.Sprintf("%s-canary", sService) + // new network provider, ingress or gateway + trController, err := newNetworkProvider(m.Client, c.Rollout, c.NewStatus, sService, cService) + if err != nil { + klog.Errorf("rollout(%s/%s) newNetworkProvider failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return err + } + return trController.Initialize(context.TODO()) +} + +func (m *Manager) DoTrafficRouting(c *util.RolloutContext) (bool, error) { + if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { + return true, nil + } + trafficRouting := c.Rollout.Spec.Strategy.Canary.TrafficRoutings[0] + if trafficRouting.GracePeriodSeconds <= 0 { + trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds + } + canaryStatus := c.NewStatus.CanaryStatus + if canaryStatus.StableRevision == "" || canaryStatus.PodTemplateHash == "" { + klog.Warningf("rollout(%s/%s) stableRevision or podTemplateHash can not be empty, and wait a moment", c.Rollout.Namespace, c.Rollout.Name) + return false, nil + } + //fetch stable service + stableService := &corev1.Service{} + err := m.Get(context.TODO(), client.ObjectKey{Namespace: c.Rollout.Namespace, Name: trafficRouting.Service}, stableService) + if err != nil { + klog.Errorf("rollout(%s/%s) get stable service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service, err.Error()) + // not found, wait a moment, retry + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + // canary service name + canaryServiceName := fmt.Sprintf("%s-canary", trafficRouting.Service) + // fetch canary service + canaryService := &corev1.Service{} + err = m.Get(context.TODO(), client.ObjectKey{Namespace: c.Rollout.Namespace, Name: canaryServiceName}, canaryService) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("rollout(%s/%s) get canary service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, canaryServiceName, err.Error()) + return false, err + } else if errors.IsNotFound(err) { + canaryService, err = m.createCanaryService(c, canaryServiceName, *stableService.Spec.DeepCopy()) + if err != nil { + return false, err + } + } + + // patch canary service only selector the canary pods + if canaryService.Spec.Selector[c.Workload.RevisionLabelKey] != canaryStatus.PodTemplateHash { + body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, c.Workload.RevisionLabelKey, canaryStatus.PodTemplateHash) + if err = m.Patch(context.TODO(), canaryService, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { + klog.Errorf("rollout(%s/%s) patch canary service(%s) selector failed: %s", c.Rollout.Namespace, c.Rollout.Name, canaryService.Name, err.Error()) + return false, err + } + // update canary service time, and wait 3 seconds, just to be safe + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + klog.Infof("rollout(%s/%s) patch canary service(%s) selector(%s=%s) success", + c.Rollout.Namespace, c.Rollout.Name, canaryService.Name, c.Workload.RevisionLabelKey, canaryStatus.PodTemplateHash) + } + // patch stable service only selector the stable pods + if stableService.Spec.Selector[c.Workload.RevisionLabelKey] != canaryStatus.StableRevision { + body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, c.Workload.RevisionLabelKey, canaryStatus.StableRevision) + if err = m.Patch(context.TODO(), stableService, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { + klog.Errorf("rollout(%s/%s) patch stable service(%s) selector failed: %s", c.Rollout.Namespace, c.Rollout.Name, stableService.Name, err.Error()) + return false, err + } + // update stable service time, and wait 3 seconds, just to be safe + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + klog.Infof("add rollout(%s/%s) stable service(%s) selector(%s=%s) success", + c.Rollout.Namespace, c.Rollout.Name, stableService.Name, c.Workload.RevisionLabelKey, canaryStatus.StableRevision) + return false, nil + } + // After modify stable service configuration, give the network provider 3 seconds to react + if verifyTime := canaryStatus.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { + klog.Infof("rollout(%s/%s) update service selector, and wait 3 seconds", c.Rollout.Namespace, c.Rollout.Name) + return false, nil + } + + // new network provider, ingress or gateway + trController, err := newNetworkProvider(m.Client, c.Rollout, c.NewStatus, stableService.Name, canaryService.Name) + if err != nil { + klog.Errorf("rollout(%s/%s) newNetworkProvider failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return false, err + } + cStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] + steps := len(c.Rollout.Spec.Strategy.Canary.Steps) + cond := util.GetRolloutCondition(*c.NewStatus, v1alpha1.RolloutConditionProgressing) + cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and route traffic weight(%d)", canaryStatus.CurrentStepIndex, steps, *cStep.Weight) + verify, err := trController.EnsureRoutes(context.TODO(), *cStep.Weight) + if err != nil { + return false, err + } else if !verify { + klog.Infof("rollout(%s/%s) is doing step(%d) trafficRouting(%d)", c.Rollout.Namespace, c.Rollout.Name, canaryStatus.CurrentStepIndex, *cStep.Weight) + return false, nil + } + klog.Infof("rollout(%s/%s) do step(%d) trafficRouting(%d) success", c.Rollout.Namespace, c.Rollout.Name, canaryStatus.CurrentStepIndex, *cStep.Weight) + return true, nil +} + +func (m *Manager) FinalisingTrafficRouting(c *util.RolloutContext, onlyRestoreStableService bool) (bool, error) { + if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { + return true, nil + } + trafficRouting := c.Rollout.Spec.Strategy.Canary.TrafficRoutings[0] + if trafficRouting.GracePeriodSeconds <= 0 { + trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds + } + if c.NewStatus.CanaryStatus == nil { + c.NewStatus.CanaryStatus = &v1alpha1.CanaryStatus{} + } + klog.Infof("rollout(%s/%s) start finalising traffic routing", c.Rollout.Namespace, c.Rollout.Name) + // remove stable service the pod revision selector, so stable service will be selector all version pods. + verify, err := m.restoreStableService(c) + if err != nil || !verify { + return false, err + } else if onlyRestoreStableService { + return true, nil + } + + cServiceName := fmt.Sprintf("%s-canary", trafficRouting.Service) + trController, err := newNetworkProvider(m.Client, c.Rollout, c.NewStatus, trafficRouting.Service, cServiceName) + if err != nil { + klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) + return false, err + } + // First route 100% traffic to stable service + verify, err = trController.EnsureRoutes(context.TODO(), 0) + if err != nil { + return false, err + } else if !verify { + c.NewStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + return false, nil + } + if c.NewStatus.CanaryStatus.LastUpdateTime != nil { + // After restore the stable service configuration, give network provider 3 seconds to react + if verifyTime := c.NewStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { + klog.Infof("rollout(%s/%s) route 100% traffic to stable service, and wait a moment", c.Rollout.Namespace, c.Rollout.Name) + return false, nil + } + } + + // modify network(ingress & gateway api) configuration, route all traffic to stable service + if err = trController.Finalise(context.TODO()); err != nil { + return false, err + } + // remove canary service + cService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: c.Rollout.Namespace, Name: cServiceName}} + err = m.Delete(context.TODO(), cService) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("rollout(%s/%s) remove canary service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, cService.Name, err.Error()) + return false, err + } + klog.Infof("rollout(%s/%s) remove canary service(%s) success", c.Rollout.Namespace, c.Rollout.Name, cService.Name) + return true, nil +} + +func newNetworkProvider(c client.Client, rollout *v1alpha1.Rollout, newStatus *v1alpha1.RolloutStatus, sService, cService string) (network.NetworkProvider, error) { + trafficRouting := rollout.Spec.Strategy.Canary.TrafficRoutings[0] + if trafficRouting.Ingress != nil { + return nginx.NewNginxTrafficRouting(c, newStatus, nginx.Config{ + RolloutName: rollout.Name, + RolloutNs: rollout.Namespace, + CanaryService: cService, + StableService: sService, + TrafficConf: trafficRouting.Ingress, + OwnerRef: *metav1.NewControllerRef(rollout, rolloutControllerKind), + }) + } + if trafficRouting.Gateway != nil { + return gateway.NewGatewayTrafficRouting(c, newStatus, gateway.Config{ + RolloutName: rollout.Name, + RolloutNs: rollout.Namespace, + CanaryService: cService, + StableService: sService, + TrafficConf: trafficRouting.Gateway, + }) + } + return nil, fmt.Errorf("TrafficRouting current only support Ingress or Gateway API") +} + +func (m *Manager) createCanaryService(c *util.RolloutContext, cService string, spec corev1.ServiceSpec) (*corev1.Service, error) { + canaryService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: c.Rollout.Namespace, + Name: cService, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(c.Rollout, rolloutControllerKind)}, + }, + Spec: spec, + } + + // set field nil + canaryService.Spec.ClusterIP = "" + canaryService.Spec.ClusterIPs = nil + canaryService.Spec.ExternalIPs = nil + canaryService.Spec.IPFamilyPolicy = nil + canaryService.Spec.IPFamilies = nil + canaryService.Spec.LoadBalancerIP = "" + canaryService.Spec.Selector[c.Workload.RevisionLabelKey] = c.NewStatus.CanaryStatus.PodTemplateHash + err := m.Create(context.TODO(), canaryService) + if err != nil && !errors.IsAlreadyExists(err) { + klog.Errorf("rollout(%s/%s) create canary service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, cService, err.Error()) + return nil, err + } + klog.Infof("rollout(%s/%s) create canary service(%s) success", c.Rollout.Namespace, c.Rollout.Name, util.DumpJSON(canaryService)) + return canaryService, nil +} + +// remove stable service the pod revision selector, so stable service will be selector all version pods. +func (m *Manager) restoreStableService(c *util.RolloutContext) (bool, error) { + if c.Workload == nil { + return true, nil + } + trafficRouting := c.Rollout.Spec.Strategy.Canary.TrafficRoutings[0] + //fetch stable service + stableService := &corev1.Service{} + err := m.Get(context.TODO(), client.ObjectKey{Namespace: c.Rollout.Namespace, Name: trafficRouting.Service}, stableService) + if err != nil { + if errors.IsNotFound(err) { + return true, nil + } + klog.Errorf("rollout(%s/%s) get stable service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service, err.Error()) + return false, err + } + if stableService.Spec.Selector[c.Workload.RevisionLabelKey] != "" { + body := fmt.Sprintf(`{"spec":{"selector":{"%s":null}}}`, c.Workload.RevisionLabelKey) + if err = m.Patch(context.TODO(), stableService, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { + klog.Errorf("rollout(%s/%s) patch stable service(%s) failed: %s", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service, err.Error()) + return false, err + } + klog.Infof("remove rollout(%s/%s) stable service(%s) pod revision selector, and wait a moment", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service) + c.NewStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + return false, nil + } + if c.NewStatus.CanaryStatus.LastUpdateTime == nil { + return true, nil + } + // After restore the stable service configuration, give network provider 3 seconds to react + if verifyTime := c.NewStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { + klog.Infof("rollout(%s/%s) restoring stable service(%s), and wait a moment", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service) + return false, nil + } + klog.Infof("rollout(%s/%s) doFinalising stable service(%s) success", c.Rollout.Namespace, c.Rollout.Name, trafficRouting.Service) + return true, nil +} diff --git a/pkg/trafficrouting/manager_test.go b/pkg/trafficrouting/manager_test.go new file mode 100644 index 00000000..a63b325b --- /dev/null +++ b/pkg/trafficrouting/manager_test.go @@ -0,0 +1,630 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trafficrouting + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + kruisev1aplphal "github.com/openkruise/kruise-api/apps/v1alpha1" + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + scheme *runtime.Scheme + nginxIngressAnnotationDefaultPrefix = "nginx.ingress.kubernetes.io" + + demoService = corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "echoserver", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(8080), + }, + }, + Selector: map[string]string{ + "app": "echoserver", + }, + }, + } + + demoIngress = netv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "networking.k8s.io/v1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "echoserver", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + }, + Spec: netv1.IngressSpec{ + Rules: []netv1.IngressRule{ + { + Host: "echoserver.example.com", + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{ + Paths: []netv1.HTTPIngressPath{ + { + Path: "/apis/echo", + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: "echoserver", + Port: netv1.ServiceBackendPort{ + Name: "http", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + demoRollout = &v1alpha1.Rollout{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rollout-demo", + Labels: map[string]string{}, + Annotations: map[string]string{ + util.RolloutHashAnnotation: "rollout-hash-v1", + }, + }, + Spec: v1alpha1.RolloutSpec{ + ObjectRef: v1alpha1.ObjectRef{ + WorkloadRef: &v1alpha1.WorkloadRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "echoserver", + }, + }, + Strategy: v1alpha1.RolloutStrategy{ + Canary: &v1alpha1.CanaryStrategy{ + Steps: []v1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(5), + Replicas: &intstr.IntOrString{IntVal: 1}, + }, + { + Weight: utilpointer.Int32(20), + Replicas: &intstr.IntOrString{IntVal: 2}, + }, + { + Weight: utilpointer.Int32(60), + Replicas: &intstr.IntOrString{IntVal: 6}, + }, + { + Weight: utilpointer.Int32(100), + Replicas: &intstr.IntOrString{IntVal: 10}, + }, + }, + TrafficRoutings: []*v1alpha1.TrafficRouting{ + { + Service: "echoserver", + Ingress: &v1alpha1.IngressTrafficRouting{ + Name: "echoserver", + }, + }, + }, + }, + }, + }, + Status: v1alpha1.RolloutStatus{ + Phase: v1alpha1.RolloutPhaseProgressing, + CanaryStatus: &v1alpha1.CanaryStatus{ + ObservedWorkloadGeneration: 1, + RolloutHash: "rollout-hash-v1", + ObservedRolloutID: "rollout-id-1", + StableRevision: "podtemplatehash-v1", + CanaryRevision: "revision-v2", + CurrentStepIndex: 1, + CurrentStepState: v1alpha1.CanaryStepStateTrafficRouting, + PodTemplateHash: "podtemplatehash-v2", + LastUpdateTime: &metav1.Time{Time: time.Now()}, + }, + Conditions: []v1alpha1.RolloutCondition{ + { + Type: v1alpha1.RolloutConditionProgressing, + Reason: v1alpha1.ProgressingReasonInRolling, + Status: corev1.ConditionFalse, + }, + }, + }, + } +) + +func init() { + scheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = kruisev1aplphal.AddToScheme(scheme) + _ = v1alpha1.AddToScheme(scheme) +} + +func TestDoTrafficRouting(t *testing.T) { + cases := []struct { + name string + getObj func() ([]*corev1.Service, []*netv1.Ingress) + getRollout func() (*v1alpha1.Rollout, *util.Workload) + expectObj func() ([]*corev1.Service, []*netv1.Ingress) + expectDone bool + }{ + { + name: "DoTrafficRouting test1", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + return []*corev1.Service{demoService.DeepCopy()}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *util.Workload) { + return demoRollout.DeepCopy(), &util.Workload{RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey} + }, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s1.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v1" + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + expectDone: false, + }, + { + name: "DoTrafficRouting test2", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s1.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v1" + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *util.Workload) { + obj := demoRollout.DeepCopy() + obj.Status.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + return obj, &util.Workload{RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey} + }, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s1.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v1" + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1} + }, + expectDone: false, + }, + { + name: "DoTrafficRouting test3", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s1.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v1" + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{demoIngress.DeepCopy()} + }, + getRollout: func() (*v1alpha1.Rollout, *util.Workload) { + obj := demoRollout.DeepCopy() + obj.Status.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now().Add(-10 * time.Second)} + return obj, &util.Workload{RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey} + }, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s1.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v1" + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "5" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + expectDone: false, + }, + { + name: "DoTrafficRouting test4", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s1.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v1" + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "5" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + getRollout: func() (*v1alpha1.Rollout, *util.Workload) { + obj := demoRollout.DeepCopy() + obj.Status.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now().Add(-10 * time.Second)} + return obj, &util.Workload{RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey} + }, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s1.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v1" + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "5" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + expectDone: true, + }, + { + name: "DoTrafficRouting test5", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s1.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v1" + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "5" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + getRollout: func() (*v1alpha1.Rollout, *util.Workload) { + obj := demoRollout.DeepCopy() + obj.Status.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now().Add(-10 * time.Second)} + obj.Status.CanaryStatus.CurrentStepIndex = 2 + return obj, &util.Workload{RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey} + }, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s1.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v1" + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "20" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + expectDone: false, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + ss, ig := cs.getObj() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ig[0], ss[0]).Build() + if len(ss) == 2 { + _ = client.Create(context.TODO(), ss[1]) + } + if len(ig) == 2 { + _ = client.Create(context.TODO(), ig[1]) + } + c := &util.RolloutContext{} + c.Rollout, c.Workload = cs.getRollout() + c.NewStatus = c.Rollout.Status.DeepCopy() + manager := NewTrafficRoutingManager(client) + done, err := manager.DoTrafficRouting(c) + if err != nil { + t.Fatalf("DoTrafficRouting failed: %s", err) + } + if cs.expectDone != done { + t.Fatalf("DoTrafficRouting expect(%v), but get(%v)", cs.expectDone, done) + } + ss, ig = cs.expectObj() + for _, obj := range ss { + checkObjEqual(client, t, obj) + } + for _, obj := range ig { + checkObjEqual(client, t, obj) + } + }) + } +} + +func TestFinalisingTrafficRouting(t *testing.T) { + cases := []struct { + name string + getObj func() ([]*corev1.Service, []*netv1.Ingress) + getRollout func() (*v1alpha1.Rollout, *util.Workload) + onlyRestoreStableService bool + expectObj func() ([]*corev1.Service, []*netv1.Ingress) + expectDone bool + }{ + { + name: "FinalisingTrafficRouting test1", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s1.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v1" + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "100" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + getRollout: func() (*v1alpha1.Rollout, *util.Workload) { + obj := demoRollout.DeepCopy() + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + obj.Status.CanaryStatus.CurrentStepIndex = 4 + return obj, &util.Workload{RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey} + }, + onlyRestoreStableService: true, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "100" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + expectDone: false, + }, + { + name: "FinalisingTrafficRouting test2", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "100" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + getRollout: func() (*v1alpha1.Rollout, *util.Workload) { + obj := demoRollout.DeepCopy() + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + obj.Status.CanaryStatus.CurrentStepIndex = 4 + return obj, &util.Workload{RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey} + }, + onlyRestoreStableService: true, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "100" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + expectDone: false, + }, + { + name: "FinalisingTrafficRouting test3", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "100" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + getRollout: func() (*v1alpha1.Rollout, *util.Workload) { + obj := demoRollout.DeepCopy() + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + obj.Status.CanaryStatus.CurrentStepIndex = 4 + obj.Status.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now().Add(-10 * time.Second)} + return obj, &util.Workload{RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey} + }, + onlyRestoreStableService: true, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "100" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + expectDone: true, + }, + { + name: "FinalisingTrafficRouting test4", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "100" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + getRollout: func() (*v1alpha1.Rollout, *util.Workload) { + obj := demoRollout.DeepCopy() + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + obj.Status.CanaryStatus.CurrentStepIndex = 4 + obj.Status.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now().Add(-3 * time.Second)} + return obj, &util.Workload{RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey} + }, + onlyRestoreStableService: false, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "0" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + expectDone: false, + }, + { + name: "FinalisingTrafficRouting test5", + getObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + s2 := demoService.DeepCopy() + s2.Name = "echoserver-canary" + s2.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey] = "podtemplatehash-v2" + c1 := demoIngress.DeepCopy() + c2 := demoIngress.DeepCopy() + c2.Name = "echoserver-canary" + c2.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" + c2.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = "0" + c2.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*corev1.Service{s1, s2}, []*netv1.Ingress{c1, c2} + }, + getRollout: func() (*v1alpha1.Rollout, *util.Workload) { + obj := demoRollout.DeepCopy() + obj.Status.CanaryStatus.CurrentStepState = v1alpha1.CanaryStepStateCompleted + obj.Status.CanaryStatus.CurrentStepIndex = 4 + obj.Status.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now().Add(-3 * time.Second)} + return obj, &util.Workload{RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey} + }, + onlyRestoreStableService: false, + expectObj: func() ([]*corev1.Service, []*netv1.Ingress) { + s1 := demoService.DeepCopy() + c1 := demoIngress.DeepCopy() + return []*corev1.Service{s1}, []*netv1.Ingress{c1} + }, + expectDone: true, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + ss, ig := cs.getObj() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ig[0], ss[0]).Build() + if len(ss) == 2 { + _ = client.Create(context.TODO(), ss[1]) + } + if len(ig) == 2 { + _ = client.Create(context.TODO(), ig[1]) + } + c := &util.RolloutContext{} + c.Rollout, c.Workload = cs.getRollout() + c.NewStatus = c.Rollout.Status.DeepCopy() + manager := NewTrafficRoutingManager(client) + done, err := manager.FinalisingTrafficRouting(c, cs.onlyRestoreStableService) + if err != nil { + t.Fatalf("DoTrafficRouting failed: %s", err) + } + if cs.expectDone != done { + t.Fatalf("DoTrafficRouting expect(%v), but get(%v)", cs.expectDone, done) + } + ss, ig = cs.expectObj() + for _, obj := range ss { + checkObjEqual(client, t, obj) + } + for _, obj := range ig { + checkObjEqual(client, t, obj) + } + }) + } +} + +func checkObjEqual(c client.WithWatch, t *testing.T, expect client.Object) { + gvk := expect.GetObjectKind().GroupVersionKind() + obj := getEmptyObject(gvk) + err := c.Get(context.TODO(), client.ObjectKey{Namespace: expect.GetNamespace(), Name: expect.GetName()}, obj) + if err != nil { + t.Fatalf("get object failed: %s", err.Error()) + } + switch gvk.Kind { + case "Service": + s1 := obj.(*corev1.Service) + s2 := expect.(*corev1.Service) + if !reflect.DeepEqual(s1.Spec, s2.Spec) { + t.Fatalf("expect(%s), but get object(%s)", util.DumpJSON(s2.Spec), util.DumpJSON(s1.Spec)) + } + case "Ingress": + s1 := obj.(*netv1.Ingress) + s2 := expect.(*netv1.Ingress) + if !reflect.DeepEqual(s1.Spec, s2.Spec) || !reflect.DeepEqual(s1.Annotations, s2.Annotations) { + t.Fatalf("expect(%s), but get object(%s)", util.DumpJSON(s2), util.DumpJSON(s1)) + } + } +} + +func getEmptyObject(gvk schema.GroupVersionKind) client.Object { + switch gvk.Kind { + case "Service": + return &corev1.Service{} + case "Ingress": + return &netv1.Ingress{} + } + return nil +} diff --git a/pkg/controller/rollout/trafficrouting/gateway/gateway.go b/pkg/trafficrouting/network/gateway/gateway.go similarity index 97% rename from pkg/controller/rollout/trafficrouting/gateway/gateway.go rename to pkg/trafficrouting/network/gateway/gateway.go index 35bc06b5..7aa717f1 100644 --- a/pkg/controller/rollout/trafficrouting/gateway/gateway.go +++ b/pkg/trafficrouting/network/gateway/gateway.go @@ -21,7 +21,7 @@ import ( "reflect" rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting" + "github.com/openkruise/rollouts/pkg/trafficrouting/network" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" @@ -46,7 +46,7 @@ type gatewayController struct { } // NewGatewayTrafficRouting The Gateway API is a part of the SIG Network. -func NewGatewayTrafficRouting(client client.Client, newStatus *rolloutv1alpha1.RolloutStatus, conf Config) (trafficrouting.Controller, error) { +func NewGatewayTrafficRouting(client client.Client, newStatus *rolloutv1alpha1.RolloutStatus, conf Config) (network.NetworkProvider, error) { r := &gatewayController{ Client: client, conf: conf, diff --git a/pkg/controller/rollout/trafficrouting/gateway/gateway_test.go b/pkg/trafficrouting/network/gateway/gateway_test.go similarity index 100% rename from pkg/controller/rollout/trafficrouting/gateway/gateway_test.go rename to pkg/trafficrouting/network/gateway/gateway_test.go diff --git a/pkg/controller/rollout/trafficrouting/interface.go b/pkg/trafficrouting/network/interface.go similarity index 81% rename from pkg/controller/rollout/trafficrouting/interface.go rename to pkg/trafficrouting/network/interface.go index ae40445e..5888804c 100644 --- a/pkg/controller/rollout/trafficrouting/interface.go +++ b/pkg/trafficrouting/network/interface.go @@ -14,13 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package trafficrouting +package network -import "context" +import ( + "context" +) -// Controller common function across all TrafficRouting implementation -type Controller interface { - // Initialize will validate the traffic routing resource +// NetworkProvider common function across all TrafficRouting implementation +type NetworkProvider interface { + // Initialize determine if the network resources(ingress & gateway api) exist. + // If it is Ingress, init method will create the canary ingress resources, and set weight=0. Initialize(ctx context.Context) error // EnsureRoutes check and set canary desired weight. desiredWeight[0,100] // 1. check if canary has been set desired weight. diff --git a/pkg/controller/rollout/trafficrouting/nginx/nginx.go b/pkg/trafficrouting/network/nginx/nginx.go similarity index 98% rename from pkg/controller/rollout/trafficrouting/nginx/nginx.go rename to pkg/trafficrouting/network/nginx/nginx.go index 0d932936..32ace643 100644 --- a/pkg/controller/rollout/trafficrouting/nginx/nginx.go +++ b/pkg/trafficrouting/network/nginx/nginx.go @@ -22,7 +22,7 @@ import ( "strconv" rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting" + "github.com/openkruise/rollouts/pkg/trafficrouting/network" "github.com/openkruise/rollouts/pkg/util" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -53,7 +53,7 @@ type Config struct { OwnerRef metav1.OwnerReference } -func NewNginxTrafficRouting(client client.Client, newStatus *rolloutv1alpha1.RolloutStatus, conf Config) (trafficrouting.Controller, error) { +func NewNginxTrafficRouting(client client.Client, newStatus *rolloutv1alpha1.RolloutStatus, conf Config) (network.NetworkProvider, error) { r := &nginxController{ Client: client, conf: conf, diff --git a/pkg/util/condition.go b/pkg/util/condition.go index 5d7750f6..27f1d22c 100644 --- a/pkg/util/condition.go +++ b/pkg/util/condition.go @@ -50,7 +50,8 @@ func GetRolloutCondition(status rolloutv1alpha1.RolloutStatus, condType rolloutv // by returning false. Returns true if the condition was updated func SetRolloutCondition(status *rolloutv1alpha1.RolloutStatus, condition rolloutv1alpha1.RolloutCondition) bool { currentCond := GetRolloutCondition(*status, condition.Type) - if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason { + if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason && + currentCond.Message == condition.Message { return false } // Do not update lastTransitionTime if the status of the condition doesn't change. @@ -62,11 +63,6 @@ func SetRolloutCondition(status *rolloutv1alpha1.RolloutStatus, condition rollou return true } -// RemoveRolloutCondition removes the rollout condition with the provided type. -func RemoveRolloutCondition(status *rolloutv1alpha1.RolloutStatus, condType rolloutv1alpha1.RolloutConditionType) { - status.Conditions = filterOutCondition(status.Conditions, condType) -} - // filterOutCondition returns a new slice of rollout conditions without conditions with the provided type. func filterOutCondition(conditions []rolloutv1alpha1.RolloutCondition, condType rolloutv1alpha1.RolloutConditionType) []rolloutv1alpha1.RolloutCondition { var newConditions []rolloutv1alpha1.RolloutCondition diff --git a/pkg/util/constant.go b/pkg/util/constant.go index c68aa985..579a36bd 100644 --- a/pkg/util/constant.go +++ b/pkg/util/constant.go @@ -25,8 +25,6 @@ const ( InRolloutProgressingAnnotation = "rollouts.kruise.io/in-progressing" // RolloutHashAnnotation record observed rollout spec hash RolloutHashAnnotation = "rollouts.kruise.io/hash" - // RollbackInBatchAnnotation allow use disable quick rollback, and will roll back in batch style. - RollbackInBatchAnnotation = "rollouts.kruise.io/rollback-in-batch" ) // For Workloads @@ -43,13 +41,6 @@ const ( // For Pods const ( - // RolloutIDLabel is designed to distinguish each workload revision publications. - // The value of RolloutIDLabel corresponds Rollout.Spec.RolloutID. - RolloutIDLabel = "rollouts.kruise.io/rollout-id" - // RolloutBatchIDLabel is the label key of batch id that will be patched to pods during rollout. - // Only when RolloutIDLabel is set, RolloutBatchIDLabel will be patched. - // Users can use RolloutIDLabel and RolloutBatchIDLabel to select the pods that are upgraded in some certain batch and release. - RolloutBatchIDLabel = "rollouts.kruise.io/rollout-batch-id" // NoNeedUpdatePodLabel will be patched to pod when rollback in batches if the pods no need to rollback NoNeedUpdatePodLabel = "rollouts.kruise.io/no-need-update" ) diff --git a/pkg/util/context.go b/pkg/util/context.go new file mode 100644 index 00000000..1314d829 --- /dev/null +++ b/pkg/util/context.go @@ -0,0 +1,32 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "time" + + "github.com/openkruise/rollouts/api/v1alpha1" +) + +type RolloutContext struct { + Rollout *v1alpha1.Rollout + NewStatus *v1alpha1.RolloutStatus + // related workload + Workload *Workload + // reconcile RequeueAfter recheckTime + RecheckTime *time.Time +} diff --git a/pkg/util/controller_finder.go b/pkg/util/controller_finder.go index e3075490..b3037484 100644 --- a/pkg/util/controller_finder.go +++ b/pkg/util/controller_finder.go @@ -47,10 +47,6 @@ type Workload struct { CanaryRevision string // pod template hash is used as service selector hash PodTemplateHash string - // canary replicas - CanaryReplicas int32 - // canary ready replicas - CanaryReadyReplicas int32 // Revision hash key RevisionLabelKey string @@ -130,22 +126,19 @@ func (r *ControllerFinder) getKruiseCloneSet(namespace string, ref *rolloutv1alp return &Workload{IsStatusConsistent: false}, nil } workload := &Workload{ - RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey, - StableRevision: cloneSet.Status.CurrentRevision[strings.LastIndex(cloneSet.Status.CurrentRevision, "-")+1:], - CanaryRevision: cloneSet.Status.UpdateRevision[strings.LastIndex(cloneSet.Status.UpdateRevision, "-")+1:], - CanaryReplicas: cloneSet.Status.UpdatedReplicas, - CanaryReadyReplicas: cloneSet.Status.UpdatedReadyReplicas, - ObjectMeta: cloneSet.ObjectMeta, - Replicas: *cloneSet.Spec.Replicas, - PodTemplateHash: cloneSet.Status.UpdateRevision[strings.LastIndex(cloneSet.Status.UpdateRevision, "-")+1:], - IsStatusConsistent: true, + RevisionLabelKey: apps.DefaultDeploymentUniqueLabelKey, + StableRevision: cloneSet.Status.CurrentRevision[strings.LastIndex(cloneSet.Status.CurrentRevision, "-")+1:], + CanaryRevision: cloneSet.Status.UpdateRevision[strings.LastIndex(cloneSet.Status.UpdateRevision, "-")+1:], + ObjectMeta: cloneSet.ObjectMeta, + Replicas: *cloneSet.Spec.Replicas, + PodTemplateHash: cloneSet.Status.UpdateRevision[strings.LastIndex(cloneSet.Status.UpdateRevision, "-")+1:], + IsStatusConsistent: true, } // not in rollout progressing if _, ok = workload.Annotations[InRolloutProgressingAnnotation]; !ok { return workload, nil } - // in rollout progressing workload.InRolloutProgressing = true // Is it in rollback phase @@ -193,7 +186,6 @@ func (r *ControllerFinder) getDeployment(namespace string, ref *rolloutv1alpha1. if _, ok = workload.Annotations[InRolloutProgressingAnnotation]; !ok { return workload, nil } - // in rollout progressing workload.InRolloutProgressing = true // workload is continuous release, indicates rollback(v1 -> v2 -> v1) @@ -203,14 +195,11 @@ func (r *ControllerFinder) getDeployment(namespace string, ref *rolloutv1alpha1. workload.IsInRollback = true return workload, nil } - // canary deployment canary, err := r.getLatestCanaryDeployment(stable) if err != nil || canary == nil { return workload, err } - workload.CanaryReplicas = canary.Status.Replicas - workload.CanaryReadyReplicas = canary.Status.ReadyReplicas canaryRs, err := r.getDeploymentStableRs(canary) if err != nil || canaryRs == nil { return workload, err @@ -243,29 +232,24 @@ func (r *ControllerFinder) getStatefulSetLikeWorkload(namespace string, ref *rol return &Workload{IsStatusConsistent: false}, nil } workload := &Workload{ - RevisionLabelKey: apps.ControllerRevisionHashLabelKey, - StableRevision: workloadInfo.Status.StableRevision, - CanaryRevision: workloadInfo.Status.UpdateRevision, - CanaryReplicas: workloadInfo.Status.UpdatedReplicas, - CanaryReadyReplicas: workloadInfo.Status.UpdatedReadyReplicas, - ObjectMeta: workloadInfo.ObjectMeta, - Replicas: *workloadInfo.Replicas, - PodTemplateHash: workloadInfo.Status.UpdateRevision, - IsStatusConsistent: true, + RevisionLabelKey: apps.ControllerRevisionHashLabelKey, + StableRevision: workloadInfo.Status.StableRevision, + CanaryRevision: workloadInfo.Status.UpdateRevision, + ObjectMeta: workloadInfo.ObjectMeta, + Replicas: *workloadInfo.Replicas, + PodTemplateHash: workloadInfo.Status.UpdateRevision, + IsStatusConsistent: true, } // not in rollout progressing if _, ok := workload.Annotations[InRolloutProgressingAnnotation]; !ok { return workload, nil } - // in rollout progressing workload.InRolloutProgressing = true - if workloadInfo.Status.UpdateRevision == workloadInfo.Status.StableRevision && workloadInfo.Status.UpdatedReplicas != workloadInfo.Status.Replicas { workload.IsInRollback = true } - return workload, nil } diff --git a/pkg/util/parse_utils.go b/pkg/util/parse_utils.go index 6e1fccab..25e491d7 100644 --- a/pkg/util/parse_utils.go +++ b/pkg/util/parse_utils.go @@ -20,12 +20,12 @@ import ( func ParseStatefulSetInfo(object client.Object, namespacedName types.NamespacedName) *WorkloadInfo { workloadGVKWithName := fmt.Sprintf("%v(%v)", object.GetObjectKind().GroupVersionKind(), namespacedName) - selector, err := getSelector(object) + selector, err := GetSelector(object) if err != nil { klog.Errorf("Failed to parse selector for workload(%v)", workloadGVKWithName) } return &WorkloadInfo{ - ObjectMeta: *getMetadata(object), + ObjectMeta: *GetMetadata(object), MaxUnavailable: getStatefulSetMaxUnavailable(object), Replicas: pointer.Int32(GetReplicas(object)), Status: ParseWorkloadStatus(object), @@ -254,8 +254,8 @@ func GetTemplate(object client.Object) *corev1.PodTemplateSpec { } } -// getSelector can find labelSelector and return labels.Selector after parsed from it for client object -func getSelector(object client.Object) (labels.Selector, error) { +// GetSelector can find labelSelector and return labels.Selector after parsed from it for client object +func GetSelector(object client.Object) (labels.Selector, error) { switch o := object.(type) { case *apps.Deployment: return metav1.LabelSelectorAsSelector(o.Spec.Selector) @@ -272,8 +272,8 @@ func getSelector(object client.Object) (labels.Selector, error) { } } -// getMetadata can parse the whole metadata field from client workload object -func getMetadata(object client.Object) *metav1.ObjectMeta { +// GetMetadata can parse the whole metadata field from client workload object +func GetMetadata(object client.Object) *metav1.ObjectMeta { switch o := object.(type) { case *apps.Deployment: return &o.ObjectMeta diff --git a/pkg/util/parse_utils_test.go b/pkg/util/parse_utils_test.go index 493a90f1..7c8cfc23 100644 --- a/pkg/util/parse_utils_test.go +++ b/pkg/util/parse_utils_test.go @@ -314,14 +314,14 @@ func TestWorkloadParse(t *testing.T) { Expect(GetReplicas(object)).Should(BeNumerically("==", *o.Spec.Replicas)) selector, err := metav1.LabelSelectorAsSelector(o.Spec.Selector) Expect(err).NotTo(HaveOccurred()) - parsedSelector, err := getSelector(object) + parsedSelector, err := GetSelector(object) Expect(err).NotTo(HaveOccurred()) Expect(reflect.DeepEqual(parsedSelector, selector)).Should(BeTrue()) case *appsv1alpha1.CloneSet: Expect(GetReplicas(object)).Should(BeNumerically("==", *o.Spec.Replicas)) selector, err := metav1.LabelSelectorAsSelector(o.Spec.Selector) Expect(err).NotTo(HaveOccurred()) - parsedSelector, err := getSelector(object) + parsedSelector, err := GetSelector(object) Expect(err).NotTo(HaveOccurred()) Expect(reflect.DeepEqual(parsedSelector, selector)).Should(BeTrue()) case *appsv1.StatefulSet: diff --git a/pkg/util/pod_utils.go b/pkg/util/pod_utils.go index fa967a2d..abc92f83 100644 --- a/pkg/util/pod_utils.go +++ b/pkg/util/pod_utils.go @@ -113,7 +113,7 @@ func IsCompletedPod(pod *v1.Pod) bool { // ListOwnedPods will list all pods belong to workload, including terminating pods func ListOwnedPods(c client.Client, workload client.Object) ([]*v1.Pod, error) { - selector, err := getSelector(workload) + selector, err := GetSelector(workload) if err != nil { return nil, err } diff --git a/pkg/util/rollout_utils.go b/pkg/util/rollout_utils.go index 28d6481f..8e47cc97 100644 --- a/pkg/util/rollout_utils.go +++ b/pkg/util/rollout_utils.go @@ -20,6 +20,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "fmt" "strings" "time" @@ -43,16 +44,6 @@ type RolloutState struct { RolloutName string `json:"rolloutName"` } -func GetRolloutState(annotations map[string]string) (*RolloutState, error) { - value, ok := annotations[InRolloutProgressingAnnotation] - if !ok || value == "" { - return nil, nil - } - var obj *RolloutState - err := json.Unmarshal([]byte(value), &obj) - return obj, err -} - func IsRollbackInBatchPolicy(rollout *rolloutv1alpha1.Rollout, labels map[string]string) bool { // currently, only support the case of no traffic routing if len(rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { @@ -63,8 +54,7 @@ func IsRollbackInBatchPolicy(rollout *rolloutv1alpha1.Rollout, labels map[string if workloadRef.Kind == ControllerKindSts.Kind || workloadRef.Kind == ControllerKruiseKindCS.Kind || strings.EqualFold(labels[WorkloadTypeLabel], ControllerKindSts.Kind) { - value, ok := rollout.Annotations[RollbackInBatchAnnotation] - if ok && value == "true" { + if rollout.Annotations[rolloutv1alpha1.RollbackInBatchAnnotation] == "true" { return true } } @@ -161,3 +151,8 @@ func DumpJSON(o interface{}) string { by, _ := json.Marshal(o) return string(by) } + +// hash hashes `data` with sha256 and returns the hex string +func EncodeHash(data string) string { + return fmt.Sprintf("%x", sha256.Sum256([]byte(data))) +} diff --git a/pkg/webhook/workload/mutating/workload_update_handler.go b/pkg/webhook/workload/mutating/workload_update_handler.go index 5f8de092..0d1052e8 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler.go +++ b/pkg/webhook/workload/mutating/workload_update_handler.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "net/http" + "reflect" kruiseappsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" @@ -52,17 +53,24 @@ type WorkloadHandler struct { var _ admission.Handler = &WorkloadHandler{} // Handle handles admission requests. -// TODO -// Currently there is an implicit condition for rollout: the workload must be currently in a stable version (only one version of Pods), -// if not, it will not enter the rollout process. There is an additional problem here, the user may not be aware of this. -// when user does a release and thinks it enters the rollout process, but due to the implicit condition above, -// it actually goes through the normal release process. No good idea to solve this problem has been found yet. func (h *WorkloadHandler) Handle(ctx context.Context, req admission.Request) admission.Response { // if subResources, then ignore if req.Operation != admissionv1.Update || req.SubResource != "" { return admission.Allowed("") } + // Because kruise Rollout is a bypassed approach, needs to be determined in the webhook if the workload meet to enter the rollout progressing: + // 1. Traffic Routing, all the following conditions must be met + // a. PodTemplateSpec is changed + // b. Workload must only contain one version of Pods + // 2. No Traffic Routing, Only Release in batches + // a. No RolloutId + // - PodTemplateSpec is changed + // b. Configure RolloutId + // - RolloutId and PodTemplateSpec change, enter the rollout progressing. + // - RolloutId changes and PodTemplateSpec no change, enter the rollout progressing + // - RolloutId no change and PodTemplateSpec change, do not enter the rollout progressing + switch req.Kind.Group { // kruise cloneSet case kruiseappsv1alpha1.GroupVersion.Group: @@ -155,41 +163,32 @@ func (h *WorkloadHandler) Handle(ctx context.Context, req admission.Request) adm } } -func (h *WorkloadHandler) handleStatefulSetLikeWorkload(newObj, oldObj *unstructured.Unstructured) (changed bool, err error) { +func (h *WorkloadHandler) handleStatefulSetLikeWorkload(newObj, oldObj *unstructured.Unstructured) (bool, error) { // indicate whether the workload can enter the rollout process // 1. replicas > 0 - replicas := util.GetReplicas(newObj) - if replicas == 0 { + if util.GetReplicas(newObj) == 0 || !util.IsStatefulSetRollingUpdate(newObj) { return false, nil } - oldTemplate := util.GetTemplate(oldObj) - if oldTemplate == nil { + oldTemplate, newTemplate := util.GetTemplate(oldObj), util.GetTemplate(newObj) + if oldTemplate == nil || newTemplate == nil { return false, nil } - newTemplate := util.GetTemplate(newObj) - if newTemplate == nil { + oldMetadata, newMetadata := util.GetMetadata(oldObj), util.GetMetadata(newObj) + if newMetadata.Annotations[appsv1alpha1.RolloutIDLabel] != "" && + oldMetadata.Annotations[appsv1alpha1.RolloutIDLabel] == newMetadata.Annotations[appsv1alpha1.RolloutIDLabel] { + return false, nil + } else if newMetadata.Annotations[appsv1alpha1.RolloutIDLabel] == "" && util.EqualIgnoreHash(oldTemplate, newTemplate) { return false, nil } - // 2. statefulset.spec.template is changed - if util.EqualIgnoreHash(oldTemplate, newTemplate) { - return - } - // 3. have matched rollout crd rollout, err := h.fetchMatchedRollout(newObj) if err != nil { - return - } else if rollout == nil { - return - } - - klog.Infof("StatefulSet-Like Workload(%s/%s) will be in rollout progressing, and paused", newObj.GetNamespace(), newObj.GetName()) - if !util.IsStatefulSetRollingUpdate(newObj) { - return + return false, err + } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + return false, nil } - changed = true - util.SetStatefulSetPartition(newObj, replicas) + util.SetStatefulSetPartition(newObj, util.GetReplicas(newObj)) state := &util.RolloutState{RolloutName: rollout.Name} by, _ := json.Marshal(state) annotation := newObj.GetAnnotations() @@ -198,53 +197,50 @@ func (h *WorkloadHandler) handleStatefulSetLikeWorkload(newObj, oldObj *unstruct } annotation[util.InRolloutProgressingAnnotation] = string(by) newObj.SetAnnotations(annotation) - return + klog.Infof("StatefulSet(%s/%s) will be released incrementally based on Rollout(%s)", newMetadata.Namespace, newMetadata.Name, rollout.Name) + return true, nil } -func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (changed bool, err error) { +func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (bool, error) { // in rollout progressing - if state, _ := util.GetRolloutState(newObj.Annotations); state != nil { - // deployment paused=false is not allowed until the rollout is completed - if newObj.Spec.Paused == false { - changed = true + if newObj.Annotations[util.InRolloutProgressingAnnotation] != "" { + if !newObj.Spec.Paused || !reflect.DeepEqual(newObj.Spec.Strategy, oldObj.Spec.Strategy) { newObj.Spec.Paused = true - klog.Warningf("deployment(%s/%s) is in rollout(%s) progressing, and set paused=true", newObj.Namespace, newObj.Name, state.RolloutName) + newObj.Spec.Strategy = oldObj.Spec.Strategy + klog.Warningf("deployment(%s/%s) is in rollout progressing, and do not modify strategy", newObj.Namespace, newObj.Name) + return true, nil } - return + return false, nil } // indicate whether the workload can enter the rollout process - // 1. replicas > 0 + // replicas > 0 if newObj.Spec.Replicas != nil && *newObj.Spec.Replicas == 0 { - return - } - // 2. deployment.spec.strategy.type must be RollingUpdate - if newObj.Spec.Strategy.Type == apps.RecreateDeploymentStrategyType { - klog.Warningf("deployment(%s/%s) strategy type is 'Recreate', rollout will not work on it", newObj.Namespace, newObj.Name) - return - } - // 3. deployment.spec.PodTemplate not change - if util.EqualIgnoreHash(&oldObj.Spec.Template, &newObj.Spec.Template) { - return + return false, nil } - // 4. the deployment must be in a stable version (only one version of rs) - rss, err := h.Finder.GetReplicaSetsForDeployment(newObj) - if err != nil { - return - } else if len(rss) != 1 { - klog.Warningf("deployment(%s/%s) contains len(%d) replicaSet, can't in rollout progressing", newObj.Namespace, newObj.Name, len(rss)) - return + if newObj.Annotations[appsv1alpha1.RolloutIDLabel] != "" && + oldObj.Annotations[appsv1alpha1.RolloutIDLabel] == newObj.Annotations[appsv1alpha1.RolloutIDLabel] { + return false, nil + } else if newObj.Annotations[appsv1alpha1.RolloutIDLabel] == "" && util.EqualIgnoreHash(&oldObj.Spec.Template, &newObj.Spec.Template) { + return false, nil } - // 5. have matched rollout crd + rollout, err := h.fetchMatchedRollout(newObj) if err != nil { - return - } else if rollout == nil { - return + return false, err + } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + return false, nil + } + // if traffic routing, workload must only be one version of Pods + if len(rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { + if rss, err := h.Finder.GetReplicaSetsForDeployment(newObj); err != nil { + return false, nil + } else if len(rss) != 1 { + klog.Warningf("Because deployment(%s/%s) have multiple versions of Pods, so can not enter rollout progressing", newObj.Namespace, newObj.Name) + return false, nil + } } - klog.Infof("deployment(%s/%s) will be in rollout progressing, and set paused=true", newObj.Namespace, newObj.Name) - changed = true // need set workload paused = true newObj.Spec.Paused = true state := &util.RolloutState{RolloutName: rollout.Name} @@ -253,29 +249,35 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (cha newObj.Annotations = map[string]string{} } newObj.Annotations[util.InRolloutProgressingAnnotation] = string(by) - return + klog.Infof("Deployment(%s/%s) will be released incrementally based on Rollout(%s)", newObj.Namespace, newObj.Name, rollout.Name) + return true, nil } -func (h *WorkloadHandler) handleCloneSet(newObj, oldObj *kruiseappsv1alpha1.CloneSet) (changed bool, err error) { +func (h *WorkloadHandler) handleCloneSet(newObj, oldObj *kruiseappsv1alpha1.CloneSet) (bool, error) { // indicate whether the workload can enter the rollout process - // 1. replicas > 0 + // when cloneSet don't contain any pods, no need to enter rollout progressing if newObj.Spec.Replicas != nil && *newObj.Spec.Replicas == 0 { - return + return false, nil } - // 2. cloneSet.spec.PodTemplate is changed - if util.EqualIgnoreHash(&oldObj.Spec.Template, &newObj.Spec.Template) { - return + if newObj.Annotations[appsv1alpha1.RolloutIDLabel] != "" && + oldObj.Annotations[appsv1alpha1.RolloutIDLabel] == newObj.Annotations[appsv1alpha1.RolloutIDLabel] { + return false, nil + } else if newObj.Annotations[appsv1alpha1.RolloutIDLabel] == "" && util.EqualIgnoreHash(&oldObj.Spec.Template, &newObj.Spec.Template) { + return false, nil } - // 3. have matched rollout crd + rollout, err := h.fetchMatchedRollout(newObj) if err != nil { - return - } else if rollout == nil { - return + return false, err + } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + return false, nil + } + // if traffic routing, there must only be one version of Pods + if len(rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 && newObj.Status.Replicas != newObj.Status.UpdatedReplicas { + klog.Warningf("Because cloneSet(%s/%s) have multiple versions of Pods, so can not enter rollout progressing", newObj.Namespace, newObj.Name) + return false, nil } - klog.Infof("cloneSet(%s/%s) will be in rollout progressing, and paused", newObj.Namespace, newObj.Name) - changed = true // need set workload paused = true newObj.Spec.UpdateStrategy.Paused = true newObj.Spec.UpdateStrategy.Partition = &intstr.IntOrString{Type: intstr.String, StrVal: "100%"} @@ -285,7 +287,8 @@ func (h *WorkloadHandler) handleCloneSet(newObj, oldObj *kruiseappsv1alpha1.Clon newObj.Annotations = map[string]string{} } newObj.Annotations[util.InRolloutProgressingAnnotation] = string(by) - return + klog.Infof("CloneSet(%s/%s) will be released incrementally based on Rollout(%s)", newObj.Namespace, newObj.Name, rollout.Name) + return true, nil } func (h *WorkloadHandler) fetchMatchedRollout(obj client.Object) (*appsv1alpha1.Rollout, error) { diff --git a/pkg/webhook/workload/mutating/workload_update_handler_test.go b/pkg/webhook/workload/mutating/workload_update_handler_test.go index 04098b22..254f48c1 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler_test.go +++ b/pkg/webhook/workload/mutating/workload_update_handler_test.go @@ -220,6 +220,9 @@ var ( Name: "echoserver", }, }, + Strategy: appsv1alpha1.RolloutStrategy{ + Canary: &appsv1alpha1.CanaryStrategy{}, + }, }, } ) @@ -312,7 +315,18 @@ func TestHandlerDeployment(t *testing.T) { return []*apps.ReplicaSet{rs1, rs2} }, getRollout: func() *appsv1alpha1.Rollout { - return rolloutDemo.DeepCopy() + demo := rolloutDemo.DeepCopy() + demo.Spec.Strategy.Canary = &appsv1alpha1.CanaryStrategy{ + TrafficRoutings: []*appsv1alpha1.TrafficRouting{ + { + Service: "echoserver", + Ingress: &appsv1alpha1.IngressTrafficRouting{ + Name: "echoserver", + }, + }, + }, + } + return demo }, }, { @@ -372,6 +386,81 @@ func TestHandlerDeployment(t *testing.T) { return obj }, }, + { + name: "rolloutId and podTemplateSpec changed", + getObjs: func() (*apps.Deployment, *apps.Deployment) { + oldObj := deploymentDemo.DeepCopy() + newObj := deploymentDemo.DeepCopy() + newObj.Annotations[appsv1alpha1.RolloutIDLabel] = "v2" + newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + return oldObj, newObj + }, + expectObj: func() *apps.Deployment { + obj := deploymentDemo.DeepCopy() + obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + obj.Spec.Paused = true + obj.Annotations[appsv1alpha1.RolloutIDLabel] = "v2" + return obj + }, + getRs: func() []*apps.ReplicaSet { + rs := rsDemo.DeepCopy() + return []*apps.ReplicaSet{rs} + }, + getRollout: func() *appsv1alpha1.Rollout { + obj := rolloutDemo.DeepCopy() + return obj + }, + }, + { + name: "rolloutId change, and podTemplateSpec no change", + getObjs: func() (*apps.Deployment, *apps.Deployment) { + oldObj := deploymentDemo.DeepCopy() + newObj := deploymentDemo.DeepCopy() + newObj.Annotations[appsv1alpha1.RolloutIDLabel] = "v1-alpha1" + return oldObj, newObj + }, + expectObj: func() *apps.Deployment { + obj := deploymentDemo.DeepCopy() + obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}` + obj.Spec.Paused = true + obj.Annotations[appsv1alpha1.RolloutIDLabel] = "v1-alpha1" + return obj + }, + getRs: func() []*apps.ReplicaSet { + rs := rsDemo.DeepCopy() + return []*apps.ReplicaSet{rs} + }, + getRollout: func() *appsv1alpha1.Rollout { + obj := rolloutDemo.DeepCopy() + return obj + }, + }, + { + name: "rolloutId no change, and podTemplateSpec change", + getObjs: func() (*apps.Deployment, *apps.Deployment) { + oldObj := deploymentDemo.DeepCopy() + oldObj.Annotations[appsv1alpha1.RolloutIDLabel] = "v1" + newObj := deploymentDemo.DeepCopy() + newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + newObj.Annotations[appsv1alpha1.RolloutIDLabel] = "v1" + return oldObj, newObj + }, + expectObj: func() *apps.Deployment { + obj := deploymentDemo.DeepCopy() + obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2" + obj.Annotations[appsv1alpha1.RolloutIDLabel] = "v1" + return obj + }, + getRs: func() []*apps.ReplicaSet { + rs := rsDemo.DeepCopy() + return []*apps.ReplicaSet{rs} + }, + getRollout: func() *appsv1alpha1.Rollout { + obj := rolloutDemo.DeepCopy() + return obj + }, + }, } decoder, _ := admission.NewDecoder(scheme) diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index 79b293fd..c1ca11bd 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -27,7 +27,7 @@ import ( . "github.com/onsi/gomega" appsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" appsv1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" - rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/api/v1alpha1" "github.com/openkruise/rollouts/pkg/util" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -51,10 +51,10 @@ var _ = SIGDescribe("Rollout", func() { var namespace string DumpAllResources := func() { - rollout := &rolloutsv1alpha1.RolloutList{} + rollout := &v1alpha1.RolloutList{} k8sClient.List(context.TODO(), rollout, client.InNamespace(namespace)) fmt.Println(util.DumpJSON(rollout)) - batch := &rolloutsv1alpha1.BatchReleaseList{} + batch := &v1alpha1.BatchReleaseList{} k8sClient.List(context.TODO(), batch, client.InNamespace(namespace)) fmt.Println(util.DumpJSON(batch)) deploy := &apps.DeploymentList{} @@ -156,10 +156,10 @@ var _ = SIGDescribe("Rollout", func() { return clone } - UpdateRollout := func(object *rolloutsv1alpha1.Rollout) *rolloutsv1alpha1.Rollout { - var clone *rolloutsv1alpha1.Rollout + UpdateRollout := func(object *v1alpha1.Rollout) *v1alpha1.Rollout { + var clone *v1alpha1.Rollout Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { - clone = &rolloutsv1alpha1.Rollout{} + clone = &v1alpha1.Rollout{} err := GetObject(object.Name, clone) if err != nil { return err @@ -173,14 +173,14 @@ var _ = SIGDescribe("Rollout", func() { ResumeRolloutCanary := func(name string) { Eventually(func() bool { - clone := &rolloutsv1alpha1.Rollout{} + clone := &v1alpha1.Rollout{} Expect(GetObject(name, clone)).NotTo(HaveOccurred()) - if clone.Status.CanaryStatus.CurrentStepState != rolloutsv1alpha1.CanaryStepStatePaused { + if clone.Status.CanaryStatus.CurrentStepState != v1alpha1.CanaryStepStatePaused { fmt.Println("resume rollout success, and CurrentStepState", util.DumpJSON(clone.Status)) return true } - body := fmt.Sprintf(`{"status":{"canaryStatus":{"currentStepState":"%s"}}}`, rolloutsv1alpha1.CanaryStepStateReady) + body := fmt.Sprintf(`{"status":{"canaryStatus":{"currentStepState":"%s"}}}`, v1alpha1.CanaryStepStateReady) Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) return false }, 10*time.Second, time.Second).Should(BeTrue()) @@ -238,19 +238,19 @@ var _ = SIGDescribe("Rollout", func() { DumpAllResources() Expect(true).Should(BeFalse()) } - clone := &rolloutsv1alpha1.Rollout{} + clone := &v1alpha1.Rollout{} Expect(GetObject(name, clone)).NotTo(HaveOccurred()) if clone.Status.CanaryStatus == nil { return false } klog.Infof("current step:%v target step:%v current step state %v", clone.Status.CanaryStatus.CurrentStepIndex, stepIndex, clone.Status.CanaryStatus.CurrentStepState) - return clone.Status.CanaryStatus.CurrentStepIndex == stepIndex && clone.Status.CanaryStatus.CurrentStepState == rolloutsv1alpha1.CanaryStepStatePaused + return clone.Status.CanaryStatus.CurrentStepIndex == stepIndex && clone.Status.CanaryStatus.CurrentStepState == v1alpha1.CanaryStepStatePaused }, 20*time.Minute, time.Second).Should(BeTrue()) } - WaitRolloutStatusPhase := func(name string, phase rolloutsv1alpha1.RolloutPhase) { + WaitRolloutStatusPhase := func(name string, phase v1alpha1.RolloutPhase) { Eventually(func() bool { - clone := &rolloutsv1alpha1.Rollout{} + clone := &v1alpha1.Rollout{} Expect(GetObject(name, clone)).NotTo(HaveOccurred()) return clone.Status.Phase == phase }, 20*time.Minute, time.Second).Should(BeTrue()) @@ -258,7 +258,7 @@ var _ = SIGDescribe("Rollout", func() { WaitRolloutWorkloadGeneration := func(name string, generation int64) { Eventually(func() bool { - clone := &rolloutsv1alpha1.Rollout{} + clone := &v1alpha1.Rollout{} Expect(GetObject(name, clone)).NotTo(HaveOccurred()) return clone.Status.CanaryStatus.ObservedWorkloadGeneration == generation }, time.Minute, time.Second).Should(BeTrue()) @@ -266,7 +266,7 @@ var _ = SIGDescribe("Rollout", func() { WaitRolloutNotFound := func(name string) { Eventually(func() bool { - clone := &rolloutsv1alpha1.Rollout{} + clone := &v1alpha1.Rollout{} err := GetObject(name, clone) if err == nil { return false @@ -317,8 +317,8 @@ var _ = SIGDescribe("Rollout", func() { count := 0 for _, pod := range pods { - if pod.Labels[util.RolloutIDLabel] == rolloutID && - pod.Labels[util.RolloutBatchIDLabel] == batchID { + if pod.Labels[v1alpha1.RolloutIDLabel] == rolloutID && + pod.Labels[v1alpha1.RolloutBatchIDLabel] == batchID { count++ } } @@ -339,8 +339,8 @@ var _ = SIGDescribe("Rollout", func() { By("[TEST] Clean up resources after an integration test") k8sClient.DeleteAllOf(context.TODO(), &apps.Deployment{}, client.InNamespace(namespace)) k8sClient.DeleteAllOf(context.TODO(), &appsv1alpha1.CloneSet{}, client.InNamespace(namespace)) - k8sClient.DeleteAllOf(context.TODO(), &rolloutsv1alpha1.BatchRelease{}, client.InNamespace(namespace)) - k8sClient.DeleteAllOf(context.TODO(), &rolloutsv1alpha1.Rollout{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &v1alpha1.BatchRelease{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &v1alpha1.Rollout{}, client.InNamespace(namespace)) k8sClient.DeleteAllOf(context.TODO(), &v1.Service{}, client.InNamespace(namespace)) k8sClient.DeleteAllOf(context.TODO(), &netv1.Ingress{}, client.InNamespace(namespace)) Expect(k8sClient.Delete(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed()) @@ -350,9 +350,9 @@ var _ = SIGDescribe("Rollout", func() { KruiseDescribe("Deployment canary rollout with Ingress", func() { It("V1->V2: Percentage 20%,40%,60%,80%,90%, and replicas=3", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), }, @@ -389,7 +389,7 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -409,7 +409,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1)) cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("20")) // resume rollout canary @@ -420,7 +420,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 2)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 2)) - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("40")) // resume rollout canary @@ -431,7 +431,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 2)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 2)) - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("60")) // resume rollout canary @@ -442,7 +442,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 3)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 3)) - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("80")) // resume rollout canary @@ -453,12 +453,12 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 3)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 3)) - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("90")) // resume rollout ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) By("rollout completed, and check") // check service & ingress & deployment // ingress @@ -483,8 +483,8 @@ var _ = SIGDescribe("Rollout", func() { } // check progressing succeed Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) @@ -493,18 +493,18 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage 20%,60% Succeeded", func() { finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) replicas := intstr.FromInt(2) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), Replicas: &replicas, - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, } CreateObject(rollout) @@ -531,8 +531,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -574,18 +574,18 @@ var _ = SIGDescribe("Rollout", func() { Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) // canary ingress cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 2)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 2)) @@ -599,7 +599,7 @@ var _ = SIGDescribe("Rollout", func() { // check stable, canary service & ingress // canary ingress cIngress = &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[1].Weight))) // canary deployment cWorkload, err = GetCanaryDeployment(workload) @@ -615,7 +615,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) By("rollout completed, and check") // check service & ingress & deployment // ingress @@ -645,11 +645,11 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) // scale up replicas 5 -> 6 @@ -666,7 +666,7 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage 20%, and rollback(v1)", func() { finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) CreateObject(rollout) @@ -698,8 +698,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -725,7 +725,7 @@ var _ = SIGDescribe("Rollout", func() { UpdateDeployment(workload) By("Rollback deployment env NODE_NAME from(version2) -> to(version1)") - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) klog.Infof("rollout(%s) completed, and check", namespace) time.Sleep(time.Second * 10) @@ -761,9 +761,9 @@ var _ = SIGDescribe("Rollout", func() { // check progressing canceled Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonCanceled)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCanceled)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(string(cond.Status)).Should(Equal("False")) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) }) @@ -771,7 +771,7 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage 20%,40% and continuous release v3", func() { finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) CreateObject(rollout) @@ -797,8 +797,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -827,8 +827,8 @@ var _ = SIGDescribe("Rollout", func() { canaryRevisionV1 := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevisionV1)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) @@ -854,8 +854,8 @@ var _ = SIGDescribe("Rollout", func() { canaryRevisionV2 := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevisionV2)) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1)) @@ -866,11 +866,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevisionV2)) // canary ingress cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) // canary deployment @@ -887,7 +887,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout canary ResumeRolloutCanary(rollout.Name) By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) klog.Infof("rollout(%s) completed, and check", namespace) // check service & ingress & deployment @@ -913,39 +913,39 @@ var _ = SIGDescribe("Rollout", func() { } // check progressing succeed Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevisionV2)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevisionV2)) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage 20%,40% and scale up replicas from(5) -> to(10)", func() { finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { Weight: utilpointer.Int32(40), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { Weight: utilpointer.Int32(100), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, }, @@ -968,7 +968,7 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -994,7 +994,7 @@ var _ = SIGDescribe("Rollout", func() { canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) @@ -1008,17 +1008,17 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 4)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 4)) - Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1alpha1.CanaryStepStatePaused)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) // resume rollout canary ResumeRolloutCanary(rollout.Name) By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) klog.Infof("rollout(%s) completed, and check", namespace) // check service & ingress & deployment @@ -1044,39 +1044,39 @@ var _ = SIGDescribe("Rollout", func() { } // check progressing succeed Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage 20%,40%, and scale down replicas from(10) -> to(5)", func() { finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { Weight: utilpointer.Int32(40), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(100), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, }, @@ -1100,7 +1100,7 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -1124,7 +1124,7 @@ var _ = SIGDescribe("Rollout", func() { canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) // scale up replicas, 10 -> 5 @@ -1135,17 +1135,17 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 6)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 6)) - Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1alpha1.CanaryStepStatePaused)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) // resume rollout canary ResumeRolloutCanary(rollout.Name) By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) By("rollout completed, and check") // check service & ingress & deployment @@ -1171,47 +1171,47 @@ var _ = SIGDescribe("Rollout", func() { } // check progressing succeed Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage 20%,40%, paused and resume", func() { finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { Weight: utilpointer.Int32(40), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { Weight: utilpointer.Int32(80), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { Weight: utilpointer.Int32(100), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, }, @@ -1239,8 +1239,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -1261,21 +1261,21 @@ var _ = SIGDescribe("Rollout", func() { UpdateRollout(rollout) // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) cIndex := rollout.Status.CanaryStatus.CurrentStepIndex time.Sleep(time.Second * 15) // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonPaused)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonPaused)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", cIndex)) // resume rollout rollout.Spec.Strategy.Paused = false UpdateRollout(rollout) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) klog.Infof("rollout(%s) completed, and check", namespace) // check service & ingress & deployment @@ -1301,8 +1301,8 @@ var _ = SIGDescribe("Rollout", func() { } // check progressing succeed Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond = util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond = util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) @@ -1311,24 +1311,24 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage 20%,40%, but delete rollout crd", func() { finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { Weight: utilpointer.Int32(100), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, @@ -1356,8 +1356,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -1373,7 +1373,7 @@ var _ = SIGDescribe("Rollout", func() { By("check deployment status & paused success") // delete rollout - Expect(k8sClient.DeleteAllOf(context.TODO(), &rolloutsv1alpha1.Rollout{}, client.InNamespace(namespace), client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed()) + Expect(k8sClient.DeleteAllOf(context.TODO(), &v1alpha1.Rollout{}, client.InNamespace(namespace), client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed()) WaitRolloutNotFound(rollout.Name) WaitDeploymentAllPodsReady(workload) // check service & ingress & deployment @@ -1402,24 +1402,24 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage 20% v2 failed image, and v3 succeed image", func() { finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, { Weight: utilpointer.Int32(100), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, @@ -1447,8 +1447,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -1466,11 +1466,11 @@ var _ = SIGDescribe("Rollout", func() { time.Sleep(time.Second * 10) // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 0)) - Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateUpgrade)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1alpha1.CanaryStepStateUpgrade)) // update success image, v3 newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) @@ -1479,7 +1479,7 @@ var _ = SIGDescribe("Rollout", func() { UpdateDeployment(workload) By("Update deployment image from(v2) -> to(v3)") // wait rollout complete - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) klog.Infof("rollout(%s) completed, and check", namespace) // check service & ingress & deployment @@ -1505,8 +1505,8 @@ var _ = SIGDescribe("Rollout", func() { } // check progressing succeed Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) @@ -1515,7 +1515,7 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage, 20%,40%,60%,80%,100%, steps changed v1", func() { finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) CreateObject(rollout) By("Creating workload and waiting for all pods ready...") @@ -1548,18 +1548,18 @@ var _ = SIGDescribe("Rollout", func() { WaitRolloutCanaryStepPaused(rollout.Name, 1) // update rollout step configuration - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(10), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(30), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(100), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(5), }, }, @@ -1568,7 +1568,7 @@ var _ = SIGDescribe("Rollout", func() { By("update rollout configuration, and wait rollout next step(2)") // wait step 1 complete WaitRolloutCanaryStepPaused(rollout.Name, 2) - batch := &rolloutsv1alpha1.BatchRelease{} + batch := &v1alpha1.BatchRelease{} Expect(GetObject(rollout.Name, batch)).NotTo(HaveOccurred()) // canary deployment @@ -1580,8 +1580,8 @@ var _ = SIGDescribe("Rollout", func() { canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) // check stable, canary service & ingress @@ -1590,11 +1590,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) // canary ingress cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[1].Weight))) // canary deployment @@ -1606,7 +1606,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout canary ResumeRolloutCanary(rollout.Name) // wait rollout complete - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) klog.Infof("rollout(%s) completed, and check", namespace) // check service & ingress & deployment @@ -1632,8 +1632,8 @@ var _ = SIGDescribe("Rollout", func() { } // check progressing succeed Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) @@ -1643,9 +1643,9 @@ var _ = SIGDescribe("Rollout", func() { KruiseDescribe("Canary rollout with Gateway API", func() { It("V1->V2: Percentage 20%,40%,60%,80%,90%, and replicas=3", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/gateway/rollout-test.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), }, @@ -1682,7 +1682,7 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -1762,7 +1762,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) By("rollout completed, and check") // check service & httproute & deployment // httproute @@ -1788,8 +1788,8 @@ var _ = SIGDescribe("Rollout", func() { } // check progressing succeed Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) @@ -1797,21 +1797,21 @@ var _ = SIGDescribe("Rollout", func() { }) KruiseDescribe("CloneSet canary rollout with Ingress", func() { - It("V1->V2: Percentage, 20%,60% Succeeded", func() { + It("CloneSet V1->V2: Percentage, 20%,60% Succeeded", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, } - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1alpha1", Kind: "CloneSet", Name: "echoserver", @@ -1836,9 +1836,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -1858,8 +1858,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash @@ -1871,11 +1871,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) // canary ingress cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) @@ -1887,7 +1887,7 @@ var _ = SIGDescribe("Rollout", func() { // check stable, canary service & ingress // canary ingress cIngress = &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[1].Weight))) // cloneset Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) @@ -1897,7 +1897,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitCloneSetAllPodsReady(workload) By("rollout completed, and check") @@ -1929,11 +1929,11 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) // scale up replicas 5 -> 6 workload.Spec.Replicas = utilpointer.Int32(6) @@ -1948,9 +1948,9 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage, 20%, and rollback(v1)", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1alpha1", Kind: "CloneSet", Name: "echoserver", @@ -1976,9 +1976,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -1999,12 +1999,12 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateUpgrade)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1alpha1.CanaryStepStateUpgrade)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) // resume rollout canary @@ -2019,15 +2019,15 @@ var _ = SIGDescribe("Rollout", func() { By("Rollback deployment env NODE_NAME from(version2) -> to(version1)") time.Sleep(time.Second * 2) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitCloneSetAllPodsReady(workload) By("rollout completed, and check") // check progressing canceled Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonCanceled)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCanceled)) Expect(string(cond.Status)).Should(Equal("False")) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) // check service & ingress & deployment // ingress @@ -2056,9 +2056,9 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1alpha1", Kind: "CloneSet", Name: "echoserver", @@ -2083,9 +2083,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -2106,8 +2106,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) canaryRevisionV1 := rollout.Status.CanaryStatus.PodTemplateHash @@ -2130,10 +2130,10 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) @@ -2145,18 +2145,18 @@ var _ = SIGDescribe("Rollout", func() { Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevisionV2)) // canary ingress cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) // resume rollout canary ResumeRolloutCanary(rollout.Name) By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitCloneSetAllPodsReady(workload) By("rollout completed, and check") @@ -2188,25 +2188,25 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevisionV2)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevisionV2)) }) It("V1->V2: disable quickly rollback policy without traffic routing", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1alpha1", Kind: "CloneSet", Name: "echoserver", } rollout.Spec.Strategy.Canary.TrafficRoutings = nil rollout.Annotations = map[string]string{ - util.RollbackInBatchAnnotation: "true", + v1alpha1.RollbackInBatchAnnotation: "true", } CreateObject(rollout) @@ -2220,9 +2220,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -2243,8 +2243,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) @@ -2272,7 +2272,7 @@ var _ = SIGDescribe("Rollout", func() { WaitRolloutCanaryStepPaused(rollout.Name, 4) By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitCloneSetAllPodsReady(workload) By("rollout completed, and check") @@ -2291,17 +2291,17 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1alpha1", Kind: "CloneSet", Name: "echoserver", @@ -2333,9 +2333,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -2355,8 +2355,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash @@ -2365,7 +2365,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitCloneSetAllPodsReady(workload) By("rollout completed, and check") @@ -2382,11 +2382,11 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) // scale up replicas 5 -> 6 workload.Spec.Replicas = utilpointer.Int32(6) @@ -2405,19 +2405,19 @@ var _ = SIGDescribe("Rollout", func() { KruiseDescribe("Native StatefulSet rollout canary with Ingress", func() { It("V1->V2: Percentage, 20%,60% Succeeded", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, } - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps/v1", Kind: "StatefulSet", Name: "echoserver", @@ -2446,9 +2446,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -2468,8 +2468,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash @@ -2481,11 +2481,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) Expect(cService.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(canaryRevision)) // canary ingress cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) @@ -2497,7 +2497,7 @@ var _ = SIGDescribe("Rollout", func() { // check stable, canary service & ingress // canary ingress cIngress = &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[1].Weight))) // cloneset Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) @@ -2507,7 +2507,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitNativeStatefulSetPodsReady(workload) By("rollout completed, and check") @@ -2537,11 +2537,11 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) // scale up replicas 5 -> 6 workload.Spec.Replicas = utilpointer.Int32(6) @@ -2556,9 +2556,9 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps/v1", Kind: "StatefulSet", Name: "echoserver", @@ -2586,9 +2586,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -2607,8 +2607,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) canaryRevisionV1 := rollout.Status.CanaryStatus.PodTemplateHash @@ -2631,9 +2631,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) @@ -2645,18 +2645,18 @@ var _ = SIGDescribe("Rollout", func() { Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) Expect(cService.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(canaryRevisionV2)) // canary ingress cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) // resume rollout canary ResumeRolloutCanary(rollout.Name) By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitNativeStatefulSetPodsReady(workload) By("rollout completed, and check") @@ -2685,18 +2685,18 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevisionV2)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevisionV2)) }) It("V1->V2: Percentage, 20%, and rollback(v1)", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps/v1", Kind: "StatefulSet", Name: "echoserver", @@ -2726,9 +2726,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -2747,12 +2747,12 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateUpgrade)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1alpha1.CanaryStepStateUpgrade)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) // resume rollout canary @@ -2772,15 +2772,15 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%v-%v", workload.Name, *workload.Spec.Replicas-1), brokenPod)).NotTo(HaveOccurred()) Expect(k8sClient.Delete(context.TODO(), brokenPod)).NotTo(HaveOccurred()) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitNativeStatefulSetPodsReady(workload) By("rollout completed, and check") // check progressing canceled Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonCanceled)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCanceled)) Expect(string(cond.Status)).Should(Equal("False")) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) // check service & ingress & deployment // ingress @@ -2807,9 +2807,9 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps/v1", Kind: "StatefulSet", Name: "echoserver", @@ -2839,9 +2839,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -2859,8 +2859,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash @@ -2869,7 +2869,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitNativeStatefulSetPodsReady(workload) By("rollout completed, and check") @@ -2883,11 +2883,11 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) // scale up replicas 5 -> 6 workload.Spec.Replicas = utilpointer.Int32(6) @@ -2904,19 +2904,19 @@ var _ = SIGDescribe("Rollout", func() { KruiseDescribe("Advanced StatefulSet rollout canary with Ingress", func() { It("V1->V2: Percentage, 20%,60% Succeeded", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, } - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1beta1", Kind: "StatefulSet", Name: "echoserver", @@ -2945,9 +2945,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -2967,8 +2967,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash @@ -2980,11 +2980,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) Expect(cService.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(canaryRevision)) // canary ingress cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) @@ -2996,7 +2996,7 @@ var _ = SIGDescribe("Rollout", func() { // check stable, canary service & ingress // canary ingress cIngress = &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[1].Weight))) // workload time.Sleep(time.Second * 10) @@ -3007,7 +3007,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitAdvancedStatefulSetPodsReady(workload) By("rollout completed, and check") @@ -3037,11 +3037,11 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) // scale up replicas 5 -> 6 workload.Spec.Replicas = utilpointer.Int32(6) @@ -3056,9 +3056,9 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1beta1", Kind: "StatefulSet", Name: "echoserver", @@ -3086,9 +3086,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -3107,8 +3107,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) canaryRevisionV1 := rollout.Status.CanaryStatus.PodTemplateHash @@ -3131,9 +3131,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) @@ -3145,18 +3145,18 @@ var _ = SIGDescribe("Rollout", func() { Expect(service.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) Expect(cService.Spec.Selector[apps.ControllerRevisionHashLabelKey]).Should(Equal(canaryRevisionV2)) // canary ingress cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", *rollout.Spec.Strategy.Canary.Steps[0].Weight))) // resume rollout canary ResumeRolloutCanary(rollout.Name) By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitAdvancedStatefulSetPodsReady(workload) By("rollout completed, and check") @@ -3185,18 +3185,18 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevisionV2)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevisionV2)) }) It("V1->V2: Percentage, 20%, and rollback(v1)", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1beta1", Kind: "StatefulSet", Name: "echoserver", @@ -3226,9 +3226,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -3247,12 +3247,12 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateUpgrade)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(v1alpha1.CanaryStepStateUpgrade)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) // resume rollout canary @@ -3272,15 +3272,15 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%v-%v", workload.Name, *workload.Spec.Replicas-1), brokenPod)).NotTo(HaveOccurred()) Expect(k8sClient.Delete(context.TODO(), brokenPod)).NotTo(HaveOccurred()) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitAdvancedStatefulSetPodsReady(workload) By("rollout completed, and check") // check progressing canceled Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonCanceled)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCanceled)) Expect(string(cond.Status)).Should(Equal("False")) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) // check service & ingress & deployment // ingress @@ -3307,9 +3307,9 @@ var _ = SIGDescribe("Rollout", func() { It("V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1beta1", Kind: "StatefulSet", Name: "echoserver", @@ -3339,9 +3339,9 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + stableRevision := rollout.Status.CanaryStatus.StableRevision By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -3359,8 +3359,8 @@ var _ = SIGDescribe("Rollout", func() { // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision)) canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash @@ -3369,7 +3369,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitAdvancedStatefulSetPodsReady(workload) By("rollout completed, and check") @@ -3383,11 +3383,11 @@ var _ = SIGDescribe("Rollout", func() { // check progressing succeed Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) - Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) - Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(canaryRevision)) // scale up replicas 5 -> 6 workload.Spec.Replicas = utilpointer.Int32(6) @@ -3405,9 +3405,9 @@ var _ = SIGDescribe("Rollout", func() { KruiseDescribe("Others", func() { It("Patch batch id to pods: normal case", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1beta1", Kind: "StatefulSet", Name: "echoserver", @@ -3431,7 +3431,7 @@ var _ = SIGDescribe("Rollout", func() { // workload workload := &appsv1beta1.StatefulSet{} Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) - workload.Labels[util.RolloutIDLabel] = "1" + workload.Labels[v1alpha1.RolloutIDLabel] = "1" CreateObject(workload) WaitAdvancedStatefulSetPodsReady(workload) @@ -3454,7 +3454,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout ResumeRolloutCanary(rollout.Name) By("check rollout canary status success, resume rollout, and wait rollout canary complete") - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitAdvancedStatefulSetPodsReady(workload) // check batch id after rollout @@ -3468,33 +3468,33 @@ var _ = SIGDescribe("Rollout", func() { It("patch batch id to pods: scaling case", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1alpha1", Kind: "CloneSet", Name: "echoserver", } - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { Weight: utilpointer.Int32(40), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, { Weight: utilpointer.Int32(100), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(10), }, }, @@ -3512,7 +3512,7 @@ var _ = SIGDescribe("Rollout", func() { // workload workload := &appsv1alpha1.CloneSet{} Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - workload.Labels[util.RolloutIDLabel] = "1" + workload.Labels[v1alpha1.RolloutIDLabel] = "1" CreateObject(workload) WaitCloneSetAllPodsReady(workload) @@ -3535,7 +3535,7 @@ var _ = SIGDescribe("Rollout", func() { workload.Spec.Replicas = utilpointer.Int32(10) UpdateCloneSet(workload) Eventually(func() bool { - object := &rolloutsv1alpha1.Rollout{} + object := &v1alpha1.Rollout{} Expect(GetObject(rollout.Name, object)).NotTo(HaveOccurred()) return object.Status.CanaryStatus.CanaryReadyReplicas == 4 }, 5*time.Minute, time.Second).Should(BeTrue()) @@ -3548,7 +3548,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout canary By("check rollout canary status success, resume rollout, and wait rollout canary complete") ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) WaitCloneSetAllPodsReady(workload) By("rollout completed, and check pod batch label") @@ -3560,37 +3560,37 @@ var _ = SIGDescribe("Rollout", func() { It("patch batch id to pods: rollback case", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1alpha1", Kind: "CloneSet", Name: "echoserver", } rollout.Spec.Strategy.Canary.TrafficRoutings = nil rollout.Annotations = map[string]string{ - util.RollbackInBatchAnnotation: "true", + v1alpha1.RollbackInBatchAnnotation: "true", } - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(40), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(80), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(100), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, }, @@ -3600,7 +3600,7 @@ var _ = SIGDescribe("Rollout", func() { By("Creating workload and waiting for all pods ready...") workload := &appsv1alpha1.CloneSet{} Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) - workload.Labels[util.RolloutIDLabel] = "1" + workload.Labels[v1alpha1.RolloutIDLabel] = "1" CreateObject(workload) WaitCloneSetAllPodsReady(workload) @@ -3629,7 +3629,7 @@ var _ = SIGDescribe("Rollout", func() { By("Update cloneSet env NODE_NAME from(version2) -> to(version1)") Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) - workload.Labels[util.RolloutIDLabel] = "2" + workload.Labels[v1alpha1.RolloutIDLabel] = "2" workload.Spec.Template.Spec.Containers[0].Env = newEnvs UpdateCloneSet(workload) time.Sleep(10 * time.Second) @@ -3656,7 +3656,7 @@ var _ = SIGDescribe("Rollout", func() { By("Wait rollout complete") ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) @@ -3666,37 +3666,37 @@ var _ = SIGDescribe("Rollout", func() { It("patch batch id to pods: only rollout-id changes", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1alpha1", Kind: "CloneSet", Name: "echoserver", } rollout.Spec.Strategy.Canary.TrafficRoutings = nil rollout.Annotations = map[string]string{ - util.RollbackInBatchAnnotation: "true", + v1alpha1.RollbackInBatchAnnotation: "true", } - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(40), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(80), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(100), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, }, @@ -3710,7 +3710,7 @@ var _ = SIGDescribe("Rollout", func() { WaitCloneSetAllPodsReady(workload) By("Only update rollout id = '1', and start rollout") - workload.Labels[util.RolloutIDLabel] = "1" + workload.Labels[v1alpha1.RolloutIDLabel] = "1" workload.Annotations[util.InRolloutProgressingAnnotation] = "true" UpdateCloneSet(workload) @@ -3729,7 +3729,7 @@ var _ = SIGDescribe("Rollout", func() { CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "3", 1) By("Only update rollout id = '2', and check batch label again") - workload.Labels[util.RolloutIDLabel] = "2" + workload.Labels[v1alpha1.RolloutIDLabel] = "2" UpdateCloneSet(workload) By("wait step(3) pause again") @@ -3746,7 +3746,7 @@ var _ = SIGDescribe("Rollout", func() { By("Wait rollout complete") ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1) @@ -3756,37 +3756,37 @@ var _ = SIGDescribe("Rollout", func() { It("patch batch id to pods: only change rollout-id after rolling the first step", func() { By("Creating Rollout...") - rollout := &rolloutsv1alpha1.Rollout{} + rollout := &v1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) - rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + rollout.Spec.ObjectRef.WorkloadRef = &v1alpha1.WorkloadRef{ APIVersion: "apps.kruise.io/v1alpha1", Kind: "CloneSet", Name: "echoserver", } rollout.Spec.Strategy.Canary.TrafficRoutings = nil rollout.Annotations = map[string]string{ - util.RollbackInBatchAnnotation: "true", + v1alpha1.RollbackInBatchAnnotation: "true", } - rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ { Weight: utilpointer.Int32(20), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(40), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(60), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(80), - Pause: rolloutsv1alpha1.RolloutPause{}, + Pause: v1alpha1.RolloutPause{}, }, { Weight: utilpointer.Int32(100), - Pause: rolloutsv1alpha1.RolloutPause{ + Pause: v1alpha1.RolloutPause{ Duration: utilpointer.Int32(0), }, }, @@ -3800,7 +3800,7 @@ var _ = SIGDescribe("Rollout", func() { WaitCloneSetAllPodsReady(workload) By("Only update rollout id = '1', and start rollout") - workload.Labels[util.RolloutIDLabel] = "1" + workload.Labels[v1alpha1.RolloutIDLabel] = "1" workload.Annotations[util.InRolloutProgressingAnnotation] = "true" UpdateCloneSet(workload) @@ -3809,7 +3809,7 @@ var _ = SIGDescribe("Rollout", func() { CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "1", "1", 1) By("Only update rollout id = '2', and check batch label again") - workload.Labels[util.RolloutIDLabel] = "2" + workload.Labels[v1alpha1.RolloutIDLabel] = "2" UpdateCloneSet(workload) By("wait 30s") @@ -3836,7 +3836,7 @@ var _ = SIGDescribe("Rollout", func() { By("Wait rollout complete") ResumeRolloutCanary(rollout.Name) - WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy) CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "1", 1) CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "2", 1) CheckPodBatchLabel(workload.Namespace, workload.Spec.Selector, "2", "3", 1)