diff --git a/.github/workflows/e2e-v1beta1-1.19.yaml b/.github/workflows/e2e-v1beta1-1.19.yaml new file mode 100644 index 00000000..6ec0e39d --- /dev/null +++ b/.github/workflows/e2e-v1beta1-1.19.yaml @@ -0,0 +1,110 @@ +name: E2E-V1Beta1-1.19 + +on: + push: + branches: + - master + - release-* + pull_request: {} + workflow_dispatch: {} + +env: + # Common versions + GO_VERSION: '1.19' + KIND_IMAGE: 'kindest/node:v1.19.16' + KIND_CLUSTER_NAME: 'ci-testing' + +jobs: + + rollout: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + node_image: ${{ env.KIND_IMAGE }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + config: ./test/kind-conf.yaml + - name: Build image + run: | + export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}" + docker build --pull --no-cache . -t $IMAGE + kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; } + - name: Install Kruise + run: | + set -ex + kubectl cluster-info + make helm + helm repo add openkruise https://openkruise.github.io/charts/ + helm repo update + helm install kruise openkruise/kruise + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + echo "Wait for kruise-manager ready successfully" + else + echo "Timeout to wait for kruise-manager ready" + exit 1 + fi + - name: Install Kruise Rollout + run: | + set -ex + kubectl cluster-info + IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + set -e + if [ "$PODS" -eq "1" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + kubectl get node -o yaml + kubectl get all -n kruise-rollout -o yaml + set -e + if [ "$PODS" -eq "1" ]; then + echo "Wait for kruise-rollout ready successfully" + else + echo "Timeout to wait for kruise-rollout ready" + exit 1 + fi + - name: Run E2E Tests + run: | + export KUBECONFIG=/home/runner/.kube/config + make ginkgo + set +e + ./bin/ginkgo -timeout 60m -v --focus='Step Jump' test/e2e + retVal=$? + # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout + restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') + if [ "${restartCount}" -eq "0" ];then + echo "Kruise-rollout has not restarted" + else + kubectl get pod -n kruise-rollout --no-headers + echo "Kruise-rollout has restarted, abort!!!" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + exit 1 + fi + exit $retVal diff --git a/.github/workflows/e2e-v1beta1-1.23.yaml b/.github/workflows/e2e-v1beta1-1.23.yaml new file mode 100644 index 00000000..8dad2a8b --- /dev/null +++ b/.github/workflows/e2e-v1beta1-1.23.yaml @@ -0,0 +1,110 @@ +name: E2E-V1Beta1-1.23 + +on: + push: + branches: + - master + - release-* + pull_request: {} + workflow_dispatch: {} + +env: + # Common versions + GO_VERSION: '1.19' + KIND_IMAGE: 'kindest/node:v1.23.3' + KIND_CLUSTER_NAME: 'ci-testing' + +jobs: + + rollout: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + node_image: ${{ env.KIND_IMAGE }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + config: ./test/kind-conf.yaml + - name: Build image + run: | + export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}" + docker build --pull --no-cache . -t $IMAGE + kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; } + - name: Install Kruise + run: | + set -ex + kubectl cluster-info + make helm + helm repo add openkruise https://openkruise.github.io/charts/ + helm repo update + helm install kruise openkruise/kruise + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + echo "Wait for kruise-manager ready successfully" + else + echo "Timeout to wait for kruise-manager ready" + exit 1 + fi + - name: Install Kruise Rollout + run: | + set -ex + kubectl cluster-info + IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + set -e + if [ "$PODS" -eq "1" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + kubectl get node -o yaml + kubectl get all -n kruise-rollout -o yaml + set -e + if [ "$PODS" -eq "1" ]; then + echo "Wait for kruise-rollout ready successfully" + else + echo "Timeout to wait for kruise-rollout ready" + exit 1 + fi + - name: Run E2E Tests + run: | + export KUBECONFIG=/home/runner/.kube/config + make ginkgo + set +e + ./bin/ginkgo -timeout 60m -v --focus='Step Jump' test/e2e + retVal=$? + # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout + restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') + if [ "${restartCount}" -eq "0" ];then + echo "Kruise-rollout has not restarted" + else + kubectl get pod -n kruise-rollout --no-headers + echo "Kruise-rollout has restarted, abort!!!" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + exit 1 + fi + exit $retVal diff --git a/api/v1beta1/rollout_types.go b/api/v1beta1/rollout_types.go index 36e38bf6..9f074597 100644 --- a/api/v1beta1/rollout_types.go +++ b/api/v1beta1/rollout_types.go @@ -102,6 +102,10 @@ func (r *RolloutStrategy) IsCanaryStragegy() bool { return r.GetRollingStyle() == CanaryRollingStyle || r.GetRollingStyle() == PartitionRollingStyle } +func (r *RolloutStrategy) IsEmptyRelease() bool { + return r.BlueGreen == nil && r.Canary == nil +} + // Get the steps based on the rolling style func (r *RolloutStrategy) GetSteps() []CanaryStep { switch r.GetRollingStyle() { diff --git a/config/crd/bases/rollouts.kruise.io_rollouts.yaml b/config/crd/bases/rollouts.kruise.io_rollouts.yaml index aa01c4c1..4c1b8743 100644 --- a/config/crd/bases/rollouts.kruise.io_rollouts.yaml +++ b/config/crd/bases/rollouts.kruise.io_rollouts.yaml @@ -615,13 +615,13 @@ spec: description: CanaryStep defines a step of a canary workload. properties: matches: - description: Matches define conditions used for matching - the incoming HTTP requests to canary service. Each + description: "Matches define conditions used for matching + incoming HTTP requests to the canary service. Each match is independent, i.e. this rule will be matched - if **any** one of the matches is satisfied. If Gateway - API, current only support one match. And cannot support - both weight and matches, if both are configured, then - matches takes precedence. + as long as **any** one of the matches is satisfied. + \n It cannot support Traffic (weight-based routing) + and Matches simultaneously, if both are configured. + In such cases, Matches takes precedence." items: properties: headers: @@ -683,6 +683,88 @@ spec: type: object maxItems: 16 type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + path: + description: 'Path specifies a HTTP request path + matcher. Supported list: - Istio: https://istio.io/latest/docs/reference/config/networking/virtual-service/#HTTPMatchRequest + - GatewayAPI: If path is defined, the whole + HttpRouteMatch will be used as a standalone + matcher' + properties: + type: + default: PathPrefix + description: "Type specifies how to match + against the path Value. \n Support: Core + (Exact, PathPrefix) \n Support: Custom (RegularExpression)" + enum: + - Exact + - PathPrefix + - RegularExpression + type: string + value: + default: / + description: Value of the HTTP path to match + against. + maxLength: 1024 + type: string + type: object + queryParams: + description: 'QueryParams specifies HTTP query + parameter matchers. Multiple match values are + ANDed together, meaning, a request must match + all the specified query parameters to select + the route. Supported list: - Istio: https://istio.io/latest/docs/reference/config/networking/virtual-service/#HTTPMatchRequest + - MSE Ingress: https://help.aliyun.com/zh/ack/ack-managed-and-ack-dedicated/user-guide/annotations-supported-by-mse-ingress-gateways-1 Header/Cookie + > QueryParams - Gateway API' + items: + description: HTTPQueryParamMatch describes how + to select a HTTP route by matching HTTP query + parameters. + properties: + name: + description: "Name is the name of the HTTP + query param to be matched. This must be + an exact string match. (See https://tools.ietf.org/html/rfc7230#section-2.7.3). + \n If multiple entries specify equivalent + query param names, only the first entry + with an equivalent name MUST be considered + for a match. Subsequent entries with an + equivalent query param name MUST be ignored." + maxLength: 256 + minLength: 1 + type: string + type: + default: Exact + description: "Type specifies how to match + against the value of the query parameter. + \n Support: Extended (Exact) \n Support: + Custom (RegularExpression) \n Since RegularExpression + QueryParamMatchType has custom conformance, + implementations can support POSIX, PCRE + or any other dialects of regular expressions. + Please read the implementation's documentation + to determine the supported dialect." + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value of HTTP + query param to be matched. + maxLength: 1024 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object type: array pause: diff --git a/lua_configuration/convert_test_case_to_lua_object.go b/lua_configuration/convert_test_case_to_lua_object.go index 686bea18..d97e959a 100644 --- a/lua_configuration/convert_test_case_to_lua_object.go +++ b/lua_configuration/convert_test_case_to_lua_object.go @@ -81,7 +81,7 @@ func objectToTable(path string) error { rollout := testCase.Rollout trafficRouting := testCase.TrafficRouting if rollout != nil { - steps := rollout.Spec.Strategy.Canary.Steps + steps := rollout.Spec.Strategy.GetSteps() for i, step := range steps { var weight *int32 if step.TrafficRoutingStrategy.Traffic != nil { @@ -92,7 +92,7 @@ func objectToTable(path string) error { weight = utilpointer.Int32(-1) } var canaryService string - stableService := rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service + stableService := rollout.Spec.Strategy.GetTrafficRouting()[0].Service canaryService = fmt.Sprintf("%s-canary", stableService) data := &custom.LuaData{ Data: custom.Data{ diff --git a/pkg/controller/batchrelease/context/context.go b/pkg/controller/batchrelease/context/context.go index 5a801249..2c428180 100644 --- a/pkg/controller/batchrelease/context/context.go +++ b/pkg/controller/batchrelease/context/context.go @@ -73,20 +73,20 @@ func (bc *BatchContext) Log() string { // IsBatchReady return nil if the batch is ready func (bc *BatchContext) IsBatchReady() error { if bc.UpdatedReplicas < bc.DesiredUpdatedReplicas { - return fmt.Errorf("current batch not ready: updated replicas not satified") + return fmt.Errorf("current batch not ready: updated replicas not satisfied, UpdatedReplicas %d < DesiredUpdatedReplicas %d", bc.UpdatedReplicas, bc.DesiredUpdatedReplicas) } unavailableToleration := allowedUnavailable(bc.FailureThreshold, bc.UpdatedReplicas) if unavailableToleration+bc.UpdatedReadyReplicas < bc.DesiredUpdatedReplicas { - return fmt.Errorf("current batch not ready: updated ready replicas not satified") + return fmt.Errorf("current batch not ready: updated ready replicas not satisfied, allowedUnavailable + UpdatedReadyReplicas %d < DesiredUpdatedReplicas %d", unavailableToleration+bc.UpdatedReadyReplicas, bc.DesiredUpdatedReplicas) } if bc.DesiredUpdatedReplicas > 0 && bc.UpdatedReadyReplicas == 0 { - return fmt.Errorf("current batch not ready: no updated ready replicas") + return fmt.Errorf("current batch not ready: no updated ready replicas, DesiredUpdatedReplicas %d > 0 and UpdatedReadyReplicas %d = 0", bc.DesiredUpdatedReplicas, bc.UpdatedReadyReplicas) } if !batchLabelSatisfied(bc.Pods, bc.RolloutID, bc.PlannedUpdatedReplicas) { - return fmt.Errorf("current batch not ready: pods with batch label not satified") + return fmt.Errorf("current batch not ready: pods with batch label not satisfied, RolloutID %s, PlannedUpdatedReplicas %d", bc.RolloutID, bc.PlannedUpdatedReplicas) } return nil } diff --git a/pkg/controller/rollout/rollout_canary.go b/pkg/controller/rollout/rollout_canary.go index 6f623610..c814a750 100644 --- a/pkg/controller/rollout/rollout_canary.go +++ b/pkg/controller/rollout/rollout_canary.go @@ -70,6 +70,11 @@ func (m *canaryReleaseManager) runCanary(c *RolloutContext) error { if canaryStatus.PodTemplateHash == "" { canaryStatus.PodTemplateHash = c.Workload.PodTemplateHash } + + if m.doCanaryJump(c) { + klog.Infof("rollout(%s/%s) canary step jumped", c.Rollout.Namespace, c.Rollout.Name) + return nil + } // When the first batch is trafficRouting rolling and the next steps are rolling release, // We need to clean up the canary-related resources first and then rollout the rest of the batch. currentStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] @@ -87,6 +92,13 @@ func (m *canaryReleaseManager) runCanary(c *RolloutContext) error { } } switch canaryStatus.CurrentStepState { + // before CanaryStepStateUpgrade, handle some special cases, to prevent traffic loss + case v1beta1.CanaryStepStateInit: + // placeholder for the later traffic modification Pull Request + canaryStatus.NextStepIndex = util.NextBatchIndex(c.Rollout, canaryStatus.CurrentStepIndex) + canaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade + fallthrough + case v1beta1.CanaryStepStateUpgrade: klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", c.Rollout.Namespace, c.Rollout.Name, v1beta1.CanaryStepStateUpgrade) done, err := m.doCanaryUpgrade(c) @@ -144,7 +156,8 @@ func (m *canaryReleaseManager) runCanary(c *RolloutContext) error { if len(c.Rollout.Spec.Strategy.Canary.Steps) > int(canaryStatus.CurrentStepIndex) { canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} canaryStatus.CurrentStepIndex++ - canaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade + canaryStatus.NextStepIndex = util.NextBatchIndex(c.Rollout, canaryStatus.CurrentStepIndex) + canaryStatus.CurrentStepState = v1beta1.CanaryStepStateInit klog.Infof("rollout(%s/%s) canary step from(%d) -> to(%d)", c.Rollout.Namespace, c.Rollout.Name, canaryStatus.CurrentStepIndex-1, canaryStatus.CurrentStepIndex) } else { klog.Infof("rollout(%s/%s) canary run all steps, and completed", c.Rollout.Namespace, c.Rollout.Name) @@ -204,12 +217,6 @@ func (m *canaryReleaseManager) doCanaryPaused(c *RolloutContext) (bool, error) { canaryStatus := c.NewStatus.CanaryStatus currentStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] steps := len(c.Rollout.Spec.Strategy.Canary.Steps) - // If it is the last step, and 100% of pods, then return true - if int32(steps) == canaryStatus.CurrentStepIndex { - if currentStep.Replicas != nil && currentStep.Replicas.StrVal == "100%" { - return true, nil - } - } cond := util.GetRolloutCondition(*c.NewStatus, v1beta1.RolloutConditionProgressing) // need manual confirmation if currentStep.Pause.Duration == nil { @@ -232,6 +239,46 @@ func (m *canaryReleaseManager) doCanaryPaused(c *RolloutContext) (bool, error) { return false, nil } +func (m *canaryReleaseManager) doCanaryJump(c *RolloutContext) (jumped bool) { + canaryStatus := c.NewStatus.CanaryStatus + nextIndex := canaryStatus.NextStepIndex + /* + we set the CurrentStepIndex same as NextStepIndex to prevent currentStepIndex from out of range + for example, if we had a rollout with 4 steps and CurrentStepIndex was 2 + then, the user removed 3 steps from the plan, we can calculate NextStepIndex is 1 correctly, + but CurrentStepIndex remains 2, which could cause out of range. + */ + resetCurrentIndex := false + if int(canaryStatus.CurrentStepIndex) > len(c.Rollout.Spec.Strategy.Canary.Steps) { + canaryStatus.CurrentStepIndex = nextIndex + resetCurrentIndex = true + } + currentStep := c.Rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] + if resetCurrentIndex || nextIndex != util.NextBatchIndex(c.Rollout, canaryStatus.CurrentStepIndex) && nextIndex > 0 { + currentIndexBackup := canaryStatus.CurrentStepIndex + currentStepStateBackup := canaryStatus.CurrentStepState + canaryStatus.CurrentStepIndex = nextIndex + canaryStatus.NextStepIndex = util.NextBatchIndex(c.Rollout, nextIndex) + nextStep := c.Rollout.Spec.Strategy.Canary.Steps[nextIndex-1] + // if the Replicas between currentStep and nextStep is same, we can jump to + // the TrafficRouting step; otherwise, we should start from the Init step + if reflect.DeepEqual(nextStep.Replicas, currentStep.Replicas) && !resetCurrentIndex { + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + canaryStatus.CurrentStepState = v1beta1.CanaryStepStateTrafficRouting + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", c.Rollout.Namespace, c.Rollout.Name, + canaryStatus.CurrentStepIndex, currentStepStateBackup, canaryStatus.CurrentStepState) + } else { + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + canaryStatus.CurrentStepState = v1beta1.CanaryStepStateInit + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", c.Rollout.Namespace, c.Rollout.Name, + canaryStatus.CurrentStepIndex, currentStepStateBackup, v1beta1.CanaryStepStateInit) + } + klog.Infof("rollout(%s/%s) canary step from(%d) -> to(%d)", c.Rollout.Namespace, c.Rollout.Name, currentIndexBackup, canaryStatus.CurrentStepIndex) + return true + } + return false +} + // cleanup after rollout is completed or finished func (m *canaryReleaseManager) doCanaryFinalising(c *RolloutContext) (bool, error) { // when CanaryStatus is nil, which means canary action hasn't started yet, don't need doing cleanup diff --git a/pkg/controller/rollout/rollout_progressing.go b/pkg/controller/rollout/rollout_progressing.go index 2388eb46..24f9943b 100644 --- a/pkg/controller/rollout/rollout_progressing.go +++ b/pkg/controller/rollout/rollout_progressing.go @@ -84,7 +84,6 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *v1beta1.Rollout UpdatedRevision: rolloutContext.Workload.CanaryRevision, } } else { - commonStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade newStatus.CanaryStatus = &v1beta1.CanaryStatus{ CommonStatus: commonStatus, CanaryRevision: rolloutContext.Workload.CanaryRevision, @@ -164,7 +163,7 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *v1beta1.Rollout func (r *RolloutReconciler) doProgressingInitializing(c *RolloutContext) (bool, error) { // Traffic routing - if len(c.Rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { + if c.Rollout.Spec.Strategy.HasTrafficRoutings() { if err := r.trafficRoutingManager.InitializeTrafficRouting(newTrafficRoutingContext(c)); err != nil { return false, err } @@ -223,14 +222,15 @@ func (r *RolloutReconciler) handleRolloutPaused(rollout *v1beta1.Rollout, newSta func (r *RolloutReconciler) handleContinuousRelease(c *RolloutContext) error { r.Recorder.Eventf(c.Rollout, corev1.EventTypeNormal, "Progressing", "workload continuous publishing canaryRevision, then restart publishing") klog.Infof("rollout(%s/%s) workload continuous publishing canaryRevision from(%s) -> to(%s), then restart publishing", - c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.CanaryStatus.CanaryRevision, c.Workload.CanaryRevision) + c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.GetCanaryRevision(), c.Workload.CanaryRevision) done, err := r.doProgressingReset(c) if err != nil { klog.Errorf("rollout(%s/%s) doProgressingReset failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) return err } else if done { - c.NewStatus.CanaryStatus = nil + // clear SubStatus + c.NewStatus.Clear() progressingStateTransition(c.NewStatus, corev1.ConditionTrue, v1alpha1.ProgressingReasonInitializing, "Workload is continuous release") klog.Infof("rollout(%s/%s) workload is continuous publishing, reset complete", c.Rollout.Namespace, c.Rollout.Name) } else { @@ -243,7 +243,7 @@ func (r *RolloutReconciler) handleContinuousRelease(c *RolloutContext) error { } func (r *RolloutReconciler) handleRollbackDirectly(rollout *v1beta1.Rollout, workload *util.Workload, newStatus *v1beta1.RolloutStatus) error { - newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision + newStatus.SetCanaryRevision(workload.CanaryRevision) r.Recorder.Eventf(rollout, corev1.EventTypeNormal, "Progressing", "workload has been rollback, then rollout is canceled") klog.Infof("rollout(%s/%s) workload has been rollback directly, then rollout canceled", rollout.Namespace, rollout.Name) progressingStateTransition(newStatus, corev1.ConditionTrue, v1alpha1.ProgressingReasonCancelling, "The workload has been rolled back and the rollout process will be cancelled") @@ -252,11 +252,12 @@ func (r *RolloutReconciler) handleRollbackDirectly(rollout *v1beta1.Rollout, wor func (r *RolloutReconciler) handleRollbackInBatches(rollout *v1beta1.Rollout, workload *util.Workload, newStatus *v1beta1.RolloutStatus) error { // restart from the beginning - newStatus.CanaryStatus.CurrentStepIndex = 1 - newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision - newStatus.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade - newStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - newStatus.CanaryStatus.RolloutHash = rollout.Annotations[util.RolloutHashAnnotation] + newStatus.GetSubStatus().CurrentStepIndex = 1 + newStatus.GetSubStatus().NextStepIndex = util.NextBatchIndex(rollout, 1) + newStatus.SetCanaryRevision(workload.CanaryRevision) + newStatus.GetSubStatus().CurrentStepState = v1beta1.CanaryStepStateInit + newStatus.GetSubStatus().LastUpdateTime = &metav1.Time{Time: time.Now()} + newStatus.GetSubStatus().RolloutHash = rollout.Annotations[util.RolloutHashAnnotation] klog.Infof("rollout(%s/%s) workload has been rollback in batches, then restart from beginning", rollout.Namespace, rollout.Name) return nil } @@ -267,24 +268,46 @@ func (r *RolloutReconciler) handleRolloutPlanChanged(c *RolloutContext) error { klog.Errorf("rollout(%s/%s) reCalculate Canary StepIndex failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) return err } - // canary step configuration change causes current step index change - c.NewStatus.CanaryStatus.CurrentStepIndex = newStepIndex - c.NewStatus.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade - c.NewStatus.CanaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - c.NewStatus.CanaryStatus.RolloutHash = c.Rollout.Annotations[util.RolloutHashAnnotation] - klog.Infof("rollout(%s/%s) canary step configuration change, and stepIndex(%d) state(%s)", - c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.CanaryStatus.CurrentStepIndex, c.NewStatus.CanaryStatus.CurrentStepState) + // if the target step index is the same as the NextStepIndex + // we simply set the CurrentStepState to Ready + if c.NewStatus.GetSubStatus().NextStepIndex == newStepIndex { + c.NewStatus.GetSubStatus().CurrentStepState = v1beta1.CanaryStepStateReady + c.NewStatus.GetSubStatus().LastUpdateTime = &metav1.Time{Time: time.Now()} + c.NewStatus.GetSubStatus().RolloutHash = c.Rollout.Annotations[util.RolloutHashAnnotation] + klog.Infof("rollout(%s/%s) canary step configuration change, and NextStepIndex(%d) state(%s)", + c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.GetSubStatus().NextStepIndex, c.NewStatus.GetSubStatus().CurrentStepState) + return nil + } + + // otherwise, execute the "jump" logic + c.NewStatus.GetSubStatus().NextStepIndex = newStepIndex + c.NewStatus.GetSubStatus().LastUpdateTime = &metav1.Time{Time: time.Now()} + c.NewStatus.GetSubStatus().RolloutHash = c.Rollout.Annotations[util.RolloutHashAnnotation] + releaseManager, err := r.getReleaseManager(c.Rollout) + if err != nil { + return err + } + releaseManager.doCanaryJump(c) + klog.Infof("rollout(%s/%s) canary step configuration change, and NextStepIndex(%d) state(%s)", + c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.GetSubStatus().NextStepIndex, c.NewStatus.GetSubStatus().CurrentStepState) return nil } func (r *RolloutReconciler) handleNormalRolling(c *RolloutContext) error { // check if canary is done - if c.NewStatus.CanaryStatus.CurrentStepState == v1beta1.CanaryStepStateCompleted { + if c.NewStatus.GetSubStatus().CurrentStepState == v1beta1.CanaryStepStateCompleted { klog.Infof("rollout(%s/%s) progressing rolling done", c.Rollout.Namespace, c.Rollout.Name) progressingStateTransition(c.NewStatus, corev1.ConditionTrue, v1alpha1.ProgressingReasonFinalising, "Rollout has been completed and some closing work is being done") return nil } - return r.canaryManager.runCanary(c) + // in case user modifies it with inappropriate value + util.CheckNextBatchIndexWithCorrect(c.Rollout) + + releaseManager, err := r.getReleaseManager(c.Rollout) + if err != nil { + return err + } + return releaseManager.runCanary(c) } // name is rollout name, tr is trafficRouting name @@ -344,30 +367,41 @@ func (r *RolloutReconciler) finalizeTrafficRouting(namespace, name, tr string) e *********************************************************************** */ + +func (r *RolloutReconciler) getReleaseManager(rollout *v1beta1.Rollout) (ReleaseManager, error) { + if rollout.Spec.Strategy.IsCanaryStragegy() { + return r.canaryManager, nil + } else if rollout.Spec.Strategy.IsBlueGreenRelease() { + // placeholder for upcoming PR + // return r.blueGreenManager, nil + } + return nil, fmt.Errorf("unknown rolling style: %s, and thus cannot call corresponding release manager", rollout.Spec.Strategy.GetRollingStyle()) +} + func isRolloutPaused(rollout *v1beta1.Rollout) bool { return rollout.Spec.Strategy.Paused } func isRolloutPlanChanged(rollout *v1beta1.Rollout) bool { status := &rollout.Status - return status.CanaryStatus.RolloutHash != "" && status.CanaryStatus.RolloutHash != rollout.Annotations[util.RolloutHashAnnotation] + return status.GetSubStatus().RolloutHash != "" && status.GetSubStatus().RolloutHash != rollout.Annotations[util.RolloutHashAnnotation] } func isContinuousRelease(rollout *v1beta1.Rollout, workload *util.Workload) bool { status := &rollout.Status - return status.CanaryStatus.CanaryRevision != "" && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && !workload.IsInRollback + return status.GetCanaryRevision() != "" && workload.CanaryRevision != status.GetCanaryRevision() && !workload.IsInRollback } func isRollingBackDirectly(rollout *v1beta1.Rollout, workload *util.Workload) bool { status := &rollout.Status inBatch := util.IsRollbackInBatchPolicy(rollout, workload.Labels) - return workload.IsInRollback && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && !inBatch + return workload.IsInRollback && workload.CanaryRevision != status.GetCanaryRevision() && !inBatch } func isRollingBackInBatches(rollout *v1beta1.Rollout, workload *util.Workload) bool { status := &rollout.Status inBatch := util.IsRollbackInBatchPolicy(rollout, workload.Labels) - return workload.IsInRollback && workload.CanaryRevision != status.CanaryStatus.CanaryRevision && inBatch + return workload.IsInRollback && workload.CanaryRevision != status.GetCanaryRevision() && inBatch } // 1. modify network api(ingress or gateway api) configuration, and route 100% traffic to stable pods @@ -393,16 +427,35 @@ func (r *RolloutReconciler) doProgressingReset(c *RolloutContext) (bool, error) } func (r *RolloutReconciler) recalculateCanaryStep(c *RolloutContext) (int32, error) { - batch, err := r.canaryManager.fetchBatchRelease(c.Rollout.Namespace, c.Rollout.Name) + releaseManager, err := r.getReleaseManager(c.Rollout) + if err != nil { + return 0, err + } + batch, err := releaseManager.fetchBatchRelease(c.Rollout.Namespace, c.Rollout.Name) if errors.IsNotFound(err) { return 1, nil } else if err != nil { return 0, err } currentReplicas, _ := intstr.GetScaledValueFromIntOrPercent(&batch.Spec.ReleasePlan.Batches[*batch.Spec.ReleasePlan.BatchPartition].CanaryReplicas, int(c.Workload.Replicas), true) - var stepIndex int32 - for i := range c.Rollout.Spec.Strategy.Canary.Steps { - step := c.Rollout.Spec.Strategy.Canary.Steps[i] + var stepIndex, currentIndex int32 + if c.NewStatus != nil { + currentIndex = c.NewStatus.GetSubStatus().CurrentStepIndex - 1 + } + steps := append([]int{}, int(currentIndex)) + // we don't distinguish between the changes in Replicas and Traffic + // Whatever the change is, we recalculate the step. + // we put the current step index first for retrieval, so that if Traffic is the only change, + // usually we will get the target step index same as current step index + for i := 0; i < len(c.Rollout.Spec.Strategy.GetSteps()); i++ { + if i == int(currentIndex) { + continue + } + steps = append(steps, i) + } + + for _, i := range steps { + step := c.Rollout.Spec.Strategy.GetSteps()[i] var desiredReplicas int desiredReplicas, _ = intstr.GetScaledValueFromIntOrPercent(step.Replicas, int(c.Workload.Replicas), true) stepIndex = int32(i + 1) @@ -410,6 +463,7 @@ func (r *RolloutReconciler) recalculateCanaryStep(c *RolloutContext) (int32, err break } } + klog.Infof("RolloutPlan Change detected, rollout(%s/%s) currentStepIndex %d, jumps to %d", c.Rollout.Namespace, c.Rollout.Name, currentIndex+1, stepIndex) return stepIndex, nil } @@ -422,7 +476,11 @@ func (r *RolloutReconciler) doFinalising(c *RolloutContext) (bool, error) { return false, err } } - done, err := r.canaryManager.doCanaryFinalising(c) + releaseManager, err := r.getReleaseManager(c.Rollout) + if err != nil { + return false, err + } + done, err := releaseManager.doCanaryFinalising(c) if err != nil { klog.Errorf("rollout(%s/%s) Progressing failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error()) return false, err @@ -460,7 +518,16 @@ func setRolloutSucceededCondition(status *v1beta1.RolloutStatus, condStatus core } func newTrafficRoutingContext(c *RolloutContext) *trafficrouting.TrafficRoutingContext { - currentStep := c.Rollout.Spec.Strategy.Canary.Steps[c.NewStatus.CanaryStatus.CurrentStepIndex-1] + currentIndex := c.NewStatus.GetSubStatus().CurrentStepIndex - 1 + var currentStep v1beta1.CanaryStep + if currentIndex < 0 || int(currentIndex) >= len(c.Rollout.Spec.Strategy.GetSteps()) { + klog.Warningf("Rollout(%s/%s) encounters a special case when constructing newTrafficRoutingContext", c.Rollout.Namespace, c.Rollout.Name) + // usually this only happens when deleting the rollout or rolling back (ie. Finalising) + // in these scenarios, it's not important which step the current is + currentStep = c.Rollout.Spec.Strategy.GetSteps()[0] + } else { + currentStep = c.Rollout.Spec.Strategy.GetSteps()[currentIndex] + } var revisionLabelKey string if c.Workload != nil { revisionLabelKey = c.Workload.RevisionLabelKey @@ -468,13 +535,13 @@ func newTrafficRoutingContext(c *RolloutContext) *trafficrouting.TrafficRoutingC return &trafficrouting.TrafficRoutingContext{ Key: fmt.Sprintf("Rollout(%s/%s)", c.Rollout.Namespace, c.Rollout.Name), Namespace: c.Rollout.Namespace, - ObjectRef: c.Rollout.Spec.Strategy.Canary.TrafficRoutings, + ObjectRef: c.Rollout.Spec.Strategy.GetTrafficRouting(), Strategy: currentStep.TrafficRoutingStrategy, OwnerRef: *metav1.NewControllerRef(c.Rollout, rolloutControllerKind), RevisionLabelKey: revisionLabelKey, - StableRevision: c.NewStatus.CanaryStatus.StableRevision, - CanaryRevision: c.NewStatus.CanaryStatus.PodTemplateHash, - LastUpdateTime: c.NewStatus.CanaryStatus.LastUpdateTime, - DisableGenerateCanaryService: c.Rollout.Spec.Strategy.Canary.DisableGenerateCanaryService, + StableRevision: c.NewStatus.GetSubStatus().StableRevision, + CanaryRevision: c.NewStatus.GetSubStatus().PodTemplateHash, + LastUpdateTime: c.NewStatus.GetSubStatus().LastUpdateTime, + DisableGenerateCanaryService: c.Rollout.Spec.Strategy.DisableGenerateCanaryService(), } } diff --git a/pkg/controller/rollout/rollout_progressing_test.go b/pkg/controller/rollout/rollout_progressing_test.go index 42412b3e..6513faff 100644 --- a/pkg/controller/rollout/rollout_progressing_test.go +++ b/pkg/controller/rollout/rollout_progressing_test.go @@ -67,8 +67,11 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.StableRevision = "pod-template-hash-v1" s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.CurrentStepIndex = 1 + // s.CanaryStatus.NextStepIndex will be initialized as 0 in ReconcileRolloutProgressing. + // util.NextBatchIndex(rollout, s.CanaryStatus.CurrentStepIndex), which is 2 here. s.CanaryStatus.NextStepIndex = 2 - s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade + // now the first step is no longer StepStateUpgrade, it is StepStateInit now + s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateInit return s }, expectTr: func() *v1alpha1.TrafficRouting { @@ -101,7 +104,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.CurrentStepIndex = 1 s.CanaryStatus.NextStepIndex = 2 - s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade + s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateInit cond := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling util.SetRolloutCondition(s, *cond) @@ -142,6 +145,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { obj.Status.CanaryStatus.StableRevision = "pod-template-hash-v1" obj.Status.CanaryStatus.CanaryRevision = "6f8cc56547" obj.Status.CanaryStatus.CurrentStepIndex = 1 + obj.Status.CanaryStatus.NextStepIndex = 2 obj.Status.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade cond := util.GetRolloutCondition(obj.Status, v1beta1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling @@ -156,6 +160,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" s.CanaryStatus.CurrentStepIndex = 1 + s.CanaryStatus.NextStepIndex = 2 s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateUpgrade cond := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonInRolling @@ -212,6 +217,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.NextStepIndex = 0 s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateCompleted cond := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonFinalising @@ -270,6 +276,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.NextStepIndex = 0 s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateCompleted cond := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond.Reason = v1alpha1.ProgressingReasonFinalising @@ -330,6 +337,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.NextStepIndex = 0 s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateCompleted cond2 := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond2.Reason = v1alpha1.ProgressingReasonFinalising @@ -379,6 +387,7 @@ func TestReconcileRolloutProgressing(t *testing.T) { s.CanaryStatus.CanaryRevision = "6f8cc56547" s.CanaryStatus.PodTemplateHash = "pod-template-hash-v2" s.CanaryStatus.CurrentStepIndex = 4 + s.CanaryStatus.NextStepIndex = 0 s.CanaryStatus.CurrentStepState = v1beta1.CanaryStepStateCompleted cond2 := util.GetRolloutCondition(*s, v1beta1.RolloutConditionProgressing) cond2.Reason = v1alpha1.ProgressingReasonCompleted diff --git a/pkg/controller/rollout/rollout_releaseManager.go b/pkg/controller/rollout/rollout_releaseManager.go new file mode 100644 index 00000000..01772580 --- /dev/null +++ b/pkg/controller/rollout/rollout_releaseManager.go @@ -0,0 +1,13 @@ +package rollout + +import ( + "github.com/openkruise/rollouts/api/v1beta1" +) + +type ReleaseManager interface { + runCanary(c *RolloutContext) error + doCanaryJump(c *RolloutContext) bool + doCanaryFinalising(c *RolloutContext) (bool, error) + fetchBatchRelease(ns, name string) (*v1beta1.BatchRelease, error) + removeBatchRelease(c *RolloutContext) (bool, error) +} diff --git a/pkg/controller/rollout/rollout_status.go b/pkg/controller/rollout/rollout_status.go index 0e33522e..dbd0f103 100755 --- a/pkg/controller/rollout/rollout_status.go +++ b/pkg/controller/rollout/rollout_status.go @@ -91,10 +91,10 @@ func (r *RolloutReconciler) calculateRolloutStatus(rollout *v1beta1.Rollout) (re // update workload generation to canaryStatus.ObservedWorkloadGeneration // rollout is a target ref bypass, so there needs to be a field to identify the rollout execution process or results, // which version of deployment is targeted, ObservedWorkloadGeneration that is to compare with the workload generation - if newStatus.CanaryStatus != nil && newStatus.CanaryStatus.CanaryRevision != "" && - newStatus.CanaryStatus.CanaryRevision == workload.CanaryRevision { - newStatus.CanaryStatus.ObservedRolloutID = getRolloutID(workload) - newStatus.CanaryStatus.ObservedWorkloadGeneration = workload.Generation + if !newStatus.IsSubStatusEmpty() && newStatus.GetCanaryRevision() != "" && + newStatus.GetCanaryRevision() == workload.CanaryRevision { + newStatus.GetSubStatus().ObservedRolloutID = getRolloutID(workload) + newStatus.GetSubStatus().ObservedWorkloadGeneration = workload.Generation } switch newStatus.Phase { @@ -110,7 +110,7 @@ func (r *RolloutReconciler) calculateRolloutStatus(rollout *v1beta1.Rollout) (re cond := util.NewRolloutCondition(v1beta1.RolloutConditionProgressing, corev1.ConditionTrue, v1alpha1.ProgressingReasonInitializing, "Rollout is in Progressing") util.SetRolloutCondition(newStatus, *cond) util.RemoveRolloutCondition(newStatus, v1beta1.RolloutConditionSucceeded) - } else if newStatus.CanaryStatus == nil { + } else if newStatus.IsSubStatusEmpty() { // The following logic is to make PaaS be able to judge whether the rollout is ready // at the first deployment of the Rollout/Workload. For example: generally, a PaaS // platform can use the following code to judge whether the rollout progression is completed: @@ -159,15 +159,30 @@ func (r *RolloutReconciler) calculateRolloutStatus(rollout *v1beta1.Rollout) (re // rolloutHash mainly records the step batch information, when the user step changes, // the current batch can be recalculated func (r *RolloutReconciler) calculateRolloutHash(rollout *v1beta1.Rollout) error { - canary := rollout.Spec.Strategy.Canary.DeepCopy() - canary.FailureThreshold = nil - canary.Steps = nil - for i := range rollout.Spec.Strategy.Canary.Steps { - step := rollout.Spec.Strategy.Canary.Steps[i].DeepCopy() - step.Pause = v1beta1.RolloutPause{} - canary.Steps = append(canary.Steps, *step) + var data string + if rollout.Spec.Strategy.IsCanaryStragegy() { + canary := rollout.Spec.Strategy.Canary.DeepCopy() + canary.FailureThreshold = nil + canary.Steps = nil + for i := range rollout.Spec.Strategy.Canary.Steps { + step := rollout.Spec.Strategy.Canary.Steps[i].DeepCopy() + step.Pause = v1beta1.RolloutPause{} + canary.Steps = append(canary.Steps, *step) + } + data = util.DumpJSON(canary) + } else if rollout.Spec.Strategy.IsBlueGreenRelease() { + blueGreen := rollout.Spec.Strategy.BlueGreen.DeepCopy() + blueGreen.FailureThreshold = nil + blueGreen.Steps = nil + for i := range rollout.Spec.Strategy.BlueGreen.Steps { + step := rollout.Spec.Strategy.BlueGreen.Steps[i].DeepCopy() + step.Pause = v1beta1.RolloutPause{} + blueGreen.Steps = append(blueGreen.Steps, *step) + } + data = util.DumpJSON(blueGreen) + } else { + return fmt.Errorf("unknown rolling style: %s", rollout.Spec.Strategy.GetRollingStyle()) } - data := util.DumpJSON(canary) hash := rand.SafeEncodeString(util.EncodeHash(data)) if rollout.Annotations[util.RolloutHashAnnotation] == hash { return nil diff --git a/pkg/util/constant.go b/pkg/util/constant.go index 02ff7226..abfe2d98 100644 --- a/pkg/util/constant.go +++ b/pkg/util/constant.go @@ -27,7 +27,8 @@ const ( // BatchReleaseControlAnnotation is controller info about batchRelease when rollout BatchReleaseControlAnnotation = "batchrelease.rollouts.kruise.io/control-info" // InRolloutProgressingAnnotation marks workload as entering the rollout progressing process - //and does not allow paused=false during this process + // and does not allow paused=false during this process. However, blueGreen is an exception, + // which allows paused=false during progressing. InRolloutProgressingAnnotation = "rollouts.kruise.io/in-progressing" // RolloutHashAnnotation record observed rollout spec hash RolloutHashAnnotation = "rollouts.kruise.io/hash" diff --git a/pkg/util/controller_finder.go b/pkg/util/controller_finder.go index dc15ba1b..886b8191 100644 --- a/pkg/util/controller_finder.go +++ b/pkg/util/controller_finder.go @@ -87,7 +87,7 @@ func NewControllerFinder(c client.Client) *ControllerFinder { func (r *ControllerFinder) GetWorkloadForRef(rollout *rolloutv1beta1.Rollout) (*Workload, error) { workloadRef := rollout.Spec.WorkloadRef - if rollout.Spec.Strategy.Canary.EnableExtraWorkloadForCanary { + if rollout.Spec.Strategy.GetRollingStyle() == rolloutv1beta1.CanaryRollingStyle { for _, finder := range append(r.canaryStyleFinders(), r.partitionStyleFinders()...) { workload, err := finder(rollout.Namespace, &workloadRef) if workload != nil || err != nil { diff --git a/pkg/util/rollout_utils.go b/pkg/util/rollout_utils.go index 927be3a1..33a2b0d9 100644 --- a/pkg/util/rollout_utils.go +++ b/pkg/util/rollout_utils.go @@ -47,7 +47,7 @@ type RolloutState struct { func IsRollbackInBatchPolicy(rollout *rolloutv1beta1.Rollout, labels map[string]string) bool { // currently, only support the case of no traffic routing - if len(rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { + if rollout.Spec.Strategy.HasTrafficRoutings() { return false } workloadRef := rollout.Spec.WorkloadRef @@ -176,3 +176,17 @@ func NextBatchIndex(rollout *rolloutv1beta1.Rollout, CurrentStepIndex int32) int } return CurrentStepIndex + 1 } + +// check if NextStepIndex is legal, if not, correct it +func CheckNextBatchIndexWithCorrect(rollout *rolloutv1beta1.Rollout) { + if rollout == nil { + return + } + nextStep := rollout.Status.GetSubStatus().NextStepIndex + if nextStep <= 0 || nextStep > int32(len(rollout.Spec.Strategy.GetSteps())) { + rollout.Status.GetSubStatus().NextStepIndex = NextBatchIndex(rollout, rollout.Status.GetSubStatus().CurrentStepIndex) + if nextStep != rollout.Status.GetSubStatus().NextStepIndex { + klog.Infof("rollout(%s/%s) invalid nextStepIndex(%d), reset to %d", rollout.Namespace, rollout.Name, nextStep, rollout.Status.GetSubStatus().NextStepIndex) + } + } +} diff --git a/pkg/webhook/workload/mutating/workload_update_handler.go b/pkg/webhook/workload/mutating/workload_update_handler.go index 60a443de..45ade337 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler.go +++ b/pkg/webhook/workload/mutating/workload_update_handler.go @@ -242,7 +242,7 @@ func (h *WorkloadHandler) handleStatefulSetLikeWorkload(newObj, oldObj *unstruct rollout, err := h.fetchMatchedRollout(newObj) if err != nil { return false, err - } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + } else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() { return false, nil } @@ -310,7 +310,7 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo rollout, err := h.fetchMatchedRollout(newObj) if err != nil { return false, err - } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + } else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() { return false, nil } rss, err := h.Finder.GetReplicaSetsForDeployment(newObj) @@ -319,7 +319,7 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo return false, nil } // if traffic routing, workload must only be one version of Pods - if len(rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 { + if rollout.Spec.Strategy.HasTrafficRoutings() { if len(rss) != 1 { klog.Warningf("Because deployment(%s/%s) have multiple versions of Pods, so can not enter rollout progressing", newObj.Namespace, newObj.Name) return false, nil @@ -334,6 +334,7 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo if newObj.Labels == nil { newObj.Labels = map[string]string{} } + // blueGreen also need the stable revision label newObj.Labels[appsv1alpha1.DeploymentStableRevisionLabel] = stableRS.Labels[apps.DefaultDeploymentUniqueLabelKey] } @@ -365,11 +366,11 @@ func (h *WorkloadHandler) handleCloneSet(newObj, oldObj *kruiseappsv1alpha1.Clon rollout, err := h.fetchMatchedRollout(newObj) if err != nil { return false, err - } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + } else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() { return false, nil } // if traffic routing, there must only be one version of Pods - if len(rollout.Spec.Strategy.Canary.TrafficRoutings) > 0 && newObj.Status.Replicas != newObj.Status.UpdatedReplicas { + if rollout.Spec.Strategy.HasTrafficRoutings() && newObj.Status.Replicas != newObj.Status.UpdatedReplicas { klog.Warningf("Because cloneSet(%s/%s) have multiple versions of Pods, so can not enter rollout progressing", newObj.Namespace, newObj.Name) return false, nil } @@ -398,7 +399,7 @@ func (h *WorkloadHandler) handleDaemonSet(newObj, oldObj *kruiseappsv1alpha1.Dae rollout, err := h.fetchMatchedRollout(newObj) if err != nil { return false, err - } else if rollout == nil || rollout.Spec.Strategy.Canary == nil { + } else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() { return false, nil } diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index c9ad8b34..cc135caa 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -2684,7 +2684,7 @@ var _ = SIGDescribe("Rollout", func() { }) }) - KruiseDescribe("Canary rollout with custon network provider", func() { + KruiseDescribe("Canary rollout with custom network provider", func() { It("V1->V2: Route traffic with header/queryParams/path matches and weight using rollout for VirtualService", func() { By("Creating Rollout...") rollout := &v1beta1.Rollout{} diff --git a/test/e2e/rollout_v1beta1_test.go b/test/e2e/rollout_v1beta1_test.go new file mode 100644 index 00000000..31b1c1dc --- /dev/null +++ b/test/e2e/rollout_v1beta1_test.go @@ -0,0 +1,1634 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" + appsv1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + "github.com/openkruise/rollouts/api/v1beta1" + "github.com/openkruise/rollouts/pkg/util" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + // "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + // "k8s.io/apimachinery/pkg/util/intstr" + // gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + // "github.com/openkruise/rollouts/api/v1alpha1" + // "k8s.io/apimachinery/pkg/api/errors" +) + +var _ = SIGDescribe("Rollout v1beta1", func() { + var namespace string + + DumpAllResources := func() { + rollout := &v1beta1.RolloutList{} + k8sClient.List(context.TODO(), rollout, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(rollout)) + batch := &v1beta1.BatchReleaseList{} + k8sClient.List(context.TODO(), batch, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(batch)) + deploy := &apps.DeploymentList{} + k8sClient.List(context.TODO(), deploy, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(deploy)) + rs := &apps.ReplicaSetList{} + k8sClient.List(context.TODO(), rs, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(rs)) + cloneSet := &appsv1alpha1.CloneSetList{} + k8sClient.List(context.TODO(), cloneSet, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(cloneSet)) + sts := &apps.StatefulSetList{} + k8sClient.List(context.TODO(), sts, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(sts)) + asts := &appsv1beta1.StatefulSetList{} + k8sClient.List(context.TODO(), asts, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(asts)) + } + + CreateObject := func(object client.Object, options ...client.CreateOption) { + object.SetNamespace(namespace) + Expect(k8sClient.Create(context.TODO(), object)).NotTo(HaveOccurred()) + } + + GetObject := func(name string, object client.Object) error { + key := types.NamespacedName{Namespace: namespace, Name: name} + return k8sClient.Get(context.TODO(), key, object) + } + + getRolloutCondition := func(status v1beta1.RolloutStatus, condType v1beta1.RolloutConditionType) *v1beta1.RolloutCondition { + for i := range status.Conditions { + c := status.Conditions[i] + if c.Type == condType { + return &c + } + } + return nil + } + + UpdateDeployment := func(object *apps.Deployment) *apps.Deployment { + var clone *apps.Deployment + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &apps.Deployment{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + clone.Spec.Template = *object.Spec.Template.DeepCopy() + clone.Labels = mergeMap(clone.Labels, object.Labels) + clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + clone.Spec.Paused = object.Spec.Paused + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + + UpdateCloneSet := func(object *appsv1alpha1.CloneSet) *appsv1alpha1.CloneSet { + var clone *appsv1alpha1.CloneSet + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &appsv1alpha1.CloneSet{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + clone.Spec.Template = *object.Spec.Template.DeepCopy() + clone.Labels = mergeMap(clone.Labels, object.Labels) + clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + + // UpdateDaemonSet := func(object *appsv1alpha1.DaemonSet) *appsv1alpha1.DaemonSet { + // var daemon *appsv1alpha1.DaemonSet + // Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + // daemon = &appsv1alpha1.DaemonSet{} + // err := GetObject(object.Name, daemon) + // if err != nil { + // return err + // } + // // daemon.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + // daemon.Spec.Template = *object.Spec.Template.DeepCopy() + // daemon.Spec.UpdateStrategy = *object.Spec.UpdateStrategy.DeepCopy() + // daemon.Labels = mergeMap(daemon.Labels, object.Labels) + // daemon.Annotations = mergeMap(daemon.Annotations, object.Annotations) + // return k8sClient.Update(context.TODO(), daemon) + // })).NotTo(HaveOccurred()) + + // return daemon + // } + + // UpdateNativeStatefulSet := func(object *apps.StatefulSet) *apps.StatefulSet { + // var clone *apps.StatefulSet + // Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + // clone = &apps.StatefulSet{} + // err := GetObject(object.Name, clone) + // if err != nil { + // return err + // } + // clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + // clone.Spec.Template = *object.Spec.Template.DeepCopy() + // clone.Labels = mergeMap(clone.Labels, object.Labels) + // clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + // return k8sClient.Update(context.TODO(), clone) + // })).NotTo(HaveOccurred()) + + // return clone + // } + + // UpdateAdvancedStatefulSet := func(object *appsv1beta1.StatefulSet) *appsv1beta1.StatefulSet { + // var clone *appsv1beta1.StatefulSet + // Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + // clone = &appsv1beta1.StatefulSet{} + // err := GetObject(object.Name, clone) + // if err != nil { + // return err + // } + // clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + // clone.Spec.Template = *object.Spec.Template.DeepCopy() + // clone.Labels = mergeMap(clone.Labels, object.Labels) + // clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + // return k8sClient.Update(context.TODO(), clone) + // })).NotTo(HaveOccurred()) + + // return clone + // } + + UpdateRollout := func(object *v1beta1.Rollout) *v1beta1.Rollout { + var clone *v1beta1.Rollout + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &v1beta1.Rollout{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec = *object.Spec.DeepCopy() + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + + ResumeRolloutCanary := func(name string) { + Eventually(func() bool { + clone := &v1beta1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStatus.CurrentStepState != v1beta1.CanaryStepStatePaused { + fmt.Println("resume rollout success, and CurrentStepState", util.DumpJSON(clone.Status)) + return true + } + + body := fmt.Sprintf(`{"status":{"canaryStatus":{"currentStepState":"%s"}}}`, v1beta1.CanaryStepStateReady) + Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) + return false + }, 10*time.Second, time.Second).Should(BeTrue()) + } + + RolloutJumpCanaryStep := func(name string, target int) { + Eventually(func() bool { + clone := &v1beta1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStatus.CurrentStepState != v1beta1.CanaryStepStatePaused { + fmt.Println("Jump successfully, and current status ", util.DumpJSON(clone.Status)) + return true + } + + body := fmt.Sprintf(`{"status":{"canaryStatus":{"nextStepIndex":%d}}}`, target) + Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) + return false + }, 10*time.Second, time.Second).Should(BeTrue()) + } + + // RolloutJumpBlueGreenStep := func(name string, target int) { + // Eventually(func() bool { + // clone := &v1alpha1.Rollout{} + // Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + // if clone.Status.CanaryStatus.CurrentStepState !=v1beta1.CanaryStepStatePaused { + // fmt.Println("Jump successfully, and current status ", util.DumpJSON(clone.Status)) + // return true + // } + + // body := fmt.Sprintf(`{"status":{"blueGreenStatus":{"nextStepIndex":"%d"}}}`, target) + // Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) + // return false + // }, 10*time.Second, time.Second).Should(BeTrue()) + // } + + WaitDeploymentAllPodsReady := func(deployment *apps.Deployment) { + Eventually(func() bool { + clone := &apps.Deployment{} + Expect(GetObject(deployment.Name, clone)).NotTo(HaveOccurred()) + return clone.Status.ObservedGeneration == clone.Generation && *clone.Spec.Replicas == clone.Status.UpdatedReplicas && + *clone.Spec.Replicas == clone.Status.ReadyReplicas && *clone.Spec.Replicas == clone.Status.Replicas + }, 5*time.Minute, time.Second).Should(BeTrue()) + } + + WaitCloneSetAllPodsReady := func(cloneset *appsv1alpha1.CloneSet) { + Eventually(func() bool { + clone := &appsv1alpha1.CloneSet{} + Expect(GetObject(cloneset.Name, clone)).NotTo(HaveOccurred()) + return clone.Status.ObservedGeneration == clone.Generation && *clone.Spec.Replicas == clone.Status.UpdatedReplicas && + *clone.Spec.Replicas == clone.Status.ReadyReplicas && *clone.Spec.Replicas == clone.Status.Replicas + }, 5*time.Minute, time.Second).Should(BeTrue()) + } + + // WaitNativeStatefulSetPodsReady := func(statefulset *apps.StatefulSet) { + // Eventually(func() bool { + // set := &apps.StatefulSet{} + // Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) + // return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && + // *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas + // }, 20*time.Minute, 3*time.Second).Should(BeTrue()) + // } + + // WaitAdvancedStatefulSetPodsReady := func(statefulset *appsv1beta1.StatefulSet) { + // Eventually(func() bool { + // set := &appsv1beta1.StatefulSet{} + // Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) + // return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && + // *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas + // }, 20*time.Minute, 3*time.Second).Should(BeTrue()) + // } + + // WaitDaemonSetAllPodsReady := func(daemonset *appsv1alpha1.DaemonSet) { + // Eventually(func() bool { + // daemon := &appsv1alpha1.DaemonSet{} + // Expect(GetObject(daemonset.Name, daemon)).NotTo(HaveOccurred()) + // klog.Infof("DaemonSet updateStrategy(%s) Generation(%d) ObservedGeneration(%d) DesiredNumberScheduled(%d) UpdatedNumberScheduled(%d) NumberReady(%d)", + // util.DumpJSON(daemon.Spec.UpdateStrategy), daemon.Generation, daemon.Status.ObservedGeneration, daemon.Status.DesiredNumberScheduled, daemon.Status.UpdatedNumberScheduled, daemon.Status.NumberReady) + // return daemon.Status.ObservedGeneration == daemon.Generation && daemon.Status.DesiredNumberScheduled == daemon.Status.UpdatedNumberScheduled && daemon.Status.DesiredNumberScheduled == daemon.Status.NumberReady + // }, 5*time.Minute, time.Second).Should(BeTrue()) + // } + + // WaitDeploymentReplicas := func(deployment *apps.Deployment) { + // Eventually(func() bool { + // clone := &apps.Deployment{} + // Expect(GetObject(deployment.Name, clone)).NotTo(HaveOccurred()) + // return clone.Status.ObservedGeneration == clone.Generation && + // *clone.Spec.Replicas == clone.Status.ReadyReplicas && *clone.Spec.Replicas == clone.Status.Replicas + // }, 10*time.Minute, time.Second).Should(BeTrue()) + // } + + WaitRolloutCanaryStepPaused := func(name string, stepIndex int32) { + start := time.Now() + Eventually(func() bool { + if start.Add(time.Minute * 5).Before(time.Now()) { + DumpAllResources() + Expect(true).Should(BeFalse()) + } + clone := &v1beta1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStatus == nil { + return false + } + klog.Infof("current step:%v target step:%v current step state %v", clone.Status.CanaryStatus.CurrentStepIndex, stepIndex, clone.Status.CanaryStatus.CurrentStepState) + return clone.Status.CanaryStatus.CurrentStepIndex == stepIndex && clone.Status.CanaryStatus.CurrentStepState == v1beta1.CanaryStepStatePaused + }, 20*time.Minute, time.Second).Should(BeTrue()) + } + + WaitRolloutStatusPhase := func(name string, phase v1beta1.RolloutPhase) { + Eventually(func() bool { + clone := &v1beta1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + return clone.Status.Phase == phase + }, 20*time.Minute, time.Second).Should(BeTrue()) + } + + WaitRolloutWorkloadGeneration := func(name string, generation int64) { + Eventually(func() bool { + clone := &v1beta1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + return clone.Status.CanaryStatus.ObservedWorkloadGeneration == generation + }, time.Minute, time.Second).Should(BeTrue()) + } + + // WaitRolloutNotFound := func(name string) { + // Eventually(func() bool { + // clone := &v1beta1.Rollout{} + // err := GetObject(name, clone) + // if err == nil { + // return false + // } else if errors.IsNotFound(err) { + // return true + // } else { + // Expect(err).NotTo(HaveOccurred()) + // return false + // } + // }, 5*time.Minute, time.Second).Should(BeTrue()) + // } + + GetCanaryDeployment := func(stable *apps.Deployment) (*apps.Deployment, error) { + canaryList := &apps.DeploymentList{} + selector, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: map[string]string{util.CanaryDeploymentLabel: stable.Name}}) + err := k8sClient.List(context.TODO(), canaryList, &client.ListOptions{Namespace: stable.Namespace, LabelSelector: selector}) + if err != nil { + return nil, err + } else if len(canaryList.Items) == 0 { + return nil, nil + } + sort.Slice(canaryList.Items, func(i, j int) bool { + return canaryList.Items[j].CreationTimestamp.Before(&canaryList.Items[i].CreationTimestamp) + }) + return &canaryList.Items[0], nil + } + + // ListPods := func(namespace string, labelSelector *metav1.LabelSelector) ([]*v1.Pod, error) { + // appList := &v1.PodList{} + // selector, _ := metav1.LabelSelectorAsSelector(labelSelector) + // err := k8sClient.List(context.TODO(), appList, &client.ListOptions{Namespace: namespace, LabelSelector: selector}) + // if err != nil { + // return nil, err + // } + // apps := make([]*v1.Pod, 0) + // for i := range appList.Items { + // pod := &appList.Items[i] + // if pod.DeletionTimestamp.IsZero() { + // apps = append(apps, pod) + // } + // } + // return apps, nil + // } + + // CheckPodBatchLabel := func(namespace string, labelSelector *metav1.LabelSelector, rolloutID, batchID string, expected int) { + // pods, err := ListPods(namespace, labelSelector) + // Expect(err).NotTo(HaveOccurred()) + + // count := 0 + // for _, pod := range pods { + // if pod.Labels[v1beta1.RolloutIDLabel] == rolloutID && + // pod.Labels[v1beta1.RolloutBatchIDLabel] == batchID { + // count++ + // } + // } + // Expect(count).Should(BeNumerically("==", expected)) + // } + + ListReplicaSet := func(d *apps.Deployment) []*apps.ReplicaSet { + var rss []*apps.ReplicaSet + rsLister := &apps.ReplicaSetList{} + selectorOpt, _ := metav1.LabelSelectorAsSelector(d.Spec.Selector) + err := k8sClient.List(context.TODO(), rsLister, &client.ListOptions{LabelSelector: selectorOpt, Namespace: d.Namespace}) + Expect(err).NotTo(HaveOccurred()) + for i := range rsLister.Items { + rs := &rsLister.Items[i] + if !rs.DeletionTimestamp.IsZero() { + continue + } + rss = append(rss, rs) + } + return rss + } + + GetStableRSRevision := func(d *apps.Deployment) string { + rss := ListReplicaSet(d) + _, stable := util.FindCanaryAndStableReplicaSet(rss, d) + if stable != nil { + return stable.Labels[apps.DefaultDeploymentUniqueLabelKey] + } + return "" + } + + GetCanaryRSRevision := func(d *apps.Deployment) string { + rss := ListReplicaSet(d) + canary, _ := util.FindCanaryAndStableReplicaSet(rss, d) + if canary != nil { + return canary.Labels[apps.DefaultDeploymentUniqueLabelKey] + } + return "" + } + + BeforeEach(func() { + namespace = randomNamespaceName("rollout") + ns := v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + Expect(k8sClient.Create(context.TODO(), &ns)).Should(SatisfyAny(BeNil())) + }) + + AfterEach(func() { + By("[TEST] Clean up resources after an integration test") + k8sClient.DeleteAllOf(context.TODO(), &apps.Deployment{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &appsv1alpha1.CloneSet{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &v1beta1.BatchRelease{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &v1beta1.Rollout{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &v1.Service{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &netv1.Ingress{}, client.InNamespace(namespace)) + Expect(k8sClient.Delete(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed()) + time.Sleep(time.Second * 3) + }) + + KruiseDescribe("Step Jump", func() { + // step1-> 2-> 3-> 4-> 3-(TrafficChange)-> 3-> 2-> 1-> 5 + It("V1->V2: Deployment, Canary, patch nextStepIndex to jump", func() { + finder := util.NewControllerFinder(k8sClient) + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + CreateObject(rollout) + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + By("Update deployment image from(version1) -> to(version2)") + time.Sleep(time.Second * 3) + // wait step 1 complete + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + // canary workload + cWorkload, err := GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + crss, err := finder.GetReplicaSetsForDeployment(cWorkload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crss)).Should(BeNumerically("==", 1)) + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 1)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + + // wait step 2 complete + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + // canary workload + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 2)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + + // wait step 3 complete + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + // rollout + WaitRolloutCanaryStepPaused(rollout.Name, 3) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + // canary workload + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 3)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + + // wait step 4 complete + By("wait step(4) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 4) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 5)) + // canary workload + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + // canary service + cService := &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress := &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[3].Traffic))) + + // Jump to step 3 + By("Jump to step 3") + RolloutJumpCanaryStep(rollout.Name, 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + // canary workload (won't scale down indeed) + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + canaryRevision = crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + // canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Change traffic of current step, which shouldn't cause jump + By("Change traffic of step 3") + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // update rollout step configuration + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("21%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("41%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("61%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("81%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("100%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + } + rollout = UpdateRollout(rollout) + By("update rollout configuration, and wait rollout re-run current step(3)") + time.Sleep(time.Second * 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // batch release + batch := &v1beta1.BatchRelease{} + Expect(GetObject(rollout.Name, batch)).NotTo(HaveOccurred()) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + // canary workload (won't scale down indeed) + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + canaryRevision = crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + // canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Jump to step 2 + By("Jump to step 2") + RolloutJumpCanaryStep(rollout.Name, 2) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + // canary workload (won't scale down indeed) + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + canaryRevision = crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + // canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + + // Jump to step 1 + By("Jump to step 1") + RolloutJumpCanaryStep(rollout.Name, 1) + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + // canary workload (won't scale down indeed) + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + canaryRevision = crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 4)) + // workload + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5)) + // canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) + + // Jump to step 5 + By("Jump to step 5") + RolloutJumpCanaryStep(rollout.Name, 5) + // wait rollout complete + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy) + klog.Infof("rollout(%s) completed, and check", namespace) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", -1)) + // check service & ingress & deployment + // ingress + Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) + cIngress = &netv1.Ingress{} + Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) + // service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) + cService = &v1.Service{} + Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) + // deployment + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Spec.Paused).Should(BeFalse()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + // check progressing succeed + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + + }) + + // step1-> 2-> 3-> 4-> 3-(TrafficChange)-> 3-> 2-> 1-> 5 + It("V1->V2: Deployment, Partition, patch nextStepIndex to jump", func() { + finder := util.NewControllerFinder(k8sClient) + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + CreateObject(rollout) + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + By("Update deployment image from(version1) -> to(version2)") + time.Sleep(time.Second * 3) + + // wait step 1 complete + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + stableRevision := GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + strategy := util.GetDeploymentStrategy(workload) + extraStatus := util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService := &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress := &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) + + // wait step 2 complete + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 2)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 2)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + + // wait step 3 complete + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // wait step 4 complete + By("wait step(4) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 4) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 5)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[3].Traffic))) + + // Jump to step 3 + By("Jump to step 3") + RolloutJumpCanaryStep(rollout.Name, 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + // won't scale down + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Change traffic of current step, which shouldn't cause jump + By("Change traffic of step 3") + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // update rollout step configuration + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("21%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("41%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("61%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("81%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("100%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + } + rollout = UpdateRollout(rollout) + By("update rollout configuration, and wait rollout re-run current step(3)") + time.Sleep(time.Second * 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // batch release + batch := &v1beta1.BatchRelease{} + Expect(GetObject(rollout.Name, batch)).NotTo(HaveOccurred()) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + // won't scale down + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Jump to step 2 + By("Jump to step 2") + RolloutJumpCanaryStep(rollout.Name, 2) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + + // Jump to step 1 + By("Jump to step 1") + RolloutJumpCanaryStep(rollout.Name, 1) + WaitRolloutCanaryStepPaused(rollout.Name, 1) + stableRevision = GetStableRSRevision(workload) + By(stableRevision) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + strategy = util.GetDeploymentStrategy(workload) + extraStatus = util.GetDeploymentExtraStatus(workload) + Expect(extraStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(strategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil))) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload))) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) + + // Jump to step 5 + By("Jump to step 5") + RolloutJumpCanaryStep(rollout.Name, 5) + // wait rollout complete + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhase(v1beta1.RolloutPhaseHealthy)) + klog.Infof("rollout(%s) completed, and check", namespace) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", -1)) + // check service & ingress & deployment + // ingress + Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) + cIngress = &netv1.Ingress{} + Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) + // service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) + cService = &v1.Service{} + Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) + // deployment + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Spec.Paused).Should(BeFalse()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + // check progressing succeed + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + // step1-> 2-> 3-> 4-> 3-(TrafficChange)-> 3-> 2-> 1-> 5 + It("V1->V2: CloneSet, Partition, patch nextStepIndex to jump", func() { + By("Creating Rollout...") + rollout := &v1beta1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_v1beta1_partition_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.WorkloadRef = v1beta1.ObjectRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + CreateObject(rollout) + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.CanaryStatus.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + time.Sleep(time.Second * 3) + + // wait step 1 complete + By("wait step(1) pause") + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService := &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress := &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) + + // wait step 2 complete + By("wait step(2) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 2)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 2)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + + // wait step 3 complete + By("wait step(3) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Change traffic of current step, which shouldn't cause jump + By("Change traffic of step 3") + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // update rollout step configuration + rollout.Spec.Strategy.Canary.Steps = []v1beta1.CanaryStep{ + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("21%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "20%"}, + Pause: v1beta1.RolloutPause{}, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("41%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "40%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("61%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "60%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("81%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "80%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + { + TrafficRoutingStrategy: v1beta1.TrafficRoutingStrategy{ + Traffic: utilpointer.StringPtr("100%"), + }, + Replicas: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + Pause: v1beta1.RolloutPause{ + Duration: utilpointer.Int32(10), + }, + }, + } + rollout = UpdateRollout(rollout) + By("update rollout configuration, and wait rollout re-run current step(3)") + time.Sleep(time.Second * 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // batch release + batch := &v1beta1.BatchRelease{} + Expect(GetObject(rollout.Name, batch)).NotTo(HaveOccurred()) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // wait step 4 complete + By("wait step(4) pause") + ResumeRolloutCanary(rollout.Name) + WaitRolloutCanaryStepPaused(rollout.Name, 4) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 5)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[3].Traffic))) + + // Jump to step 3 + By("Jump to step 3") + RolloutJumpCanaryStep(rollout.Name, 3) + WaitRolloutCanaryStepPaused(rollout.Name, 3) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 4)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[2].Traffic))) + + // Jump to step 2 + By("Jump to step 2") + RolloutJumpCanaryStep(rollout.Name, 2) + WaitRolloutCanaryStepPaused(rollout.Name, 2) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 3)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic))) + + // Jump to step 1 + By("Jump to step 1") + RolloutJumpCanaryStep(rollout.Name, 1) + WaitRolloutCanaryStepPaused(rollout.Name, 1) + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 4)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision = rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService = &v1.Service{} + Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[0].Traffic))) + + // Jump to step 5 + By("Jump to step 5") + RolloutJumpCanaryStep(rollout.Name, 5) + // wait rollout complete + WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhase(v1beta1.RolloutPhaseHealthy)) + klog.Infof("rollout(%s) completed, and check", namespace) + // rollout + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.CanaryStatus.NextStepIndex).Should(BeNumerically("==", -1)) + // check service & ingress + // ingress + Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) + cIngress = &netv1.Ingress{} + Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) + // service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) + cService = &v1.Service{} + Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) + // clonese + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + // check progressing succeed + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse))) + cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + + }) + }) +}) + +func removePercentageSign(input string) string { + if input == "0" { + return "0" + } + if strings.HasSuffix(input, "%") { + return strings.TrimSuffix(input, "%") + } + fmt.Printf("input(%s) has no percentage sign!", input) + return "" +} diff --git a/test/e2e/test_data/rollout/rollout_v1beta1_bluegreen_base.yaml b/test/e2e/test_data/rollout/rollout_v1beta1_bluegreen_base.yaml new file mode 100644 index 00000000..0959f065 --- /dev/null +++ b/test/e2e/test_data/rollout/rollout_v1beta1_bluegreen_base.yaml @@ -0,0 +1,32 @@ +apiVersion: rollouts.kruise.io/v1beta1 # we use v1beta1 +kind: Rollout +metadata: + name: rollouts-demo +spec: + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: echoserver + strategy: + blueGreen: + steps: + - traffic: 20% + replicas: 20% + pause: {} + - traffic: 40% + replicas: 40% + pause: {duration: 10} + - traffic: 60% + replicas: 60% + pause: {duration: 10} + - traffic: 80% + replicas: 80% + pause: {duration: 10} + - traffic: 100% + replicas: 100% + pause: {duration: 0} + trafficRoutings: + - service: echoserver + ingress: + classType: nginx + name: echoserver diff --git a/test/e2e/test_data/rollout/rollout_v1beta1_canary_base.yaml b/test/e2e/test_data/rollout/rollout_v1beta1_canary_base.yaml new file mode 100644 index 00000000..d3b2cbc6 --- /dev/null +++ b/test/e2e/test_data/rollout/rollout_v1beta1_canary_base.yaml @@ -0,0 +1,33 @@ +apiVersion: rollouts.kruise.io/v1beta1 # we use v1beta1 +kind: Rollout +metadata: + name: rollouts-demo +spec: + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: echoserver + strategy: + canary: + enableExtraWorkloadForCanary: true + steps: + - traffic: 20% + replicas: 20% + pause: {} + - traffic: 40% + replicas: 40% + pause: {duration: 10} + - traffic: 60% + replicas: 60% + pause: {duration: 10} + - traffic: 80% + replicas: 80% + pause: {duration: 10} + - traffic: 100% + replicas: 100% + pause: {duration: 0} + trafficRoutings: + - service: echoserver + ingress: + classType: nginx + name: echoserver diff --git a/test/e2e/test_data/rollout/rollout_v1beta1_partition_base.yaml b/test/e2e/test_data/rollout/rollout_v1beta1_partition_base.yaml new file mode 100644 index 00000000..2b492ea7 --- /dev/null +++ b/test/e2e/test_data/rollout/rollout_v1beta1_partition_base.yaml @@ -0,0 +1,33 @@ +apiVersion: rollouts.kruise.io/v1beta1 # we use v1beta1 +kind: Rollout +metadata: + name: rollouts-demo +spec: + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: echoserver + strategy: + canary: + enableExtraWorkloadForCanary: false + steps: + - traffic: 20% + replicas: 20% + pause: {} + - traffic: 40% + replicas: 40% + pause: {duration: 10} + - traffic: 60% + replicas: 60% + pause: {duration: 10} + - traffic: 80% + replicas: 80% + pause: {duration: 10} + - traffic: 100% + replicas: 100% + pause: {duration: 0} + trafficRoutings: + - service: echoserver + ingress: + classType: nginx + name: echoserver