openkruise · kruise-bot · Jan 11, 2023 · Jan 3, 2023
diff --git a/.github/workflows/e2e-advanced-deployment-1.19.yaml b/.github/workflows/e2e-advanced-deployment-1.19.yaml
@@ -0,0 +1,110 @@
+name: E2E-Advanced-Deployment-1.19
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+env:
+  # Common versions
+  GO_VERSION: '1.17'
+  KIND_IMAGE: 'kindest/node:v1.19.16'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+
+  rollout:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/[email protected]
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl cluster-info
+          make helm
+          helm repo add openkruise https://openkruise.github.io/charts/
+          helm repo update
+          helm install kruise openkruise/kruise
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l)
+            set -e
+            if [ "$PODS" -eq "2" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l)
+          set -e
+          if [ "$PODS" -eq "2" ]; then
+            echo "Wait for kruise-manager ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager ready"
+            exit 1
+          fi
+      - name: Install Kruise Rollout
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "1" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-rollout -o yaml
+          set -e
+          if [ "$PODS" -eq "1" ]; then
+            echo "Wait for kruise-rollout ready successfully"
+          else
+            echo "Timeout to wait for kruise-rollout ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='Advanced Deployment' test/e2e
+          retVal=$?
+          # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout
+          restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-rollout has not restarted"
+          else
+              kubectl get pod -n kruise-rollout --no-headers
+              echo "Kruise-rollout has restarted, abort!!!"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
+              exit 1
+          fi
+          exit $retVal
diff --git a/.github/workflows/e2e-advanced-deployment-1.23.yaml b/.github/workflows/e2e-advanced-deployment-1.23.yaml
@@ -0,0 +1,110 @@
+name: E2E-Advanced-Deployment-1.23
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+env:
+  # Common versions
+  GO_VERSION: '1.17'
+  KIND_IMAGE: 'kindest/node:v1.23.3'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+
+  rollout:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/[email protected]
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl cluster-info
+          make helm
+          helm repo add openkruise https://openkruise.github.io/charts/
+          helm repo update
+          helm install kruise openkruise/kruise
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l)
+            set -e
+            if [ "$PODS" -eq "2" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l)
+          set -e
+          if [ "$PODS" -eq "2" ]; then
+            echo "Wait for kruise-manager ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager ready"
+            exit 1
+          fi
+      - name: Install Kruise Rollout
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "1" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-rollout -o yaml
+          set -e
+          if [ "$PODS" -eq "1" ]; then
+            echo "Wait for kruise-rollout ready successfully"
+          else
+            echo "Timeout to wait for kruise-rollout ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='Advanced Deployment' test/e2e
+          retVal=$?
+          # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout
+          restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-rollout has not restarted"
+          else
+              kubectl get pod -n kruise-rollout --no-headers
+              echo "Kruise-rollout has restarted, abort!!!"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
+              exit 1
+          fi
+          exit $retVal
diff --git a/api/v1alpha1/deployment_types.go b/api/v1alpha1/deployment_types.go
@@ -31,16 +31,14 @@ type DeploymentStrategy struct {
 type RollingStyleType string
 
 const (
-	// PartitionRollingStyleType means rolling in batches just like CloneSet, and will NOT create any extra Deployment;
-	PartitionRollingStyleType RollingStyleType = "Partition"
-	// CanaryRollingStyleType means rolling in canary way, and will create a canary Deployment.
-	CanaryRollingStyleType RollingStyleType = "Canary"
+	// PartitionRollingStyle means rolling in batches just like CloneSet, and will NOT create any extra Deployment;
+	PartitionRollingStyle RollingStyleType = "Partition"
+	// CanaryRollingStyle means rolling in canary way, and will create a canary Deployment.
+	CanaryRollingStyle RollingStyleType = "Canary"
 )
 
 // DeploymentExtraStatus is extra status field for Advanced Deployment
 type DeploymentExtraStatus struct {
-	// ObservedGeneration record the generation of deployment this status observed.
-	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 	// UpdatedReadyReplicas the number of pods that has been updated and ready.
 	UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"`
 	// ExpectedUpdatedReplicas is an absolute number calculated based on Partition
@@ -52,7 +50,7 @@ type DeploymentExtraStatus struct {
 }
 
 func SetDefaultDeploymentStrategy(strategy *DeploymentStrategy) {
-	if strategy.RollingStyle == CanaryRollingStyleType {
+	if strategy.RollingStyle == CanaryRollingStyle {
 		return
 	}
 	if strategy.RollingUpdate == nil {

diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml
@@ -14,4 +14,5 @@ spec:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
+        - "--feature-gates=AdvancedDeployment=true"
         - "--v=3"
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -29,6 +29,7 @@ spec:
         - /manager
         args:
         - --leader-elect
+        - --feature-gates=AdvancedDeployment=true
         image: controller:latest
         name: manager
         securityContext:

diff --git a/main.go b/main.go
@@ -24,6 +24,7 @@ import (
 	kruisev1beta1 "github.com/openkruise/kruise-api/apps/v1beta1"
 	rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
 	br "github.com/openkruise/rollouts/pkg/controller/batchrelease"
+	"github.com/openkruise/rollouts/pkg/controller/deployment"
 	"github.com/openkruise/rollouts/pkg/controller/rollout"
 	"github.com/openkruise/rollouts/pkg/controller/rollouthistory"
 	utilclient "github.com/openkruise/rollouts/pkg/util/client"
@@ -116,6 +117,10 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "rollouthistory")
 		os.Exit(1)
 	}
+	if err = deployment.Add(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "advanceddeployment")
+		os.Exit(1)
+	}
 
 	//+kubebuilder:scaffold:builder
 	setupLog.Info("setup webhook")

diff --git a/pkg/controller/deployment/controller.go b/pkg/controller/deployment/controller.go
@@ -22,14 +22,15 @@ import (
 	"encoding/json"
 	"flag"
 	"reflect"
+	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/client-go/kubernetes/scheme"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	appslisters "k8s.io/client-go/listers/apps/v1"
-	corelisters "k8s.io/client-go/listers/core/v1"
 	toolscache "k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
@@ -51,9 +52,13 @@ import (
 )
 
 func init() {
-	flag.IntVar(&concurrentReconciles, "deployment-workers", concurrentReconciles, "Max concurrent workers for StatefulSet controller.")
+	flag.IntVar(&concurrentReconciles, "deployment-workers", concurrentReconciles, "Max concurrent workers for advanced deployment controller.")
 }
 
+const (
+	DefaultRetryDuration = 2 * time.Second
+)
+
 var (
 	concurrentReconciles = 3
 )
@@ -75,10 +80,6 @@ func Add(mgr manager.Manager) error {
 // newReconciler returns a new reconcile.Reconciler
 func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
 	cacher := mgr.GetCache()
-	podInformer, err := cacher.GetInformerForKind(context.TODO(), v1.SchemeGroupVersion.WithKind("Pod"))
-	if err != nil {
-		return nil, err
-	}
 	dInformer, err := cacher.GetInformerForKind(context.TODO(), appsv1.SchemeGroupVersion.WithKind("Deployment"))
 	if err != nil {
 		return nil, err
@@ -91,7 +92,6 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
 	// Lister
 	dLister := appslisters.NewDeploymentLister(dInformer.(toolscache.SharedIndexInformer).GetIndexer())
 	rsLister := appslisters.NewReplicaSetLister(rsInformer.(toolscache.SharedIndexInformer).GetIndexer())
-	podLister := corelisters.NewPodLister(podInformer.(toolscache.SharedIndexInformer).GetIndexer())
 
 	// Client & Recorder
 	genericClient := clientutil.GetGenericClientWithName("advanced-deployment-controller")
@@ -107,7 +107,6 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
 		eventRecorder:    recorder,
 		dLister:          dLister,
 		rsLister:         rsLister,
-		podLister:        podLister,
 	}
 	return &ReconcileDeployment{Client: mgr.GetClient(), controllerFactory: factory}, nil
 }
@@ -179,8 +178,22 @@ func (r *ReconcileDeployment) Reconcile(_ context.Context, request reconcile.Req
 		return reconcile.Result{}, nil
 	}
 
+	errList := field.ErrorList{}
 	err = dc.syncDeployment(context.Background(), deployment)
-	return ctrl.Result{}, err
+	if err != nil {
+		errList = append(errList, field.InternalError(field.NewPath("syncDeployment"), err))
+	}
+	err = dc.patchExtraStatus(deployment)
+	if err != nil {
+		errList = append(errList, field.InternalError(field.NewPath("patchExtraStatus"), err))
+	}
+	err = deploymentutil.DeploymentRolloutSatisfied(deployment, dc.strategy.Partition)
+	if err != nil {
+		klog.V(3).Infof("Deployment %v is still rolling: %v", klog.KObj(deployment), err)
+		return reconcile.Result{RequeueAfter: DefaultRetryDuration}, errList.ToAggregate()
+	}
+
+	return ctrl.Result{}, errList.ToAggregate()
 }
 
 type controllerFactory DeploymentController
@@ -201,7 +214,7 @@ func (f *controllerFactory) NewController(deployment *appsv1.Deployment) *Deploy
 	}
 
 	// We do NOT process such deployment with canary rolling style
-	if strategy.RollingStyle == rolloutsv1alpha1.CanaryRollingStyleType {
+	if strategy.RollingStyle == rolloutsv1alpha1.CanaryRollingStyle {
 		return nil
 	}
 
@@ -214,7 +227,6 @@ func (f *controllerFactory) NewController(deployment *appsv1.Deployment) *Deploy
 		eventRecorder:    f.eventRecorder,
 		dLister:          f.dLister,
 		rsLister:         f.rsLister,
-		podLister:        f.podLister,
 		strategy:         strategy,
 	}
 }