Disable DeepCopy for some specific cache list (#916)

Signed-off-by: FillZpp <[email protected]>
openkruise · Mar 8, 2022 · 2953c6c · 2953c6c
1 parent 2b1eaf0
commit 2953c6c
Show file tree

Hide file tree

Showing 25 changed files with 597 additions and 96 deletions.
diff --git a/.github/workflows/e2e-1.16.yaml b/.github/workflows/e2e-1.16.yaml
@@ -70,7 +70,20 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
+          set +e
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] StatefulSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "$out"
+              echo "Kruise-manager has not restarted"
+          else
+              echo "$out"
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          exit $retVal
 
   pullimages-containerrecreate:
     runs-on: ubuntu-18.04
@@ -126,7 +139,19 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
+          set +e
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] (PullImage|ContainerRecreateRequest)' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          exit $retVal
 
   advanced-daemonset:
     runs-on: ubuntu-18.04
@@ -182,7 +207,19 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
+          set +e
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] DaemonSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          exit $retVal
 
   sidecarset:
     runs-on: ubuntu-18.04
@@ -238,7 +275,19 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
+          set +e
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] SidecarSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          exit $retVal
 
   other:
     runs-on: ubuntu-18.04
@@ -296,10 +345,14 @@ jobs:
           make ginkgo
           set +e
           ./bin/ginkgo -timeout 60m -v --skip='\[apps\] (StatefulSet|PullImage|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' test/e2e
-#          retVal=$?
-#          if [ $retVal -ne 0 ]; then
-#            kubectl get pod -n kruise-system -o wide
-#            kubectl get pod -n kruise-system --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-system
-#            kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
-#          fi
-#          exit $retVal
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          exit $retVal
diff --git a/.github/workflows/e2e-1.22.yaml → .github/workflows/e2e-1.23.yaml b/.github/workflows/e2e-1.22.yaml → .github/workflows/e2e-1.23.yaml
@@ -1,4 +1,4 @@
-name: E2E-1.22
+name: E2E-1.23
 
 on:
   push:
@@ -11,7 +11,7 @@ on:
 env:
   # Common versions
   GO_VERSION: '1.17'
-  KIND_IMAGE: 'kindest/node:v1.22.0'
+  KIND_IMAGE: 'kindest/node:v1.23.3'
   KIND_CLUSTER_NAME: 'ci-testing'
 
 jobs:
@@ -70,7 +70,19 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
+          set +e
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] StatefulSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          exit $retVal
 
   pullimages-containerrecreate:
     runs-on: ubuntu-18.04
@@ -126,7 +138,19 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
+          set +e
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] (PullImage|ContainerRecreateRequest)' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          exit $retVal
 
   advanced-daemonset:
     runs-on: ubuntu-18.04
@@ -182,7 +206,19 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
+          set +e
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] DaemonSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          exit $retVal
 
   sidecarset:
     runs-on: ubuntu-18.04
@@ -238,7 +274,19 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
+          set +e
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] SidecarSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          exit $retVal
 
   other:
     runs-on: ubuntu-18.04
@@ -296,10 +344,14 @@ jobs:
           make ginkgo
           set +e
           ./bin/ginkgo -timeout 60m -v --skip='\[apps\] (StatefulSet|PullImage|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' test/e2e
-#          retVal=$?
-#          if [ $retVal -ne 0 ]; then
-#            kubectl get pod -n kruise-system -o wide
-#            kubectl get pod -n kruise-system --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-system
-#            kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
-#          fi
-#          exit $retVal
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          exit $retVal
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -40,6 +40,8 @@ spec:
         imagePullPolicy: Always
         name: manager
         env:
+          - name: KUBE_CACHE_MUTATION_DETECTOR
+            value: "true"
           - name: POD_NAMESPACE
             valueFrom:
               fieldRef:
@@ -97,6 +99,8 @@ spec:
         imagePullPolicy: Always
         name: daemon
         env:
+        - name: KUBE_CACHE_MUTATION_DETECTOR
+          value: "true"
         - name: NODE_NAME
           valueFrom:
             fieldRef:

diff --git a/main.go b/main.go
@@ -38,6 +38,7 @@ import (
 
 	extclient "github.com/openkruise/kruise/pkg/client"
 	"github.com/openkruise/kruise/pkg/features"
+	utilclient "github.com/openkruise/kruise/pkg/util/client"
 	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
 	"github.com/openkruise/kruise/pkg/util/fieldindex"
 	"github.com/openkruise/kruise/pkg/webhook"
@@ -142,6 +143,7 @@ func main() {
 		LeaderElectionResourceLock: resourcelock.ConfigMapsResourceLock,
 		Namespace:                  namespace,
 		SyncPeriod:                 syncPeriod,
+		NewClient:                  utilclient.NewClient,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")

diff --git a/pkg/controller/cloneset/cloneset_controller.go b/pkg/controller/cloneset/cloneset_controller.go
@@ -30,6 +30,7 @@ import (
 	clonesetutils "github.com/openkruise/kruise/pkg/controller/cloneset/utils"
 	"github.com/openkruise/kruise/pkg/features"
 	"github.com/openkruise/kruise/pkg/util"
+	utilclient "github.com/openkruise/kruise/pkg/util/client"
 	utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
 	"github.com/openkruise/kruise/pkg/util/expectations"
 	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
@@ -478,7 +479,7 @@ func (r *ReconcileCloneSet) getOwnedResource(cs *appsv1alpha1.CloneSet) ([]*v1.P
 	}
 
 	pvcList := v1.PersistentVolumeClaimList{}
-	if err := r.List(context.TODO(), &pvcList, opts); err != nil {
+	if err := r.List(context.TODO(), &pvcList, opts, utilclient.DisableDeepCopy); err != nil {
 		return nil, nil, err
 	}
 	var filteredPVCs []*v1.PersistentVolumeClaim

diff --git a/pkg/controller/cloneset/sync/cloneset_scale.go b/pkg/controller/cloneset/sync/cloneset_scale.go
@@ -177,11 +177,11 @@ func (r *realControl) managePreparingDelete(cs *appsv1alpha1.CloneSet, pods, pod
 
 		klog.V(3).Infof("CloneSet %s patch pod %s lifecycle from PreparingDelete to Normal",
 			clonesetutils.GetControllerKey(cs), pod.Name)
-		if updated, err := r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStateNormal); err != nil {
+		if updated, gotPod, err := r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStateNormal); err != nil {
 			return modified, err
 		} else if updated {
 			modified = true
-			clonesetutils.ResourceVersionExpectations.Expect(pod)
+			clonesetutils.ResourceVersionExpectations.Expect(gotPod)
 		}
 		diff--
 	}
@@ -270,13 +270,13 @@ func (r *realControl) deletePods(cs *appsv1alpha1.CloneSet, podsToDelete []*v1.P
 	var modified bool
 	for _, pod := range podsToDelete {
 		if cs.Spec.Lifecycle != nil && lifecycle.IsPodHooked(cs.Spec.Lifecycle.PreDelete, pod) {
-			if updated, err := r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingDelete); err != nil {
+			if updated, gotPod, err := r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingDelete); err != nil {
 				return false, err
 			} else if updated {
 				klog.V(3).Infof("CloneSet %s scaling update pod %s lifecycle to PreparingDelete",
 					clonesetutils.GetControllerKey(cs), pod.Name)
 				modified = true
-				clonesetutils.ResourceVersionExpectations.Expect(pod)
+				clonesetutils.ResourceVersionExpectations.Expect(gotPod)
 			}
 			continue
 		}

diff --git a/pkg/controller/cloneset/sync/cloneset_update.go b/pkg/controller/cloneset/sync/cloneset_update.go
@@ -185,10 +185,10 @@ func (c *realControl) refreshPodState(cs *appsv1alpha1.CloneSet, coreControl clo
 	}
 
 	if state != "" {
-		if updated, err := c.lifecycleControl.UpdatePodLifecycle(pod, state); err != nil {
+		if updated, gotPod, err := c.lifecycleControl.UpdatePodLifecycle(pod, state); err != nil {
 			return false, 0, err
 		} else if updated {
-			clonesetutils.ResourceVersionExpectations.Expect(pod)
+			clonesetutils.ResourceVersionExpectations.Expect(gotPod)
 			klog.V(3).Infof("CloneSet %s update pod %s lifecycle to %s", clonesetutils.GetControllerKey(cs), pod.Name, state)
 			return true, res.DelayDuration, nil
 		}
@@ -217,9 +217,10 @@ func (c *realControl) updatePod(cs *appsv1alpha1.CloneSet, coreControl clonesetc
 			case "", appspub.LifecycleStateNormal:
 				var err error
 				var updated bool
+				var gotPod *v1.Pod
 				if cs.Spec.Lifecycle != nil && lifecycle.IsPodHooked(cs.Spec.Lifecycle.InPlaceUpdate, pod) {
-					if updated, err = c.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingUpdate); err == nil && updated {
-						clonesetutils.ResourceVersionExpectations.Expect(pod)
+					if updated, gotPod, err = c.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStatePreparingUpdate); err == nil && updated {
+						clonesetutils.ResourceVersionExpectations.Expect(gotPod)
 						klog.V(3).Infof("CloneSet %s update pod %s lifecycle to PreparingUpdate",
 							clonesetutils.GetControllerKey(cs), pod.Name)
 					}
@@ -228,12 +229,13 @@ func (c *realControl) updatePod(cs *appsv1alpha1.CloneSet, coreControl clonesetc
 			case appspub.LifecycleStateUpdated:
 				var err error
 				var updated bool
+				var gotPod *v1.Pod
 				var inPlaceUpdateHandler *appspub.LifecycleHook
 				if cs.Spec.Lifecycle != nil {
 					inPlaceUpdateHandler = cs.Spec.Lifecycle.InPlaceUpdate
 				}
-				if updated, err = c.lifecycleControl.UpdatePodLifecycleWithHandler(pod, appspub.LifecycleStatePreparingUpdate, inPlaceUpdateHandler); err == nil && updated {
-					clonesetutils.ResourceVersionExpectations.Expect(pod)
+				if updated, gotPod, err = c.lifecycleControl.UpdatePodLifecycleWithHandler(pod, appspub.LifecycleStatePreparingUpdate, inPlaceUpdateHandler); err == nil && updated {
+					clonesetutils.ResourceVersionExpectations.Expect(gotPod)
 					klog.V(3).Infof("CloneSet %s update pod %s lifecycle to PreparingUpdate",
 						clonesetutils.GetControllerKey(cs), pod.Name)
 				}

diff --git a/pkg/controller/cloneset/sync/cloneset_update_test.go b/pkg/controller/cloneset/sync/cloneset_update_test.go
@@ -574,7 +574,7 @@ func TestUpdate(t *testing.T) {
 		fakeClient := fake.NewClientBuilder().WithObjects(initialObjs...).Build()
 		ctrl := &realControl{
 			fakeClient,
-			lifecycle.NewForTest(fakeClient),
+			lifecycle.New(fakeClient),
 			inplaceupdate.NewForTest(fakeClient, clonesetutils.RevisionAdapterImpl, func() metav1.Time { return now }),
 			record.NewFakeRecorder(10),
 			controllerfinder.NewControllerFinder(fakeClient),

diff --git a/pkg/controller/cloneset/utils/cloneset_utils.go b/pkg/controller/cloneset/utils/cloneset_utils.go
@@ -24,6 +24,7 @@ import (
 
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
 	"github.com/openkruise/kruise/pkg/features"
+	utilclient "github.com/openkruise/kruise/pkg/util/client"
 	"github.com/openkruise/kruise/pkg/util/expectations"
 	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
 	apps "k8s.io/api/apps/v1"
@@ -84,7 +85,7 @@ func GetControllerKey(cs *appsv1alpha1.CloneSet) string {
 // GetActivePods returns all active pods in this namespace.
 func GetActivePods(reader client.Reader, opts *client.ListOptions) ([]*v1.Pod, error) {
 	podList := &v1.PodList{}
-	if err := reader.List(context.TODO(), podList, opts); err != nil {
+	if err := reader.List(context.TODO(), podList, opts, utilclient.DisableDeepCopy); err != nil {
 		return nil, err
 	}
 

diff --git a/pkg/controller/sidecarset/sidecarset_processor.go b/pkg/controller/sidecarset/sidecarset_processor.go
@@ -26,6 +26,7 @@ import (
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
 	"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
 	"github.com/openkruise/kruise/pkg/util"
+	utilclient "github.com/openkruise/kruise/pkg/util/client"
 	"github.com/openkruise/kruise/pkg/util/expectations"
 	historyutil "github.com/openkruise/kruise/pkg/util/history"
 	webhookutil "github.com/openkruise/kruise/pkg/webhook/util"
@@ -293,7 +294,7 @@ func (p *Processor) getSelectedPods(namespaces []string, selector labels.Selecto
 	for _, ns := range namespaces {
 		allPods := &corev1.PodList{}
 		listOpts.Namespace = ns
-		if listErr := p.Client.List(context.TODO(), allPods, listOpts); listErr != nil {
+		if listErr := p.Client.List(context.TODO(), allPods, listOpts, utilclient.DisableDeepCopy); listErr != nil {
 			err = fmt.Errorf("sidecarSet list pods by ns error, ns[%s], err:%v", ns, listErr)
 			return
 		}