Improve Kruise E2E image registry and fix image pulling (#894)

Signed-off-by: FillZpp <[email protected]>
openkruise · Jan 19, 2022 · af3e254 · af3e254
1 parent 2a7eb6d
commit af3e254
Show file tree

Hide file tree

Showing 44 changed files with 921 additions and 869 deletions.
diff --git a/.github/workflows/e2e-1.16.yaml b/.github/workflows/e2e-1.16.yaml
@@ -10,7 +10,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.16'
+  GO_VERSION: '1.17'
   KIND_IMAGE: 'kindest/node:v1.16.15'
   KIND_CLUSTER_NAME: 'ci-testing'
 
@@ -70,7 +70,6 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] StatefulSet' test/e2e
 
   pullimages-containerrecreate:
@@ -127,7 +126,6 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] (PullImage|ContainerRecreateRequest)' test/e2e
 
   advanced-daemonset:
@@ -184,7 +182,6 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] DaemonSet' test/e2e
 
   sidecarset:
@@ -241,7 +238,6 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] SidecarSet' test/e2e
 
   other:
@@ -298,7 +294,6 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           set +e
           ./bin/ginkgo -timeout 60m -v --skip='\[apps\] (StatefulSet|PullImage|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' test/e2e
 #          retVal=$?

diff --git a/.github/workflows/e2e-1.18.yaml b/.github/workflows/e2e-1.18.yaml
@@ -10,7 +10,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.16'
+  GO_VERSION: '1.17'
   KIND_IMAGE: 'kindest/node:v1.18.0'
   KIND_CLUSTER_NAME: 'ci-testing'
 
@@ -69,5 +69,4 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] EphemeralJob' test/e2e
diff --git a/.github/workflows/e2e-1.22.yaml b/.github/workflows/e2e-1.22.yaml
@@ -10,7 +10,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.16'
+  GO_VERSION: '1.17'
   KIND_IMAGE: 'kindest/node:v1.22.0'
   KIND_CLUSTER_NAME: 'ci-testing'
 
@@ -70,7 +70,6 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] StatefulSet' test/e2e
 
   pullimages-containerrecreate:
@@ -127,7 +126,6 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] (PullImage|ContainerRecreateRequest)' test/e2e
 
   advanced-daemonset:
@@ -184,7 +182,6 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] DaemonSet' test/e2e
 
   sidecarset:
@@ -241,7 +238,6 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] SidecarSet' test/e2e
 
   other:
@@ -298,7 +294,6 @@ jobs:
         run: |
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
-          ./scripts/generate_bindata.sh
           set +e
           ./bin/ginkgo -timeout 60m -v --skip='\[apps\] (StatefulSet|PullImage|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' test/e2e
 #          retVal=$?

diff --git a/Makefile b/Makefile
@@ -27,7 +27,6 @@ go_check:
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	@scripts/generate_client.sh
 	@scripts/generate_openapi.sh
-	@scripts/generate_bindata.sh
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./apis/..."
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
@@ -121,7 +120,7 @@ TMP_DIR=$$(mktemp -d) ;\
 cd $$TMP_DIR ;\
 go mod init tmp ;\
 echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
+GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
 rm -rf $$TMP_DIR ;\
 mv $(PROJECT_DIR)/bin/$(3) $(1);\
 }

diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -45,10 +45,10 @@ spec:
         resources:
           limits:
             cpu: 100m
-            memory: 50Mi
+            memory: 200Mi
           requests:
             cpu: 100m
-            memory: 50Mi
+            memory: 200Mi
       terminationGracePeriodSeconds: 10
 ---
 apiVersion: v1
@@ -108,7 +108,7 @@ spec:
         resources:
           limits:
             cpu: 100m
-            memory: 50Mi
+            memory: 100Mi
           requests:
             cpu: "0"
             memory: "0"

diff --git a/go.mod b/go.mod
@@ -1,6 +1,6 @@
 module github.com/openkruise/kruise
 
-go 1.15
+go 1.16
 
 require (
 	github.com/alibaba/pouch v0.0.0-20190328125340-37051654f368

diff --git a/pkg/controller/imagepulljob/imagepulljob_event_handler.go b/pkg/controller/imagepulljob/imagepulljob_event_handler.go
@@ -18,9 +18,9 @@ package imagepulljob
 
 import (
 	"reflect"
-	"strings"
 
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
+	daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
 	utilimagejob "github.com/openkruise/kruise/pkg/util/imagejob"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -106,11 +106,13 @@ func (e *nodeImageEventHandler) handleUpdate(nodeImage, oldNodeImage *appsv1alph
 	}
 	diffSet := diffJobs(newJobs, oldJobs)
 	for _, j := range newJobs {
-		for _, cImage := range changedImages.List() {
-			if j.Spec.Image == cImage || strings.HasPrefix(j.Spec.Image, cImage+":") {
-				diffSet[types.NamespacedName{Namespace: j.Namespace, Name: j.Name}] = struct{}{}
-				break
-			}
+		imageName, _, err := daemonutil.NormalizeImageRefToNameTag(j.Spec.Image)
+		if err != nil {
+			klog.Warningf("Invalid image %s in job %s/%s", j.Spec.Image, j.Namespace, j.Name)
+			continue
+		}
+		if changedImages.Has(imageName) {
+			diffSet[types.NamespacedName{Namespace: j.Namespace, Name: j.Name}] = struct{}{}
 		}
 	}
 	for name := range diffSet {

diff --git a/pkg/daemon/criruntime/factory.go b/pkg/daemon/criruntime/factory.go
@@ -24,7 +24,6 @@ import (
 
 	runtimeimage "github.com/openkruise/kruise/pkg/daemon/criruntime/imageruntime"
 	daemonutil "github.com/openkruise/kruise/pkg/daemon/util"
-	"google.golang.org/grpc"
 	criapi "k8s.io/cri-api/pkg/apis"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 	"k8s.io/klog/v2"
@@ -98,14 +97,12 @@ func NewFactory(varRunPath string, accountManager daemonutil.ImagePullAccountMan
 				continue
 			}
 		case ContainerRuntimeContainerd:
-			var conn *grpc.ClientConn
-			addr, _, _ := kubeletutil.GetAddressAndDialer(cfg.runtimeRemoteURI)
-			conn, err = getContainerdConn(addr)
+			addr, _, err := kubeletutil.GetAddressAndDialer(cfg.runtimeRemoteURI)
 			if err != nil {
-				klog.Warningf("Failed to get connection for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err)
+				klog.Warningf("Failed to get address for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err)
 				continue
 			}
-			imageService, err = runtimeimage.NewContainerdImageService(conn, accountManager)
+			imageService, err = runtimeimage.NewContainerdImageService(addr, accountManager)
 			if err != nil {
 				klog.Warningf("Failed to new image service for %v (%s, %s): %v", cfg.runtimeType, cfg.runtimeURI, cfg.runtimeRemoteURI, err)
 				continue

diff --git a/pkg/daemon/criruntime/imageruntime/containerd.go b/pkg/daemon/criruntime/imageruntime/containerd.go
@@ -54,15 +54,15 @@ const (
 
 // NewContainerdImageService returns containerd-type ImageService
 func NewContainerdImageService(
-	conn *grpc.ClientConn,
+	runtimeURI string,
 	accountManager daemonutil.ImagePullAccountManager,
 ) (ImageService, error) {
-	snapshotter, httpProxy, err := getDefaultValuesFromCRIStatus(conn)
+	client, err := containerd.New(runtimeURI)
 	if err != nil {
 		return nil, err
 	}
-
-	client, err := containerd.NewWithConn(conn)
+	conn := client.Conn()
+	snapshotter, httpProxy, err := getDefaultValuesFromCRIStatus(conn)
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/daemon/criruntime/util.go b/pkg/daemon/criruntime/util.go
diff --git a/scripts/generate_bindata.sh b/scripts/generate_bindata.sh
diff --git a/test/e2e/apps/broadcastjob.go b/test/e2e/apps/broadcastjob.go
@@ -82,7 +82,7 @@ var _ = SIGDescribe("BroadcastJob", func() {
 
 	framework.KruiseDescribe("BroadcastJob dispatching", func() {
 
-		ginkgo.It("succeeds for parallelism < number of node", func() {
+		framework.ConformanceIt("succeeds for parallelism < number of node", func() {
 			ginkgo.By("Create Fake Node " + randStr)
 			fakeNode, err := nodeTester.CreateFakeNode(randStr)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
@@ -96,7 +96,7 @@ var _ = SIGDescribe("BroadcastJob", func() {
 							Tolerations: []v1.Toleration{{Key: framework.E2eFakeKey, Operator: v1.TolerationOpEqual, Value: randStr, Effect: v1.TaintEffectNoSchedule}},
 							Containers: []v1.Container{{
 								Name:    "box",
-								Image:   "busybox:latest",
+								Image:   BusyboxImage,
 								Command: []string{"/bin/sh", "-c", "sleep 5"},
 							}},
 							RestartPolicy: v1.RestartPolicyNever,