feat: add nodepool e2e

hack/make-rules/local-up-openyurt.sh

@@ -62,7 +62,7 @@ readonly LOCAL_ARCH=$(go env GOHOSTARCH)
 readonly LOCAL_OS=$(go env GOHOSTOS)
 readonly CLUSTER_NAME="openyurt-e2e-test"
 readonly KUBERNETESVERSION=${KUBERNETESVERSION:-"v1.22"}
-readonly NODES_NUM=${NODES_NUM:-2}
+readonly NODES_NUM=${NODES_NUM:-3}
 readonly KIND_KUBECONFIG=${KIND_KUBECONFIG:-${HOME}/.kube/config}
 readonly DISABLE_DEFAULT_CNI=${DISABLE_DEFAULT_CNI:-"false"}
 ENABLE_DUMMY_IF=true

test/e2e/util/nodepool.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2022 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/openyurtio/openyurt/pkg/apis/apps"
+	"github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1"
+)
+
+func CleanupNodePool(ctx context.Context, k8sClient client.Client) error {
+	nps := &v1beta1.NodePoolList{}
+	if err := k8sClient.List(ctx, nps); err != nil {
+		return err
+	}
+	for _, tmp := range nps.Items {
+		if err := k8sClient.Delete(ctx, &tmp); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func CleanupNodePoolLabel(ctx context.Context, k8sClient client.Client) error {
+	nodes := &corev1.NodeList{}
+	if err := k8sClient.List(ctx, nodes); err != nil {
+		return err
+	}
+
+	for _, originNode := range nodes.Items {
+		labelDeleted := false
+		newNode := originNode.DeepCopy()
+		if newNode.Labels != nil {
+			for k := range newNode.Labels {
+				if k == apps.LabelDesiredNodePool {
+					delete(newNode.Labels, apps.LabelDesiredNodePool)
+					labelDeleted = true
+				}
+			}
+		}
+		if labelDeleted {
+			if err := k8sClient.Patch(context.TODO(), newNode, client.MergeFrom(&originNode)); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func InitNodeAndNodePool(ctx context.Context, k8sClient client.Client, poolToNodesMap map[string]sets.String) error {
+	nodeToPoolMap := make(map[string]string)
+	for k, v := range poolToNodesMap {
+		for _, n := range v.List() {
+			nodeToPoolMap[n] = k
+		}
+	}
+
+	for k := range poolToNodesMap {
+		if err := k8sClient.Create(ctx, &v1beta1.NodePool{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: k,
+			},
+			Spec: v1beta1.NodePoolSpec{
+				Type: v1beta1.Edge,
+			}}); err != nil {
+			return err
+		}
+	}
+
+	nodes := &corev1.NodeList{}
+	if err := k8sClient.List(ctx, nodes); err != nil {
+		return err
+	}
+
+	for _, originNode := range nodes.Items {
+		newNode := originNode.DeepCopy()
+		nodeLabels := newNode.Labels
+		if nodeLabels == nil {
+			nodeLabels = map[string]string{}
+		}
+
+		if _, ok := nodeToPoolMap[originNode.Name]; !ok {
+			continue
+		}
+
+		nodeLabels[apps.LabelDesiredNodePool] = nodeToPoolMap[originNode.Name]
+		newNode.Labels = nodeLabels
+		if err := k8sClient.Patch(ctx, newNode, client.MergeFrom(&originNode)); err != nil {
+			return err
+		}
+	}
+	return nil
+}

test/e2e/util/util.go

@@ -23,15 +23,31 @@ import (
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1"
+	appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1"
 	"github.com/openyurtio/openyurt/test/e2e/yurtconfig"
 )
 
+var (
+	scheme = runtime.NewScheme()
+)
+
+func init() {
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(appsv1alpha1.AddToScheme(scheme))
+	utilruntime.Must(appsv1beta1.AddToScheme(scheme))
+}
+
 const (
 	// DefaultNamespaceDeletionTimeout is timeout duration for waiting for a namespace deletion.
 	DefaultNamespaceDeletionTimeout = 5 * time.Minute
@@ -88,12 +104,18 @@ func WaitForNamespacesDeleted(c clientset.Interface, namespaces []string, timeou
 
 // SetYurtE2eCfg for e2e-tests
 func SetYurtE2eCfg() error {
-	config, client, err := LoadRestConfigAndClientset(*Kubeconfig)
+	config, kubeClient, err := LoadRestConfigAndClientset(*Kubeconfig)
 	if err != nil {
 		klog.Errorf("pre_check_load_client_set failed errmsg:%v", err)
 		return err
 	}
-	yurtconfig.YurtE2eCfg.KubeClient = client
+	yurtconfig.YurtE2eCfg.KubeClient = kubeClient
+
+	runtimeClient, err := client.New(config, client.Options{Scheme: scheme})
+	if err != nil {
+		return err
+	}
+	yurtconfig.YurtE2eCfg.RuntimeClient = runtimeClient
 	yurtconfig.YurtE2eCfg.RestConfig = config
 	return nil
 }

test/e2e/yurt/nodepool.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2022 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package yurt
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/util/sets"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1"
+	"github.com/openyurtio/openyurt/test/e2e/util"
+	ycfg "github.com/openyurtio/openyurt/test/e2e/yurtconfig"
+)
+
+var _ = Describe("nodepool test", func() {
+	ctx := context.Background()
+	var k8sClient runtimeclient.Client
+	poolToNodesMap := make(map[string]sets.String)
+
+	checkNodePoolStatus := func(poolToNodesMap map[string]sets.String) error {
+		nps := &v1beta1.NodePoolList{}
+		if err := k8sClient.List(ctx, nps); err != nil {
+			return err
+		}
+		for _, tmp := range nps.Items {
+			if int(tmp.Status.ReadyNodeNum) != poolToNodesMap[tmp.Name].Len() {
+				return errors.New("nodepool size not match")
+			}
+		}
+		return nil
+	}
+
+	BeforeEach(func() {
+		By("Start to run nodepool test, cleanup previous resources")
+		k8sClient = ycfg.YurtE2eCfg.RuntimeClient
+		poolToNodesMap = map[string]sets.String{}
+
+		util.CleanupNodePoolLabel(ctx, k8sClient)
+		util.CleanupNodePool(ctx, k8sClient)
+	})
+
+	AfterEach(func() {
+		By("Cleanup resources after test")
+		util.CleanupNodePoolLabel(ctx, k8sClient)
+		util.CleanupNodePool(ctx, k8sClient)
+	})
+
+	It("Test NodePool empty", func() {
+		By("Run noolpool empty")
+		Eventually(
+			func() error {
+				return util.InitNodeAndNodePool(ctx, k8sClient, poolToNodesMap)
+			},
+			time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil()))
+
+		Eventually(
+			func() error {
+				return checkNodePoolStatus(poolToNodesMap)
+			},
+			time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil()))
+	})
+
+	It("Test NodePool create", func() {
+		By("Run nodepool create")
+
+		bjNpName := "beijing"
+		poolToNodesMap[bjNpName] = sets.NewString("openyurt-e2e-test-worker", "openyurt-e2e-test-worker2")
+		Eventually(
+			func() error {
+				return util.InitNodeAndNodePool(ctx, k8sClient, poolToNodesMap)
+			},
+			time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil()))
+
+		Eventually(
+			func() error {
+				return checkNodePoolStatus(poolToNodesMap)
+			},
+			time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil()))
+	})
+
+	It(" Test Multiple NodePools With Nodes", func() {
+		poolToNodesMap["beijing"] = sets.NewString("openyurt-e2e-test-worker")
+		poolToNodesMap["hangzhou"] = sets.NewString("openyurt-e2e-test-worker2")
+
+		Eventually(
+			func() error {
+				return util.InitNodeAndNodePool(ctx, k8sClient, poolToNodesMap)
+			},
+			time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil()))
+
+		Eventually(
+			func() error {
+				return checkNodePoolStatus(poolToNodesMap)
+			},
+			time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil()))
+	})
+
+})

test/e2e/yurtconfig/yurtconfig.go

@@ -19,11 +19,13 @@ package yurtconfig
 import (
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type YurtE2eConfig struct {
-	KubeClient *clientset.Clientset
-	RestConfig *restclient.Config
+	KubeClient    *clientset.Clientset
+	RuntimeClient client.Client
+	RestConfig    *restclient.Config
 }
 
 var YurtE2eCfg YurtE2eConfig