From 41894d9835aa884152b7669d75e7e0beab9ff067 Mon Sep 17 00:00:00 2001 From: huiwq1990 Date: Tue, 11 Apr 2023 09:45:15 +0800 Subject: [PATCH] feat: add nodepool e2e Signed-off-by: huiwq1990 --- hack/make-rules/local-up-openyurt.sh | 2 +- test/e2e/util/nodepool.go | 141 +++++++++++++++++++++++++++ test/e2e/util/util.go | 26 ++++- test/e2e/yurt/nodepool.go | 117 ++++++++++++++++++++++ test/e2e/yurt/yurt.go | 6 ++ test/e2e/yurtconfig/yurtconfig.go | 6 +- 6 files changed, 293 insertions(+), 5 deletions(-) create mode 100644 test/e2e/util/nodepool.go create mode 100644 test/e2e/yurt/nodepool.go diff --git a/hack/make-rules/local-up-openyurt.sh b/hack/make-rules/local-up-openyurt.sh index 179812f6a92..5181eec46f8 100755 --- a/hack/make-rules/local-up-openyurt.sh +++ b/hack/make-rules/local-up-openyurt.sh @@ -62,7 +62,7 @@ readonly LOCAL_ARCH=$(go env GOHOSTARCH) readonly LOCAL_OS=$(go env GOHOSTOS) readonly CLUSTER_NAME="openyurt-e2e-test" readonly KUBERNETESVERSION=${KUBERNETESVERSION:-"v1.22"} -readonly NODES_NUM=${NODES_NUM:-2} +readonly NODES_NUM=${NODES_NUM:-3} readonly KIND_KUBECONFIG=${KIND_KUBECONFIG:-${HOME}/.kube/config} readonly DISABLE_DEFAULT_CNI=${DISABLE_DEFAULT_CNI:-"false"} ENABLE_DUMMY_IF=true diff --git a/test/e2e/util/nodepool.go b/test/e2e/util/nodepool.go new file mode 100644 index 00000000000..4ad609356fe --- /dev/null +++ b/test/e2e/util/nodepool.go @@ -0,0 +1,141 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +func LabelNodeIsEdge(ctx context.Context, k8sClient client.Client) error { + nodes := &corev1.NodeList{} + if err := k8sClient.List(ctx, nodes); err != nil { + return err + } + + for _, originNode := range nodes.Items { + nodeLabels := originNode.Labels + if nodeLabels == nil { + nodeLabels = map[string]string{} + } + switch originNode.Name { + case "openyurt-e2e-test-control-plane": + nodeLabels["openyurt.io/is-edge-worker"] = "false" + default: + nodeLabels["openyurt.io/is-edge-worker"] = "true" + } + + newNode := originNode.DeepCopy() + newNode.Labels = nodeLabels + if err := k8sClient.Patch(ctx, newNode, client.MergeFrom(&originNode)); err != nil { + return err + } + } + return nil +} + +func CleanupNodePool(ctx context.Context, k8sClient client.Client) error { + nps := &v1beta1.NodePoolList{} + if err := k8sClient.List(ctx, nps); err != nil { + return err + } + for _, tmp := range nps.Items { + if err := k8sClient.Delete(ctx, &tmp); err != nil { + return err + } + } + return nil +} + +func CleanupNodeLabel(ctx context.Context, k8sClient client.Client) error { + nodes := &corev1.NodeList{} + if err := k8sClient.List(ctx, nodes); err != nil { + return err + } + + for _, originNode := range nodes.Items { + newNode := originNode.DeepCopy() + if newNode.Labels != nil { + for k := range newNode.Labels { + if k == apps.LabelDesiredNodePool { + delete(newNode.Labels, apps.LabelDesiredNodePool) + } + if k == "openyurt.io/is-edge-worker" { + delete(newNode.Labels, "openyurt.io/is-edge-worker") + } + } + } + + if err := k8sClient.Patch(context.TODO(), newNode, client.MergeFrom(&originNode)); err != nil { + return err + } + } + return nil +} + +func InitNodeAndNodePool(ctx context.Context, k8sClient client.Client, poolToNodesMap map[string]sets.String) error { + nodeToPoolMap := make(map[string]string) + for k, v := range poolToNodesMap { + for _, n := range v.List() { + nodeToPoolMap[n] = k + } + } + + for k := range poolToNodesMap { + if err := k8sClient.Create(ctx, &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: k, + }, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, + }}); err != nil { + return err + } + } + + nodes := &corev1.NodeList{} + if err := k8sClient.List(ctx, nodes); err != nil { + return err + } + + for _, originNode := range nodes.Items { + newNode := originNode.DeepCopy() + nodeLabels := newNode.Labels + if nodeLabels == nil { + nodeLabels = map[string]string{} + } + + if _, ok := nodeToPoolMap[originNode.Name]; !ok { + continue + } + + nodeLabels[apps.LabelDesiredNodePool] = nodeToPoolMap[originNode.Name] + nodeLabels["openyurt.io/is-edge-worker"] = "true" + newNode.Labels = nodeLabels + if err := k8sClient.Patch(ctx, newNode, client.MergeFrom(&originNode)); err != nil { + return err + } + } + return nil +} diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index 398b7a6d458..d64a3a329b7 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -23,15 +23,31 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" "github.com/openyurtio/openyurt/test/e2e/yurtconfig" ) +var ( + scheme = runtime.NewScheme() +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(appsv1alpha1.AddToScheme(scheme)) + utilruntime.Must(appsv1beta1.AddToScheme(scheme)) +} + const ( // DefaultNamespaceDeletionTimeout is timeout duration for waiting for a namespace deletion. DefaultNamespaceDeletionTimeout = 5 * time.Minute @@ -88,12 +104,18 @@ func WaitForNamespacesDeleted(c clientset.Interface, namespaces []string, timeou // SetYurtE2eCfg for e2e-tests func SetYurtE2eCfg() error { - config, client, err := LoadRestConfigAndClientset(*Kubeconfig) + config, kubeClient, err := LoadRestConfigAndClientset(*Kubeconfig) if err != nil { klog.Errorf("pre_check_load_client_set failed errmsg:%v", err) return err } - yurtconfig.YurtE2eCfg.KubeClient = client + yurtconfig.YurtE2eCfg.KubeClient = kubeClient + + runtimeClient, err := client.New(config, client.Options{Scheme: scheme}) + if err != nil { + return err + } + yurtconfig.YurtE2eCfg.RuntimeClient = runtimeClient yurtconfig.YurtE2eCfg.RestConfig = config return nil } diff --git a/test/e2e/yurt/nodepool.go b/test/e2e/yurt/nodepool.go new file mode 100644 index 00000000000..894bdcaa2cc --- /dev/null +++ b/test/e2e/yurt/nodepool.go @@ -0,0 +1,117 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yurt + +import ( + "context" + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/sets" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + "github.com/openyurtio/openyurt/test/e2e/util" + ycfg "github.com/openyurtio/openyurt/test/e2e/yurtconfig" +) + +var _ = Describe("nodepool test", func() { + ctx := context.Background() + var k8sClient runtimeclient.Client + poolToNodesMap := make(map[string]sets.String) + + checkNodePoolStatus := func(poolToNodesMap map[string]sets.String) error { + nps := &v1beta1.NodePoolList{} + if err := k8sClient.List(ctx, nps); err != nil { + return err + } + for _, tmp := range nps.Items { + if int(tmp.Status.ReadyNodeNum) != poolToNodesMap[tmp.Name].Len() { + return errors.New("nodepool size not match") + } + } + return nil + } + + BeforeEach(func() { + By("Start to run nodepool test, cleanup previous resources") + k8sClient = ycfg.YurtE2eCfg.RuntimeClient + poolToNodesMap = map[string]sets.String{} + + util.CleanupNodeLabel(ctx, k8sClient) + util.CleanupNodePool(ctx, k8sClient) + }) + + AfterEach(func() { + By("Cleanup resources after test") + util.CleanupNodeLabel(ctx, k8sClient) + util.CleanupNodePool(ctx, k8sClient) + }) + + It("Test NodePool empty", func() { + By("Run noolpool empty") + Eventually( + func() error { + return util.InitNodeAndNodePool(ctx, k8sClient, poolToNodesMap) + }, + time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil())) + + Eventually( + func() error { + return checkNodePoolStatus(poolToNodesMap) + }, + time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil())) + }) + + It("Test NodePool create", func() { + By("Run nodepool create") + + bjNpName := "beijing" + poolToNodesMap[bjNpName] = sets.NewString("openyurt-e2e-test-worker", "openyurt-e2e-test-worker2") + Eventually( + func() error { + return util.InitNodeAndNodePool(ctx, k8sClient, poolToNodesMap) + }, + time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil())) + + Eventually( + func() error { + return checkNodePoolStatus(poolToNodesMap) + }, + time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil())) + }) + + It("Test NodePool With Multi Nodes", func() { + poolToNodesMap["beijing"] = sets.NewString("openyurt-e2e-test-worker") + poolToNodesMap["hangzhou"] = sets.NewString("openyurt-e2e-test-worker2") + + Eventually( + func() error { + return util.InitNodeAndNodePool(ctx, k8sClient, poolToNodesMap) + }, + time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil())) + + Eventually( + func() error { + return checkNodePoolStatus(poolToNodesMap) + }, + time.Second*5, time.Millisecond*500).Should(SatisfyAny(BeNil())) + }) + +}) diff --git a/test/e2e/yurt/yurt.go b/test/e2e/yurt/yurt.go index 52350acc45c..54a3abbbc14 100644 --- a/test/e2e/yurt/yurt.go +++ b/test/e2e/yurt/yurt.go @@ -17,6 +17,8 @@ limitations under the License. package yurt import ( + "context" + "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" apiv1 "k8s.io/api/core/v1" @@ -26,6 +28,7 @@ import ( "github.com/openyurtio/openyurt/test/e2e/common/ns" p "github.com/openyurtio/openyurt/test/e2e/common/pod" "github.com/openyurtio/openyurt/test/e2e/constants" + "github.com/openyurtio/openyurt/test/e2e/util" "github.com/openyurtio/openyurt/test/e2e/yurtconfig" ) @@ -57,6 +60,9 @@ var _ = ginkgo.Describe(constants.YurtE2ENamespaceName, ginkgo.Ordered, ginkgo.L }) ginkgo.Describe(constants.YurtE2ETestDesc+": pod_operate_test", func() { + ginkgo.It("mark nodes edge label", func() { + util.LabelNodeIsEdge(context.TODO(), yurtconfig.YurtE2eCfg.RuntimeClient) + }) ginkgo.It("pod_operate", func() { cs := yurtconfig.YurtE2eCfg.KubeClient podName := "yurt-test-busybox" diff --git a/test/e2e/yurtconfig/yurtconfig.go b/test/e2e/yurtconfig/yurtconfig.go index a27ceaaaf47..7d94eff6a22 100644 --- a/test/e2e/yurtconfig/yurtconfig.go +++ b/test/e2e/yurtconfig/yurtconfig.go @@ -19,11 +19,13 @@ package yurtconfig import ( clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" ) type YurtE2eConfig struct { - KubeClient *clientset.Clientset - RestConfig *restclient.Config + KubeClient *clientset.Clientset + RuntimeClient client.Client + RestConfig *restclient.Config } var YurtE2eCfg YurtE2eConfig