From f5009a702c0bcb8271a1d2f98f543fd030421d9d Mon Sep 17 00:00:00 2001 From: Amulyam24 Date: Tue, 11 Jan 2022 11:41:05 +0530 Subject: [PATCH] Adding e2e test cases --- Makefile | 21 +- test/e2e/config/ibmcloud-e2e-powervs.yaml | 68 ++++++ ...bmcloud-e2e.yaml => ibmcloud-e2e-vpc.yaml} | 12 +- test/e2e/e2e_test.go | 231 ++++++++++++++++++ test/e2e/suite_test.go | 6 + 5 files changed, 329 insertions(+), 9 deletions(-) create mode 100644 test/e2e/config/ibmcloud-e2e-powervs.yaml rename test/e2e/config/{ibmcloud-e2e.yaml => ibmcloud-e2e-vpc.yaml} (84%) create mode 100644 test/e2e/e2e_test.go diff --git a/Makefile b/Makefile index 9351daf6b..73c54a610 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,6 @@ ARTIFACTS ?= $(REPO_ROOT)/_artifacts TOOLS_DIR := hack/tools TOOLS_BIN_DIR := $(TOOLS_DIR)/bin GO_INSTALL = ./scripts/go_install.sh -E2E_CONF_FILE ?= $(REPO_ROOT)/test/e2e/config/ibmcloud-e2e.yaml E2E_CONF_FILE_ENVSUBST := $(REPO_ROOT)/test/e2e/config/ibmcloud-e2e-envsubst.yaml GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint @@ -135,6 +134,14 @@ generate-go: $(MOCKGEN) images: docker-build +set-flavor: +ifeq ($(E2E_FLAVOR), vpc) + $(eval E2E_CONF_FILE=$(REPO_ROOT)/test/e2e/config/ibmcloud-e2e-vpc.yaml) +else + $(eval E2E_CONF_FILE=$(REPO_ROOT)/test/e2e/config/ibmcloud-e2e-powervs.yaml) +endif + @echo "Setting e2e test flavour to ${E2E_CONF_FILE}" + ## -------------------------------------- ## Linting ## -------------------------------------- @@ -155,6 +162,7 @@ test: generate fmt vet manifests GINKGO_FOCUS ?= Workload cluster creation GINKGO_NODES ?= 3 GINKGO_NOCOLOR ?= false +E2E_FLAVOR ?= powervs GINKGO_ARGS ?= -v -trace -progress -v -tags=e2e -focus=$(GINKGO_FOCUS) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ARTIFACTS ?= $(REPO_ROOT)/_artifacts SKIP_CLEANUP ?= false @@ -162,13 +170,14 @@ SKIP_CREATE_MGMT_CLUSTER ?= false #Run the end-to-end tests .PHONY: test-e2e -test-e2e: $(KUBECTL) $(GINKGO) $(ENVSUBST) e2e-image +test-e2e: $(KUBECTL) $(GINKGO) $(ENVSUBST) set-flavor e2e-image $(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) $(GINKGO) $(GINKGO_ARGS) ./test/e2e -- \ - -e2e.artifacts-folder="$(ARTIFACTS)" \ - -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ - -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) \ - -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) + -e2e.artifacts-folder="$(ARTIFACTS)" \ + -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ + -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) \ + -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) \ + -e2e.flavor="$(E2E_FLAVOR)" ## -------------------------------------- ## Docker diff --git a/test/e2e/config/ibmcloud-e2e-powervs.yaml b/test/e2e/config/ibmcloud-e2e-powervs.yaml new file mode 100644 index 000000000..9f76cb362 --- /dev/null +++ b/test/e2e/config/ibmcloud-e2e-powervs.yaml @@ -0,0 +1,68 @@ +managementClusterName: capi-ibmcloud-e2e + +images: + # Use local built images for e2e tests + - name: gcr.io/k8s-staging-capi-ibmcloud/cluster-api-ibmcloud-controller:e2e + loadBehavior: mustLoad + +providers: + - name: cluster-api + type: CoreProvider + versions: + - name: v1.0.2 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.2/core-components.yaml + type: url + files: + - sourcePath: "${PWD}/test/e2e/data/shared/metadata.yaml" + - name: kubeadm + type: BootstrapProvider + versions: + - name: v1.0.2 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.2/bootstrap-components.yaml + type: url + files: + - sourcePath: "${PWD}/test/e2e/data/shared/metadata.yaml" + - name: kubeadm + type: ControlPlaneProvider + versions: + - name: v1.0.2 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.2/control-plane-components.yaml + type: url + files: + - sourcePath: "${PWD}/test/e2e/data/shared/metadata.yaml" + - name: ibmcloud + type: InfrastructureProvider + versions: + - name: v0.2.0 + value: "${PWD}/config/default" + files: + - sourcePath: "${PWD}/metadata.yaml" + targetName: "metadata.yaml" + - sourcePath: "${PWD}/templates/cluster-template-powervs.yaml" + targetName: "cluster-template-powervs.yaml" + +variables: + KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.22.4}" + # Cluster Addons + CNI: "${PWD}/test/e2e/data/cni/calico/calico.yaml" + IP_FAMILY: "IPv4" + # Following variables should be set based on the flavour being tested + IBMPOWERVS_SSHKEY_NAME: "${IBMPOWERVS_SSHKEY_NAME:-}" + IBMPOWERVS_VIP: "${IBMPOWERVS_VIP:-}" + IBMPOWERVS_VIP_EXTERNAL: "${IBMPOWERVS_VIP_EXTERNAL:-}" + IBMPOWERVS_VIP_CIDR: "${IBMPOWERVS_VIP_CIDR:-}" + IBMPOWERVS_IMAGE_NAME: "${IBMPOWERVS_IMAGE_NAME:-}" + IBMPOWERVS_SERVICE_INSTANCE_ID: "${IBMPOWERVS_SERVICE_INSTANCE_ID:-}" + IBMPOWERVS_NETWORK_NAME: "${IBMPOWERVS_NETWORK_NAME:-}" + +intervals: + default/wait-controllers: ["3m", "10s"] + default/wait-cluster: ["20m", "10s"] + default/wait-control-plane: ["30m", "10s"] + default/wait-worker-nodes: ["30m", "10s"] + default/wait-delete-cluster: ["20m", "10s"] + default/wait-machine-upgrade: ["50m", "10s"] + default/wait-machine-remediation: ["30m", "10s"] + default/wait-deployment: ["5m", "10s"] + default/wait-job: ["5m", "10s"] + default/wait-service: ["3m", "10s"] diff --git a/test/e2e/config/ibmcloud-e2e.yaml b/test/e2e/config/ibmcloud-e2e-vpc.yaml similarity index 84% rename from test/e2e/config/ibmcloud-e2e.yaml rename to test/e2e/config/ibmcloud-e2e-vpc.yaml index 43898f132..4bf1deecb 100644 --- a/test/e2e/config/ibmcloud-e2e.yaml +++ b/test/e2e/config/ibmcloud-e2e-vpc.yaml @@ -38,8 +38,6 @@ providers: files: - sourcePath: "${PWD}/metadata.yaml" targetName: "metadata.yaml" - - sourcePath: "${PWD}/templates/cluster-template-powervs.yaml" - targetName: "cluster-template-powervs.yaml" - sourcePath: "${PWD}/templates/cluster-template.yaml" targetName: "cluster-template-vpc.yaml" @@ -48,7 +46,15 @@ variables: # Cluster Addons CNI: "${PWD}/test/e2e/data/cni/calico/calico.yaml" IP_FAMILY: "IPv4" - + # Following variables should be set based on the flavour being tested + IBMVPC_REGION: "${IBMVPC_REGION:-}" + IBMVPC_ZONE: "${IBMVPC_ZONE:-}" + IBMVPC_RESOURCEGROUP: "${IBMVPC_RESOURCEGROUP:-}" + IBMVPC_NAME: "${IBMVPC_NAME:-}" + IBMVPC_IMAGE_ID: "${IBMVPC_IMAGE_ID:-}" + IBMVPC_PROFILE: "${IBMVPC_PROFILE:-}" + IBMVPC_SSHKEY_ID: "${IBMVPC_SSHKEY_ID:-}" + intervals: default/wait-controllers: ["3m", "10s"] default/wait-cluster: ["20m", "10s"] diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 000000000..6ed102e7c --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,231 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" +) + +var _ = Describe("Workload cluster creation", func() { + var ( + ctx = context.TODO() + specName = "create-workload-cluster" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + result *clusterctl.ApplyClusterTemplateAndWaitResult + clusterName string + clusterctlLogFolder string + cniPath string + ) + + BeforeEach(func() { + Expect(e2eConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName) + Expect(clusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. clusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(bootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. bootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid argument. artifactFolder can't be created for %s spec", specName) + + Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) + + clusterName = fmt.Sprintf("capi-ibmcloud-e2e-%s", util.RandomString(6)) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + + result = new(clusterctl.ApplyClusterTemplateAndWaitResult) + + // We need to override clusterctl apply log folder to avoid getting our credentials exposed. + clusterctlLogFolder = filepath.Join(os.TempDir(), "clusters", bootstrapClusterProxy.GetName()) + + // Path to the CNI file is defined in the config + Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath), "Missing %s variable in the config", capi_e2e.CNIPath) + cniPath = e2eConfig.GetVariable(capi_e2e.CNIPath) + }) + + AfterEach(func() { + cleanInput := cleanupInput{ + SpecName: specName, + Cluster: result.Cluster, + ClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + CancelWatches: cancelWatches, + IntervalsGetter: e2eConfig.GetIntervals, + SkipCleanup: skipCleanup, + ArtifactFolder: artifactFolder, + } + + dumpSpecResourcesAndCleanup(ctx, cleanInput) + }) + + Context("Creating a single control-plane cluster", func() { + It("Should create a cluster with 1 worker node and can be scaled", func() { + By("Initializing with 1 worker node") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: flavor, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + CNIManifestPath: cniPath, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + + By("Scaling worker node to 3") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: flavor, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(3), + }, + CNIManifestPath: cniPath, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + }) + + Context("Creating a highly available control-plane cluster", func() { + It("Should create a cluster with 3 control-plane nodes and 1 worker node", func() { + By("Creating a high available cluster") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: flavor, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(3), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + CNIManifestPath: cniPath, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + }) +}) + +func Byf(format string, a ...interface{}) { + By(fmt.Sprintf(format, a...)) +} + +type cleanupInput struct { + SpecName string + ClusterProxy framework.ClusterProxy + ArtifactFolder string + Namespace *corev1.Namespace + CancelWatches context.CancelFunc + Cluster *clusterv1.Cluster + IntervalsGetter func(spec, key string) []interface{} + SkipCleanup bool + AdditionalCleanup func() +} + +func setupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) { + Byf("Creating a namespace for hosting the %q test spec", specName) + namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + Creator: clusterProxy.GetClient(), + ClientSet: clusterProxy.GetClientSet(), + Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), + LogFolder: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName()), + }) + + return namespace, cancelWatches +} + +func dumpSpecResourcesAndCleanup(ctx context.Context, input cleanupInput) { + defer func() { + input.CancelWatches() + }() + + if input.Cluster == nil { + By("Unable to dump workload cluster logs as the cluster is nil") + } else { + Byf("Dumping logs from the %q workload cluster", input.Cluster.Name) + input.ClusterProxy.CollectWorkloadClusterLogs(ctx, input.Cluster.Namespace, input.Cluster.Name, filepath.Join(input.ArtifactFolder, "clusters", input.Cluster.Name)) + } + + Byf("Dumping all the Cluster API resources in the %q namespace", input.Namespace.Name) + // Dump all Cluster API related resources to artifacts before deleting them. + framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{ + Lister: input.ClusterProxy.GetClient(), + Namespace: input.Namespace.Name, + LogPath: filepath.Join(input.ArtifactFolder, "clusters", input.ClusterProxy.GetName(), "resources"), + }) + + if input.SkipCleanup { + return + } + + Byf("Deleting all clusters in the %s namespace", input.Namespace.Name) + framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{ + Client: input.ClusterProxy.GetClient(), + Namespace: input.Namespace.Name, + }, input.IntervalsGetter(input.SpecName, "wait-delete-cluster")...) + + Byf("Deleting namespace used for hosting the %q test spec", input.SpecName) + framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ + Deleter: input.ClusterProxy.GetClient(), + Name: input.Namespace.Name, + }) + + if input.AdditionalCleanup != nil { + Byf("Running additional cleanup for the %q test spec", input.SpecName) + input.AdditionalCleanup() + } +} diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index da6f32660..183b087f6 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -1,3 +1,4 @@ +//go:build e2e // +build e2e /* @@ -62,6 +63,9 @@ var ( // skipCleanup prevents cleanup of test resources e.g. for debug purposes. skipCleanup bool + + // flavor instructs the tests to be run on the passed infrastructure type + flavor string ) // Test suite global vars. @@ -86,6 +90,7 @@ func init() { flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") + flag.StringVar(&flavor, "e2e.flavor", "", "flavor of e2e tests to be run") } func TestE2E(t *testing.T) { @@ -102,6 +107,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { // Before all ParallelNodes. Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.") + Expect(flavor).ToNot(BeNil(), "Invalid test argument. e2e.flavor should not be nil") Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder) By("Initializing a runtime.Scheme with all the GVK relevant for this test")