From 170b69e4f305751c942a5b3e38a26ecfa1d219e2 Mon Sep 17 00:00:00 2001 From: Divyansh Kamboj Date: Thu, 1 Aug 2024 16:45:51 +0530 Subject: [PATCH] setup basic daemonset tests for ceph-csi-operator-system Signed-off-by: Divyansh Kamboj --- Makefile | 4 + config/manager/kustomization.yaml | 4 +- test/e2e/e2e_test.go | 154 ++++++++++++++++++------------ test/utils/utils.go | 12 +-- 4 files changed, 100 insertions(+), 74 deletions(-) diff --git a/Makefile b/Makefile index 3106df55..5a93494a 100644 --- a/Makefile +++ b/Makefile @@ -206,3 +206,7 @@ GOBIN=$(LOCALBIN) go install $${package} ;\ mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\ } endef + +.PHONY: test-e2e +test-e2e: + go test ./test/e2e -v -ginkgo.v diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index a818d7ae..d98951a4 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: quay.io/cephcsi/ceph-csi-operator - newTag: latest + newName: example.com/ceph-csi-operator + newTag: v0.0.1 diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 650b894b..236bfc1e 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "context" "fmt" "os/exec" "time" @@ -25,101 +26,128 @@ import ( . "github.com/onsi/gomega" "github.com/ceph/ceph-csi-operator/test/utils" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/config" + + csiv1alpha1 "github.com/ceph/ceph-csi-operator/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" ) const namespace = "ceph-csi-operator-system" -var _ = Describe("controller", Ordered, func() { +var k8sClient client.Client +var clientset *kubernetes.Clientset + +var _ = Describe("Ceph CSI Operator", Ordered, func() { BeforeAll(func() { - By("installing prometheus operator") - Expect(utils.InstallPrometheusOperator()).To(Succeed()) + var err error + cfg, err := config.GetConfig() + Expect(err).NotTo(HaveOccurred()) + + err = csiv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) - By("installing the cert-manager") - Expect(utils.InstallCertManager()).To(Succeed()) + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + + clientset, err = kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) By("creating manager namespace") - //nolint:gosec - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} + err = k8sClient.Create(context.TODO(), ns) + Expect(err).NotTo(HaveOccurred()) }) AfterAll(func() { - By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() - - By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() - By("removing manager namespace") - //nolint:gosec - cmd := exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} + err := k8sClient.Delete(context.TODO(), ns) + Expect(err).NotTo(HaveOccurred()) }) - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string + Context("Operator Deployment", func() { + It("should deploy and run successfully", func() { var err error - // projectimage stores the name of the image used in the example - var projectimage = "example.com/ceph-csi-operator:v0.0.1" + projectimage := "example.com/ceph-csi-operator:v0.0.1" - By("building the manager(Operator) image") - //nolint:gosec + By("building the operator image") cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("loading the operator image to the cluster") + err = utils.LoadImageToCluster(projectimage) + Expect(err).NotTo(HaveOccurred()) By("installing CRDs") - //nolint:gosec cmd = exec.Command("make", "install") _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) - By("deploying the controller-manager") - //nolint:gosec + By("deploying the operator") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { - // Get pod name - //nolint:gosec - cmd = exec.Command("kubectl", "get", - "pods", "-l", "control-plane=controller-manager", - "-o", "go-template={{ range .items }}"+ - "{{ if not .metadata.deletionTimestamp }}"+ - "{{ .metadata.name }}"+ - "{{ \"\\n\" }}{{ end }}{{ end }}", - "-n", namespace, - ) - - podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) + Expect(err).NotTo(HaveOccurred()) + + By("validating that the operator pod is running") + Eventually(func() error { + pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "control-plane=ceph-csi-op-controller-manager", + }) + if err != nil { + return err + } + if len(pods.Items) != 1 { + return fmt.Errorf("expected 1 operator pod, but got %d", len(pods.Items)) } - controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) - - // Validate pod status - //nolint:gosec - cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) + if pods.Items[0].Status.Phase != corev1.PodRunning { + return fmt.Errorf("operator pod is in %s status", pods.Items[0].Status.Phase) } return nil + }, time.Minute, time.Second).Should(Succeed()) + }) + }) + + Context("Driver CR", func() { + It("should create a RBD Driver CR and verify resources", func() { + driver := &csiv1alpha1.Driver{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "csi.ceph.io/v1alpha1", + Kind: "Driver", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test.rbd.csi.ceph.com", + Namespace: namespace, + }, + Spec: csiv1alpha1.DriverSpec{ + AttachRequired: ptr.To(true), + NodePlugin: &csiv1alpha1.NodePluginSpec{}, + ControllerPlugin: &csiv1alpha1.ControllerPluginSpec{}, + }, } - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + By("creating a Driver CR") + Expect(k8sClient.Create(context.TODO(), driver)).To(Succeed()) + + By("verifying the Driver daemonset") + Eventually(func() error { + daemonset := &appsv1.DaemonSet{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: "test.rbd.csi.ceph.com-nodeplugin", Namespace: namespace}, daemonset) + if err != nil { + return err + } + if daemonset.Status.NumberReady != daemonset.Status.DesiredNumberScheduled { + return fmt.Errorf("daemonset not ready, expected %d nodes, got %d", daemonset.Status.DesiredNumberScheduled, daemonset.Status.NumberReady) + } + return nil + }, time.Minute*5, time.Second*10).Should(Succeed()) }) }) }) diff --git a/test/utils/utils.go b/test/utils/utils.go index 5e04e047..fdd7e2c6 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -108,15 +108,9 @@ func InstallCertManager() error { return err } -// LoadImageToKindCluster loads a local docker image to the kind cluster -func LoadImageToKindClusterWithName(name string) error { - cluster := "kind" - if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { - cluster = v - } - kindOptions := []string{"load", "docker-image", name, "--name", cluster} - //nolint:gosec - cmd := exec.Command("kind", kindOptions...) +// LoadImageToKindCluster loads a local docker image to the minikube cluster +func LoadImageToCluster(name string) error { + cmd := exec.Command("minikube", "image", "load", name) _, err := Run(cmd) return err }