From 4a1f2143dcc2ed362cd6a10233f27e8eb2b10f17 Mon Sep 17 00:00:00 2001 From: Diwakar Sharma Date: Wed, 19 Jun 2024 09:01:17 +0000 Subject: [PATCH] test: create a shared LV for use by two app pods Signed-off-by: Diwakar Sharma --- tests/provision_test.go | 36 +++-- tests/suite_test.go | 3 +- tests/utils.go | 294 ++++++++++++++++++++++------------------ 3 files changed, 193 insertions(+), 140 deletions(-) diff --git a/tests/provision_test.go b/tests/provision_test.go index 71809665..b3c0f2a6 100644 --- a/tests/provision_test.go +++ b/tests/provision_test.go @@ -30,9 +30,11 @@ var _ = Describe("[lvmpv] TEST VOLUME PROVISIONING", func() { }) }) -func deleteAppAndPvc(appname string, pvcname string) { - By("Deleting the application deployment") - deleteAppDeployment(appName) +func deleteAppAndPvc(appnames []string, pvcname string) { + for _, appName := range appnames { + By("Deleting the application deployment " + appName) + deleteAppDeployment(appName) + } By("Deleting the PVC") deleteAndVerifyPVC(pvcName) } @@ -56,7 +58,7 @@ func fsVolCreationTest() { resizeAndVerifyPVC(false, "10Gi") } - deleteAppAndPvc(appName, pvcName) + deleteAppAndPvc(appNames, pvcName) // PV should be present after PVC deletion since snapshot is present By("Verifying that PV exists after PVC deletion") @@ -81,7 +83,7 @@ func blockVolCreationTest() { createSnapshot(pvcName, snapName, snapYAML) By("verify snapshot") verifySnapshotCreated(snapName) - deleteAppAndPvc(appName, pvcName) + deleteAppAndPvc(appNames, pvcName) By("Verifying that PV exists after PVC deletion") verifyPVForPVC(true, pvcName) By("Deleting snapshot") @@ -91,6 +93,21 @@ func blockVolCreationTest() { By("Deleting storage class", deleteStorageClass) } +func sharedVolumeTest() { + By("Creating shared LV storage class", createSharedVolStorageClass) + By("creating and verifying PVC bound status", createAndVerifyPVC) + //we use two fio app pods for this test. + appNames = append(appNames, "fio-ci-1") + By("Creating and deploying app pod", createDeployVerifyApp) + By("verifying LVMVolume object", VerifyLVMVolume) + By("Online resizing the shared volume") + resizeAndVerifyPVC(true, "8Gi") + deleteAppAndPvc(appNames, pvcName) + By("Deleting storage class", deleteStorageClass) + // Reset the app list back to original + appNames = appNames[:len(appNames)-1] +} + func thinVolCreationTest() { By("Creating thinProvision storage class", createThinStorageClass) By("creating and verifying PVC bound status", createAndVerifyPVC) @@ -102,7 +119,7 @@ func thinVolCreationTest() { createSnapshot(pvcName, snapName, snapYAML) By("verify snapshot") verifySnapshotCreated(snapName) - deleteAppAndPvc(appName, pvcName) + deleteAppAndPvc(appNames, pvcName) By("Verifying that PV exists after PVC deletion") verifyPVForPVC(true, pvcName) By("Deleting snapshot") @@ -119,7 +136,7 @@ func thinVolCapacityTest() { By("Creating and deploying app pod", createDeployVerifyApp) By("verifying thinpool auto-extended", VerifyThinpoolExtend) By("verifying LVMVolume object", VerifyLVMVolume) - deleteAppAndPvc(appName, pvcName) + deleteAppAndPvc(appNames, pvcName) By("Deleting thinProvision storage class", deleteStorageClass) } @@ -130,7 +147,7 @@ func sizedSnapFSTest() { By("verifying LVMVolume object", VerifyLVMVolume) createSnapshot(pvcName, snapName, sizedsnapYAML) verifySnapshotCreated(snapName) - deleteAppAndPvc(appName, pvcName) + deleteAppAndPvc(appNames, pvcName) deleteSnapshot(pvcName, snapName, sizedsnapYAML) By("Deleting storage class", deleteStorageClass) } @@ -142,7 +159,7 @@ func sizedSnapBlockTest() { By("verifying LVMVolume object", VerifyLVMVolume) createSnapshot(pvcName, snapName, sizedsnapYAML) verifySnapshotCreated(snapName) - deleteAppAndPvc(appName, pvcName) + deleteAppAndPvc(appNames, pvcName) deleteSnapshot(pvcName, snapName, sizedsnapYAML) By("Deleting storage class", deleteStorageClass) } @@ -178,6 +195,7 @@ func volumeCreationTest() { By("Running block volume creation test", blockVolCreationTest) By("Running thin volume creation test", thinVolCreationTest) By("Running leak protection test", leakProtectionTest) + By("Running shared volume for two app pods on same node test", sharedVolumeTest) } func capacityTest() { diff --git a/tests/suite_test.go b/tests/suite_test.go index 2e950139..4fb8d247 100644 --- a/tests/suite_test.go +++ b/tests/suite_test.go @@ -60,7 +60,8 @@ var ( LocalProvisioner = "local.csi.openebs.io" pvcName = "lvmpv-pvc" snapName = "lvmpv-snap" - appName = "fio-ci" + // default one fio app for most tests. + appNames = []string{"fio-ci-0"} nodeDaemonSet = "openebs-lvm-node" controllerDeployment = "openebs-lvm-controller" diff --git a/tests/utils.go b/tests/utils.go index 8f65cc9f..aae02a98 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -148,6 +148,29 @@ func createStorageClass() { gomega.Expect(err).To(gomega.BeNil(), "while creating a default storageclass {%s}", scName) } +func createSharedVolStorageClass() { + var ( + err error + ) + + parameters := map[string]string{ + "volgroup": VOLGROUP, + "shared": "yes", + } + + ginkgo.By("building a shared volume storage class") + scObj, err = sc.NewBuilder(). + WithGenerateName(scName). + WithVolumeExpansion(true). + WithParametersNew(parameters). + WithProvisioner(LocalProvisioner).Build() + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), + "while building shared volume storageclass obj with prefix {%s}", scName) + + scObj, err = SCClient.Create(scObj) + gomega.Expect(err).To(gomega.BeNil(), "while creating a shared volume storageclass {%s}", scName) +} + func createThinStorageClass() { var ( err error @@ -187,7 +210,6 @@ func VerifyLVMVolume() { gomega.Expect(vol.Finalizers[0]).To(gomega.Equal(lvm.LVMFinalizer), "while checking finializer to be set {%s}", pvcObj.Spec.VolumeName) } - func deleteStorageClass() { err := SCClient.Delete(scObj.Name, &metav1.DeleteOptions{}) gomega.Expect(err).To(gomega.BeNil(), @@ -321,137 +343,146 @@ func resizeAndVerifyPVC(shouldPass bool, size string) { } func createDeployVerifyApp() { ginkgo.By("creating and deploying app pod") - createAndDeployAppPod(appName) + createAndDeployAppPod(appNames) time.Sleep(30 * time.Second) - ginkgo.By("verifying app pod is running", verifyAppPodRunning) + ginkgo.By("verifying app pods are running", verifyAppPodRunning) } -func createAndDeployAppPod(appname string) { +func createAndDeployAppPod(appnames []string) { var err error - ginkgo.By("building a fio app pod deployment using above lvm volume") - deployObj, err = deploy.NewBuilder(). - WithName(appname). - WithNamespace(OpenEBSNamespace). - WithLabelsNew( - map[string]string{ - "app": "fio-ci", - }, - ). - WithSelectorMatchLabelsNew( - map[string]string{ - "app": "fio-ci", - }, - ). - WithPodTemplateSpecBuilder( - pts.NewBuilder(). - WithLabelsNew( - map[string]string{ - "app": "fio-ci", - }, - ). - WithContainerBuilders( - container.NewBuilder(). - WithImage("xridge/fio"). - WithName("fio"). - WithImagePullPolicy(corev1.PullIfNotPresent). - WithCommandNew( - []string{ - "sh", - "-c", - "fio --filename=/mnt/datadir/fioFile --direct=1 --rw=write --bs=4k --ioengine=linuxaio --iodepth=32 --size=3GiB --numjobs=1 --name=fio-ci", - }, - ). - WithVolumeMountsNew( - []corev1.VolumeMount{ - corev1.VolumeMount{ - Name: "datavol1", - // If this path changes, modify the above fio command line accordingly. - MountPath: "/mnt/datadir", + rwmode := "write" + for index, appname := range appnames { + // only one app should do the write to ensure data safety. + if index > 0 { + rwmode = "read" + } + ginkgo.By("building app " + appname + " " + rwmode + " mode pod deployment using above lvm volume") + deployObj, err = deploy.NewBuilder(). + WithName(appname). + WithNamespace(OpenEBSNamespace). + WithLabelsNew( + map[string]string{ + "app": appname, + }, + ). + WithSelectorMatchLabelsNew( + map[string]string{ + "app": appname, + }, + ). + WithPodTemplateSpecBuilder( + pts.NewBuilder(). + WithLabelsNew( + map[string]string{ + "app": appname, + }, + ). + WithContainerBuilders( + container.NewBuilder(). + WithImage("xridge/fio"). + WithName("fio"). + WithImagePullPolicy(corev1.PullIfNotPresent). + WithCommandNew( + []string{ + "sh", + "-c", + "fio --filename=/mnt/datadir/fioFile --direct=1 --rw=" + rwmode + " --bs=4k --ioengine=linuxaio --iodepth=32 --size=3GiB --numjobs=1 --name=fio-ci", }, - }, - ), - ). - WithVolumeBuilders( - k8svolume.NewBuilder(). - WithName("datavol1"). - WithPVCSource(pvcObj.Name), - ), - ). - Build() - - gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "while building app deployement {%s}", appName) - - deployObj, err = DeployClient.WithNamespace(OpenEBSNamespace).Create(deployObj) - gomega.Expect(err).ShouldNot( - gomega.HaveOccurred(), - "while creating pod {%s} in namespace {%s}", - appName, - OpenEBSNamespace, - ) + ). + WithVolumeMountsNew( + []corev1.VolumeMount{ + corev1.VolumeMount{ + Name: "datavol1", + // If this path changes, modify the above fio command line accordingly. + MountPath: "/mnt/datadir", + }, + }, + ), + ). + WithVolumeBuilders( + k8svolume.NewBuilder(). + WithName("datavol1"). + WithPVCSource(pvcObj.Name), + ), + ). + Build() + + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "while building app deployement {%s}", appname) + + deployObj, err = DeployClient.WithNamespace(OpenEBSNamespace).Create(deployObj) + gomega.Expect(err).ShouldNot( + gomega.HaveOccurred(), + "while creating pod {%s} in namespace {%s}", + appname, + OpenEBSNamespace, + ) + } } func createAndDeployBlockAppPod() { var err error - ginkgo.By("building a fio app pod deployment using above lvm volume") - deployObj, err = deploy.NewBuilder(). - WithName(appName). - WithNamespace(OpenEBSNamespace). - WithLabelsNew( - map[string]string{ - "app": "fio-ci", - }, - ). - WithSelectorMatchLabelsNew( - map[string]string{ - "app": "fio-ci", - }, - ). - WithPodTemplateSpecBuilder( - pts.NewBuilder(). - WithLabelsNew( - map[string]string{ - "app": "fio-ci", - }, - ). - WithContainerBuilders( - container.NewBuilder(). - WithImage("xridge/fio"). - WithName("fio"). - WithImagePullPolicy(corev1.PullIfNotPresent). - WithCommandNew( - []string{ - "sh", - "-c", - "fio --filename=/dev/xvda/fioFile --direct=1 --rw=write --bs=4k --ioengine=linuxaio --iodepth=32 --size=3GiB --numjobs=1 --name=fio-ci", - }, - ). - WithVolumeDevicesNew( - []corev1.VolumeDevice{ - corev1.VolumeDevice{ - Name: "datavol1", - // If this path changes, modify the above fio command line accordingly. - DevicePath: "/dev/xvda", + for _, appName := range appNames { + ginkgo.By("building app " + appName + " pod deployment using above lvm volume") + deployObj, err = deploy.NewBuilder(). + WithName(appName). + WithNamespace(OpenEBSNamespace). + WithLabelsNew( + map[string]string{ + "app": appName, + }, + ). + WithSelectorMatchLabelsNew( + map[string]string{ + "app": appName, + }, + ). + WithPodTemplateSpecBuilder( + pts.NewBuilder(). + WithLabelsNew( + map[string]string{ + "app": appName, + }, + ). + WithContainerBuilders( + container.NewBuilder(). + WithImage("xridge/fio"). + WithName("fio"). + WithImagePullPolicy(corev1.PullIfNotPresent). + WithCommandNew( + []string{ + "sh", + "-c", + "fio --filename=/dev/xvda/fioFile --direct=1 --rw=write --bs=4k --ioengine=linuxaio --iodepth=32 --size=3GiB --numjobs=1 --name=fio-ci", }, - }, - ), - ). - WithVolumeBuilders( - k8svolume.NewBuilder(). - WithName("datavol1"). - WithPVCSource(pvcObj.Name), - ), - ). - Build() - - gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "while building app deployement {%s}", appName) - - deployObj, err = DeployClient.WithNamespace(OpenEBSNamespace).Create(deployObj) - gomega.Expect(err).ShouldNot( - gomega.HaveOccurred(), - "while creating pod {%s} in namespace {%s}", - appName, - OpenEBSNamespace, - ) + ). + WithVolumeDevicesNew( + []corev1.VolumeDevice{ + corev1.VolumeDevice{ + Name: "datavol1", + // If this path changes, modify the above fio command line accordingly. + DevicePath: "/dev/xvda", + }, + }, + ), + ). + WithVolumeBuilders( + k8svolume.NewBuilder(). + WithName("datavol1"). + WithPVCSource(pvcObj.Name), + ), + ). + Build() + + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "while building app deployement {%s}", appName) + + deployObj, err = DeployClient.WithNamespace(OpenEBSNamespace).Create(deployObj) + gomega.Expect(err).ShouldNot( + gomega.HaveOccurred(), + "while creating pod {%s} in namespace {%s}", + appName, + OpenEBSNamespace, + ) + } } func createDeployVerifyBlockApp() { @@ -462,15 +493,18 @@ func createDeployVerifyBlockApp() { func verifyAppPodRunning() { var err error - appPod, err = PodClient.WithNamespace(OpenEBSNamespace). - List(metav1.ListOptions{ - LabelSelector: "app=fio-ci", - }, - ) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "while verifying application pod") + for _, appName := range appNames { + labelValue := fmt.Sprintf("app=%s", appName) + appPod, err = PodClient.WithNamespace(OpenEBSNamespace). + List(metav1.ListOptions{ + LabelSelector: labelValue, + }, + ) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "while verifying application pod") - status := IsPodRunningEventually(OpenEBSNamespace, appPod.Items[0].Name) - gomega.Expect(status).To(gomega.Equal(true), "while checking status of pod {%s}", appPod.Items[0].Name) + status := IsPodRunningEventually(OpenEBSNamespace, appPod.Items[0].Name) + gomega.Expect(status).To(gomega.Equal(true), "while checking status of pod {%s}", appPod.Items[0].Name) + } } func deleteAppDeployment(appname string) {