diff --git a/e2e/ceph_user.go b/e2e/ceph_user.go index 7d21ae473146..00b273d6ba6d 100644 --- a/e2e/ceph_user.go +++ b/e2e/ceph_user.go @@ -21,6 +21,8 @@ const ( rbdProvisionerSecretName = "cephcsi-rbd-provisioner" rbdNamespaceNodePluginSecretName = "cephcsi-rbd-ns-node" rbdNamespaceProvisionerSecretName = "cephcsi-rbd-ns-provisioner" + rbdMigrationNodePluginSecretName = "cephcsi-rbd-mig-node" + rbdMigrationProvisionerSecretName = "cephcsi-rbd-mig-provisioner" cephFSNodePluginSecretName = "cephcsi-cephfs-node" cephFSProvisionerSecretName = "cephcsi-cephfs-provisioner" ) diff --git a/e2e/migration.go b/e2e/migration.go index ddeaeaf8bc50..04390fee326e 100644 --- a/e2e/migration.go +++ b/e2e/migration.go @@ -96,7 +96,7 @@ func validateRBDStaticMigrationPVDeletion(f *framework.Framework, appPath, scNam err = deletePVCAndApp("", f, pvc, app) if err != nil { - return fmt.Errorf("failed to delete PVC and application with error %w", err) + return fmt.Errorf("failed to delete PVC and application: %w", err) } return err @@ -134,12 +134,142 @@ func generateClusterIDConfigMapForMigration(f *framework.Framework, c kubernetes // create custom configmap err = createCustomConfigMap(f.ClientSet, rbdDirPath, clusterInfo) if err != nil { - return fmt.Errorf("failed to create configmap with error %w", err) + return fmt.Errorf("failed to create configmap: %w", err) } // restart csi pods for the configmap to take effect. err = recreateCSIRBDPods(f) if err != nil { - return fmt.Errorf("failed to recreate rbd csi pods with error %w", err) + return fmt.Errorf("failed to recreate rbd csi pods: %w", err) + } + + return nil +} + +// createRBDMigrationSecret creates a migration secret with the passed in user name. +// this secret differs from csi secret data on below aspects. +// equivalent to the `UserKey` field, migration secret has `key` field. +// if 'userName' has passed and if it is not admin, the passed in userName will be +// set as the `adminId` field in the secret. +func createRBDMigrationSecret(f *framework.Framework, secretName, userName, userKey string) error { + secPath := fmt.Sprintf("%s/%s", rbdExamplePath, "secret.yaml") + sec, err := getSecret(secPath) + if err != nil { + return err + } + if secretName != "" { + sec.Name = secretName + } + // if its admin, we dont need to change anything in the migration secret, the CSI driver + // will use the key from existing secret and continue. + if userName != "admin" { + sec.StringData["adminId"] = userName + } + sec.StringData["key"] = userKey + sec.Namespace = cephCSINamespace + _, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sec, metav1.CreateOptions{}) + + return err +} + +// createMigrationUserSecretAndSC creates migration user and a secret associated with this user first, +// then create SC based on the same. +func createMigrationUserSecretAndSC(f *framework.Framework, scName string) error { + if scName == "" { + scName = defaultSCName + } + err := createProvNodeCephUserAndSecret(f, true, true) + if err != nil { + return err + } + + err = createMigrationSC(f, scName) + if err != nil { + return err + } + + return nil +} + +func createMigrationSC(f *framework.Framework, scName string) error { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + return fmt.Errorf("failed to delete storageclass: %w", err) + } + param := make(map[string]string) + // add new secrets to the SC parameters + param["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace + param["csi.storage.k8s.io/provisioner-secret-name"] = rbdMigrationProvisionerSecretName + param["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace + param["csi.storage.k8s.io/controller-expand-secret-name"] = rbdMigrationProvisionerSecretName + param["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace + param["csi.storage.k8s.io/node-stage-secret-name"] = rbdMigrationNodePluginSecretName + err = createRBDStorageClass(f.ClientSet, f, scName, nil, param, deletePolicy) + if err != nil { + return fmt.Errorf("failed to create storageclass: %w", err) + } + + return nil +} + +// createProvNodeCephUserAndSecret fetches the ceph migration user's key and create migration secret +// with it based on the arg values of 'provSecret' and 'nodeSecret'. +func createProvNodeCephUserAndSecret(f *framework.Framework, provisionerSecret, nodeSecret bool) error { + if provisionerSecret { + // Fetch the key. + key, err := createCephUser( + f, + keyringRBDProvisionerUsername, + rbdProvisionerCaps(defaultRBDPool, radosNamespace), + ) + if err != nil { + return fmt.Errorf("failed to create user %q: %w", keyringRBDProvisionerUsername, err) + } + err = createRBDMigrationSecret(f, rbdMigrationProvisionerSecretName, keyringRBDProvisionerUsername, key) + if err != nil { + return fmt.Errorf("failed to create provisioner secret: %w", err) + } + } + + if nodeSecret { + // Fetch the key. + key, err := createCephUser( + f, + keyringRBDNodePluginUsername, + rbdNodePluginCaps(defaultRBDPool, radosNamespace)) + if err != nil { + return fmt.Errorf("failed to create user %q: %w", keyringRBDNodePluginUsername, err) + } + err = createRBDMigrationSecret(f, rbdMigrationNodePluginSecretName, keyringRBDNodePluginUsername, key) + if err != nil { + return fmt.Errorf("failed to create node secret: %w", err) + } + } + + return nil +} + +// deleteProvNodeMigrationSecret deletes ceph migration secrets based on the +// arg values of 'provisionerSecret' and 'nodeSecret'. +func deleteProvNodeMigrationSecret(f *framework.Framework, provisionerSecret, nodeSecret bool) error { + c := f.ClientSet + if provisionerSecret { + // delete RBD provisioner secret. + err := c.CoreV1(). + Secrets(cephCSINamespace). + Delete(context.TODO(), rbdMigrationProvisionerSecretName, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete provisioner secret: %w", err) + } + } + + if nodeSecret { + // delete RBD node secret. + err := c.CoreV1(). + Secrets(cephCSINamespace). + Delete(context.TODO(), rbdMigrationNodePluginSecretName, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete node secret: %w", err) + } } return nil diff --git a/e2e/rbd.go b/e2e/rbd.go index 629975503ee6..54815c6c8628 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -374,7 +374,9 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to generate clusterID configmap with error %v", err) } - err = createRBDStorageClass(f.ClientSet, f, "migrationsc", nil, nil, deletePolicy) + + // create a sc with different migration secret + err = createMigrationUserSecretAndSC(f, "migrationsc") if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } @@ -392,6 +394,15 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to create configmap with error %v", err) } + + err = deleteProvNodeMigrationSecret(f, true, true) + if err != nil { + e2elog.Failf("failed to delete migration users and Secrets associated with error %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } }) By("create a PVC and validate owner", func() { @@ -1606,12 +1617,24 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to generate clusterID configmap with error %v", err) } - err = validateRBDStaticMigrationPV(f, appPath, false) + // create node user and migration secret. + err = createProvNodeCephUserAndSecret(f, false, true) + if err != nil { + e2elog.Failf("failed to create users and secret with error %v", err) + } + + err = validateRBDStaticMigrationPV(f, appPath, rbdMigrationNodePluginSecretName, false) if err != nil { e2elog.Failf("failed to validate rbd migrated static pv with error %v", err) } // validate created backend rbd images validateRBDImageCount(f, 0, defaultRBDPool) + + err = deleteProvNodeMigrationSecret(f, false, true) + if err != nil { + e2elog.Failf("failed to delete users and secret with error %v", err) + } + err = deleteConfigMap(rbdDirPath) if err != nil { e2elog.Failf("failed to delete configmap with error %v", err) @@ -1627,12 +1650,24 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to generate clusterID configmap with error %v", err) } - err = validateRBDStaticMigrationPV(f, rawAppPath, true) + // create node user and migration secret. + err = createProvNodeCephUserAndSecret(f, false, true) + if err != nil { + e2elog.Failf("failed to create users and secret with error %v", err) + } + + err = validateRBDStaticMigrationPV(f, rawAppPath, rbdMigrationNodePluginSecretName, true) if err != nil { e2elog.Failf("failed to validate rbd migrated static block pv with error %v", err) } // validate created backend rbd images validateRBDImageCount(f, 0, defaultRBDPool) + + err = deleteProvNodeMigrationSecret(f, false, true) + if err != nil { + e2elog.Failf("failed to delete users and secret with error %v", err) + } + err = deleteConfigMap(rbdDirPath) if err != nil { e2elog.Failf("failed to delete configmap with error %v", err) diff --git a/e2e/staticpvc.go b/e2e/staticpvc.go index 3ef2c96c582a..3bcf6d26107a 100644 --- a/e2e/staticpvc.go +++ b/e2e/staticpvc.go @@ -221,7 +221,7 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI return err } -func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBlock bool) error { +func validateRBDStaticMigrationPV(f *framework.Framework, appPath, nodeSecretName string, isBlock bool) error { opt := make(map[string]string) var ( rbdImageName = "test-static-pv" @@ -254,6 +254,9 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc if e != "" { return fmt.Errorf("failed to create rbd image %s", e) } + if nodeSecretName == "" { + nodeSecretName = rbdNodePluginSecretName + } opt["migration"] = "true" opt["clusterID"] = getMonsHash(mon) @@ -265,7 +268,7 @@ func validateRBDStaticMigrationPV(f *framework.Framework, appPath string, isBloc pvName, rbdImageName, size, - rbdNodePluginSecretName, + nodeSecretName, cephCSINamespace, sc, "rbd.csi.ceph.com",