From 38230ef84f8336b9ffe4c27d5f236e7c244f8a61 Mon Sep 17 00:00:00 2001 From: shubham Date: Wed, 1 Jul 2020 15:56:44 +0530 Subject: [PATCH 1/6] feat(migrate): add support to rename spc while migrating to cspc Signed-off-by: shubham --- cmd/migrate/executor/cstor_pool.go | 9 ++++++ cmd/migrate/executor/options.go | 1 + pkg/migrate/cstor/cspc_generator.go | 7 +++-- pkg/migrate/cstor/pool.go | 45 ++++++++++++++++++++++++----- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/cmd/migrate/executor/cstor_pool.go b/cmd/migrate/executor/cstor_pool.go index 0e848a37..97d1d0cf 100644 --- a/cmd/migrate/executor/cstor_pool.go +++ b/cmd/migrate/executor/cstor_pool.go @@ -55,6 +55,11 @@ func NewMigratePoolJob() *cobra.Command { options.spcName, "cstor SPC name to be migrated. Run \"kubectl get spc\", to get spc-name") + cmd.Flags().StringVarP(&options.cspcName, + "cspc-name", "", + options.cspcName, + "[optional] custom cspc name. By default cspc is created with same name as spc") + return cmd } @@ -72,6 +77,10 @@ func (m *MigrateOptions) RunCStorSPCMigrate() error { klog.Infof("Migrating spc %s to cspc", m.spcName) migrator := cstor.CSPCMigrator{} + if m.cspcName != "" { + klog.Infof("using custom cspc name as %s", m.cspcName) + migrator.SetCSPCName(m.cspcName) + } err := migrator.Migrate(m.spcName, m.openebsNamespace) if err != nil { klog.Error(err) diff --git a/cmd/migrate/executor/options.go b/cmd/migrate/executor/options.go index 39de1e11..a34395b2 100644 --- a/cmd/migrate/executor/options.go +++ b/cmd/migrate/executor/options.go @@ -27,6 +27,7 @@ import ( type MigrateOptions struct { openebsNamespace string spcName string + cspcName string pvName string } diff --git a/pkg/migrate/cstor/cspc_generator.go b/pkg/migrate/cstor/cspc_generator.go index a4d72bda..987a09b1 100644 --- a/pkg/migrate/cstor/cspc_generator.go +++ b/pkg/migrate/cstor/cspc_generator.go @@ -74,11 +74,12 @@ func (c *CSPCMigrator) getCSPCSpecForSPC() (*cstor.CStorPoolCluster, error) { return nil, err } cspcObj := &cstor.CStorPoolCluster{} - cspcObj.Name = c.SPCObj.Name + cspcObj.Name = c.CSPCName cspcObj.Annotations = map[string]string{ // This annotation will be used to disable reconciliation on the dependants. // In this case that will be CSPI types.OpenEBSDisableDependantsReconcileKey: "true", + "openebs.io/migrated": "true", } for _, cspObj := range cspList.Items { cspDeployList, err := deploy.NewKubeClient().WithNamespace(c.OpenebsNamespace). @@ -133,7 +134,7 @@ func getCSPAuxResources(cspDeploy appsv1.Deployment) *corev1.ResourceRequirement func (c *CSPCMigrator) generateCSPC() ( *cstor.CStorPoolCluster, error) { cspcObj, err := c.OpenebsClientset.CstorV1(). - CStorPoolClusters(c.OpenebsNamespace).Get(c.SPCObj.Name, metav1.GetOptions{}) + CStorPoolClusters(c.OpenebsNamespace).Get(c.CSPCName, metav1.GetOptions{}) if !k8serrors.IsNotFound(err) && err != nil { return nil, err } @@ -176,7 +177,7 @@ func (c *CSPCMigrator) generateCSPC() ( return nil, err } cspcObj, err = c.OpenebsClientset.CstorV1(). - CStorPoolClusters(c.OpenebsNamespace).Get(c.SPCObj.Name, metav1.GetOptions{}) + CStorPoolClusters(c.OpenebsNamespace).Get(c.CSPCName, metav1.GetOptions{}) if err != nil { return nil, err } diff --git a/pkg/migrate/cstor/pool.go b/pkg/migrate/cstor/pool.go index 9c7e9b67..577d2be3 100644 --- a/pkg/migrate/cstor/pool.go +++ b/pkg/migrate/cstor/pool.go @@ -65,6 +65,12 @@ type CSPCMigrator struct { CSPCObj *cstor.CStorPoolCluster SPCObj *apis.StoragePoolClaim OpenebsNamespace string + CSPCName string +} + +// SetCSPCName is used to initialize custom name if provided +func (c *CSPCMigrator) SetCSPCName(name string) { + c.CSPCName = name } // Migrate ... @@ -86,6 +92,13 @@ func (c *CSPCMigrator) Migrate(name, namespace string) error { if err != nil { return err } + if c.CSPCName == "" { + c.CSPCName = name + } + err = c.checkForExistingCSPC() + if err != nil { + return err + } err = c.migrate(name) return err } @@ -133,11 +146,11 @@ func (c *CSPCMigrator) migrate(spcName string) error { if err != nil { return errors.Wrapf(err, "failed to validate spc %s", spcName) } - err = c.updateBDCLabels(spcName) + err = c.updateBDCLabels() if err != nil { return errors.Wrapf(err, "failed to update bdc labels for spc %s", spcName) } - klog.Infof("Creating equivalent cspc for spc %s", spcName) + klog.Infof("Creating equivalent cspc %s for spc %s", c.CSPCName, spcName) c.CSPCObj, err = c.generateCSPC() if err != nil { return err @@ -178,6 +191,22 @@ func (c *CSPCMigrator) migrate(spcName string) error { return nil } +func (c *CSPCMigrator) checkForExistingCSPC() error { + cspc, err := c.OpenebsClientset.CstorV1(). + CStorPoolClusters(c.OpenebsNamespace). + Get(c.CSPCName, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + return nil + } + if err == nil { + if cspc.Annotations == nil || cspc.Annotations["openebs.io/migrated"] != "true" { + return errors.Errorf("cspc with same name %s already exists. Use the flag \"--cspc-name\" to rename the spc during migration", c.CSPCName) + } + return nil + } + return err +} + // validateSPC determines that if the spc is allowed to migrate or not. // If the max pool count does not match the number of csp in case auto spc provisioning, // or the blocldevice list in spc does not match bds from the csp, in case of manual provisioning @@ -232,9 +261,9 @@ func (c *CSPCMigrator) getSPCWithMigrationStatus(spcName string) (*apis.StorageP if k8serrors.IsNotFound(err) { klog.Infof("spc %s not found.", spcName) _, err = c.OpenebsClientset.CstorV1(). - CStorPoolClusters(c.OpenebsNamespace).Get(spcName, metav1.GetOptions{}) + CStorPoolClusters(c.OpenebsNamespace).Get(c.CSPCName, metav1.GetOptions{}) if err != nil { - return nil, false, errors.Wrapf(err, "failed to get equivalent cspc for spc %s", spcName) + return nil, false, errors.Wrapf(err, "failed to get equivalent cspc %s for spc %s", c.CSPCName, spcName) } return nil, true, nil } @@ -248,7 +277,7 @@ func (c *CSPCMigrator) getSPCWithMigrationStatus(spcName string) (*apis.StorageP func (c *CSPCMigrator) cspTocspi(cspiObj *cstor.CStorPoolInstance) error { var err1 error hostnameLabel := types.HostNameLabelKey + "=" + cspiObj.Labels[types.HostNameLabelKey] - spcLabel := string(apis.StoragePoolClaimCPK) + "=" + c.CSPCObj.Name + spcLabel := string(apis.StoragePoolClaimCPK) + "=" + c.SPCObj.Name cspLabel := hostnameLabel + "," + spcLabel cspObj, err := getCSP(cspLabel) if err != nil { @@ -374,9 +403,9 @@ func (c *CSPCMigrator) scaleDownDeployment(cspObj *apis.CStorPool, openebsNamesp // Update the bdc with the cspc labels instead of spc labels to allow // filtering of bds claimed by the migrated cspc. -func (c *CSPCMigrator) updateBDCLabels(cspcName string) error { +func (c *CSPCMigrator) updateBDCLabels() error { bdcList, err := c.OpenebsClientset.OpenebsV1alpha1().BlockDeviceClaims(c.OpenebsNamespace).List(metav1.ListOptions{ - LabelSelector: string(apis.StoragePoolClaimCPK) + "=" + cspcName, + LabelSelector: string(apis.StoragePoolClaimCPK) + "=" + c.SPCObj.Name, }) if err != nil { return err @@ -387,7 +416,7 @@ func (c *CSPCMigrator) updateBDCLabels(cspcName string) error { bdcObj := &bdcItem klog.Infof("Updating bdc %s with cspc labels & finalizer.", bdcObj.Name) delete(bdcObj.Labels, string(apis.StoragePoolClaimCPK)) - bdcObj.Labels[types.CStorPoolClusterLabelKey] = cspcName + bdcObj.Labels[types.CStorPoolClusterLabelKey] = c.CSPCName for i, finalizer := range bdcObj.Finalizers { if finalizer == spcFinalizer { bdcObj.Finalizers[i] = cspcFinalizer From 9e26e898d389a1981de7f2991b98872cf4fe1ea9 Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 2 Jul 2020 11:16:30 +0530 Subject: [PATCH 2/6] skip snapshot checks if no snapshots exist Signed-off-by: shubham --- pkg/migrate/cstor/snapshot.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/migrate/cstor/snapshot.go b/pkg/migrate/cstor/snapshot.go index a1915a95..d77ae2fd 100644 --- a/pkg/migrate/cstor/snapshot.go +++ b/pkg/migrate/cstor/snapshot.go @@ -57,11 +57,6 @@ func (s *SnapshotMigrator) migrate(pvName string) error { } func (s *SnapshotMigrator) migrateSnapshots() error { - _, err := s.snapClient.SnapshotV1beta1().VolumeSnapshotClasses(). - Get(snapClass, metav1.GetOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to get snapshotclass %s", snapClass) - } snapshotList, err := snap.NewKubeClient(). WithNamespace(""). List(metav1.ListOptions{ @@ -70,6 +65,14 @@ func (s *SnapshotMigrator) migrateSnapshots() error { if err != nil { return err } + if len(snapshotList.Items) == 0 { + return nil + } + _, err = s.snapClient.SnapshotV1beta1().VolumeSnapshotClasses(). + Get(snapClass, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to get snapshotclass %s", snapClass) + } for _, snapshot := range snapshotList.Items { snapshot := snapshot // pin it if len(snapshot.Spec.SnapshotDataName) == 0 { From 4fc2334350e3e88d4682fa261f3d38b1a64fc5f8 Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 2 Jul 2020 16:37:06 +0530 Subject: [PATCH 3/6] add check to prevent chnages to cspc-name flag on the same spc Signed-off-by: shubham --- pkg/migrate/cstor/pool.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/pkg/migrate/cstor/pool.go b/pkg/migrate/cstor/pool.go index 577d2be3..34e5e00e 100644 --- a/pkg/migrate/cstor/pool.go +++ b/pkg/migrate/cstor/pool.go @@ -95,7 +95,7 @@ func (c *CSPCMigrator) Migrate(name, namespace string) error { if c.CSPCName == "" { c.CSPCName = name } - err = c.checkForExistingCSPC() + err = c.checkForExistingCSPC(name) if err != nil { return err } @@ -191,7 +191,30 @@ func (c *CSPCMigrator) migrate(spcName string) error { return nil } -func (c *CSPCMigrator) checkForExistingCSPC() error { +func (c *CSPCMigrator) checkForExistingCSPC(spcName string) error { + spcObj, err := spc.NewKubeClient().Get(spcName, metav1.GetOptions{}) + if err == nil { + if spcObj.Annotations == nil || spcObj.Annotations[types.CStorPoolClusterLabelKey] == "" { + if spcObj.Annotations == nil { + spcObj.Annotations = map[string]string{} + } + spcObj.Annotations[types.CStorPoolClusterLabelKey] = c.CSPCName + _, err := spc.NewKubeClient().Update(spcObj) + if err != nil { + return err + } + } else { + if spcObj.Annotations[types.CStorPoolClusterLabelKey] != c.CSPCName { + return errors.Errorf("invalid cspc-name: the spc %s already set to be renamed as %s", + spcName, + spcObj.Annotations[types.CStorPoolClusterLabelKey], + ) + } + } + } + if !k8serrors.IsNotFound(err) { + return err + } cspc, err := c.OpenebsClientset.CstorV1(). CStorPoolClusters(c.OpenebsNamespace). Get(c.CSPCName, metav1.GetOptions{}) From 3263818e2f598227a56116573434a99d060d6b99 Mon Sep 17 00:00:00 2001 From: shubham Date: Fri, 3 Jul 2020 20:06:29 +0530 Subject: [PATCH 4/6] addressed review comments Signed-off-by: shubham --- pkg/migrate/cstor/cspc_generator.go | 8 ++--- pkg/migrate/cstor/pool.go | 51 ++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/pkg/migrate/cstor/cspc_generator.go b/pkg/migrate/cstor/cspc_generator.go index 987a09b1..7980c0b7 100644 --- a/pkg/migrate/cstor/cspc_generator.go +++ b/pkg/migrate/cstor/cspc_generator.go @@ -65,7 +65,7 @@ func getBDList(rg apis.BlockDeviceGroup) []cstor.CStorPoolInstanceBlockDevice { return list } -func (c *CSPCMigrator) getCSPCSpecForSPC() (*cstor.CStorPoolCluster, error) { +func (c *CSPCMigrator) getCSPCSpecForSPC(spcName string) (*cstor.CStorPoolCluster, error) { cspClient := csp.KubeClient() cspList, err := cspClient.List(metav1.ListOptions{ LabelSelector: string(apis.StoragePoolClaimCPK) + "=" + c.SPCObj.Name, @@ -79,7 +79,7 @@ func (c *CSPCMigrator) getCSPCSpecForSPC() (*cstor.CStorPoolCluster, error) { // This annotation will be used to disable reconciliation on the dependants. // In this case that will be CSPI types.OpenEBSDisableDependantsReconcileKey: "true", - "openebs.io/migrated": "true", + "openebs.io/migrated-from": spcName, } for _, cspObj := range cspList.Items { cspDeployList, err := deploy.NewKubeClient().WithNamespace(c.OpenebsNamespace). @@ -131,7 +131,7 @@ func getCSPAuxResources(cspDeploy appsv1.Deployment) *corev1.ResourceRequirement } // generateCSPC creates an equivalent cspc for the given spc object -func (c *CSPCMigrator) generateCSPC() ( +func (c *CSPCMigrator) generateCSPC(spcName string) ( *cstor.CStorPoolCluster, error) { cspcObj, err := c.OpenebsClientset.CstorV1(). CStorPoolClusters(c.OpenebsNamespace).Get(c.CSPCName, metav1.GetOptions{}) @@ -139,7 +139,7 @@ func (c *CSPCMigrator) generateCSPC() ( return nil, err } if err != nil { - cspcObj, err = c.getCSPCSpecForSPC() + cspcObj, err = c.getCSPCSpecForSPC(spcName) if err != nil { return nil, err } diff --git a/pkg/migrate/cstor/pool.go b/pkg/migrate/cstor/pool.go index 34e5e00e..be7de059 100644 --- a/pkg/migrate/cstor/pool.go +++ b/pkg/migrate/cstor/pool.go @@ -151,7 +151,7 @@ func (c *CSPCMigrator) migrate(spcName string) error { return errors.Wrapf(err, "failed to update bdc labels for spc %s", spcName) } klog.Infof("Creating equivalent cspc %s for spc %s", c.CSPCName, spcName) - c.CSPCObj, err = c.generateCSPC() + c.CSPCObj, err = c.generateCSPC(spcName) if err != nil { return err } @@ -194,17 +194,29 @@ func (c *CSPCMigrator) migrate(spcName string) error { func (c *CSPCMigrator) checkForExistingCSPC(spcName string) error { spcObj, err := spc.NewKubeClient().Get(spcName, metav1.GetOptions{}) if err == nil { + // if cspc-name flag is used and the spc does not have the cspc + // annotation add the cspc annotation if spcObj.Annotations == nil || spcObj.Annotations[types.CStorPoolClusterLabelKey] == "" { - if spcObj.Annotations == nil { - spcObj.Annotations = map[string]string{} - } - spcObj.Annotations[types.CStorPoolClusterLabelKey] = c.CSPCName - _, err := spc.NewKubeClient().Update(spcObj) - if err != nil { - return err + if spcName != c.CSPCName { + err = addCSPCAnnotationToSPC(spcObj, c.CSPCName) + if err != nil { + return err + } } } else { + // if the first attempt at using cspc-name flag fails check for the new cspc + // name proivded and if no cspc exists update the annotation if spcObj.Annotations[types.CStorPoolClusterLabelKey] != c.CSPCName { + _, err := c.OpenebsClientset.CstorV1(). + CStorPoolClusters(c.OpenebsNamespace). + Get(c.CSPCName, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + err = addCSPCAnnotationToSPC(spcObj, c.CSPCName) + return err + } + if err != nil { + return err + } return errors.Errorf("invalid cspc-name: the spc %s already set to be renamed as %s", spcName, spcObj.Annotations[types.CStorPoolClusterLabelKey], @@ -212,7 +224,7 @@ func (c *CSPCMigrator) checkForExistingCSPC(spcName string) error { } } } - if !k8serrors.IsNotFound(err) { + if err != nil && !k8serrors.IsNotFound(err) { return err } cspc, err := c.OpenebsClientset.CstorV1(). @@ -222,7 +234,7 @@ func (c *CSPCMigrator) checkForExistingCSPC(spcName string) error { return nil } if err == nil { - if cspc.Annotations == nil || cspc.Annotations["openebs.io/migrated"] != "true" { + if cspc.Annotations == nil || cspc.Annotations["openebs.io/migrated-from"] != spcName { return errors.Errorf("cspc with same name %s already exists. Use the flag \"--cspc-name\" to rename the spc during migration", c.CSPCName) } return nil @@ -518,9 +530,10 @@ func (c *CSPCMigrator) updateCVRsLabels(cspObj *apis.CStorPool, cspiObj *cstor.C func addSkipAnnotationToSPC(spcObj *apis.StoragePoolClaim) error { retry: - spcObj.Annotations = map[string]string{ - "openebs.io/skip-validations": "true", + if spcObj.Annotations == nil { + spcObj.Annotations = map[string]string{} } + spcObj.Annotations["openebs.io/skip-validations"] = "true" _, err := spc.NewKubeClient().Update(spcObj) if k8serrors.IsConflict(err) { klog.Errorf("failed to update spc with skip-validation annotation due to conflict error") @@ -528,3 +541,17 @@ retry: } return err } + +func addCSPCAnnotationToSPC(spcObj *apis.StoragePoolClaim, cspcName string) error { +retry: + if spcObj.Annotations == nil { + spcObj.Annotations = map[string]string{} + } + spcObj.Annotations[types.CStorPoolClusterLabelKey] = cspcName + _, err := spc.NewKubeClient().Update(spcObj) + if k8serrors.IsConflict(err) { + klog.Errorf("failed to update spc with cspc annotation due to conflict error") + goto retry + } + return err +} From 7680ba2fad58142d4d802e49f4e780bf174bf643 Mon Sep 17 00:00:00 2001 From: shubham Date: Mon, 6 Jul 2020 14:20:39 +0530 Subject: [PATCH 5/6] made the verification more readable Signed-off-by: shubham --- pkg/migrate/cstor/pool.go | 80 +++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/pkg/migrate/cstor/pool.go b/pkg/migrate/cstor/pool.go index be7de059..4a03ecf9 100644 --- a/pkg/migrate/cstor/pool.go +++ b/pkg/migrate/cstor/pool.go @@ -191,55 +191,61 @@ func (c *CSPCMigrator) migrate(spcName string) error { return nil } +// checkForExistingCSPC verifies the migration as follows: +// spc = getSPC +// 1. if spc does not exist +// return nil (getSPCWithMigrationStatus will handle it) +// cspc = getCSPC(from flag or same as spc) +// 2. if cspc exist && +// a. spc has anno with cspc-name && cspc has migration anno with spc-name +// return nil +// b. else +// return err cspc already exists +// 3. if cspc does not exist && +// a. spc has no anno +// patch spc with anno +// b. spc has diff anno than current cspc-name +// return err func (c *CSPCMigrator) checkForExistingCSPC(spcName string) error { spcObj, err := spc.NewKubeClient().Get(spcName, metav1.GetOptions{}) - if err == nil { - // if cspc-name flag is used and the spc does not have the cspc - // annotation add the cspc annotation - if spcObj.Annotations == nil || spcObj.Annotations[types.CStorPoolClusterLabelKey] == "" { - if spcName != c.CSPCName { - err = addCSPCAnnotationToSPC(spcObj, c.CSPCName) - if err != nil { - return err - } - } - } else { - // if the first attempt at using cspc-name flag fails check for the new cspc - // name proivded and if no cspc exists update the annotation - if spcObj.Annotations[types.CStorPoolClusterLabelKey] != c.CSPCName { - _, err := c.OpenebsClientset.CstorV1(). - CStorPoolClusters(c.OpenebsNamespace). - Get(c.CSPCName, metav1.GetOptions{}) - if k8serrors.IsNotFound(err) { - err = addCSPCAnnotationToSPC(spcObj, c.CSPCName) - return err - } - if err != nil { - return err - } - return errors.Errorf("invalid cspc-name: the spc %s already set to be renamed as %s", - spcName, - spcObj.Annotations[types.CStorPoolClusterLabelKey], - ) - } - } - } if err != nil && !k8serrors.IsNotFound(err) { return err } + if k8serrors.IsNotFound(err) { + return nil + } cspc, err := c.OpenebsClientset.CstorV1(). CStorPoolClusters(c.OpenebsNamespace). Get(c.CSPCName, metav1.GetOptions{}) - if k8serrors.IsNotFound(err) { - return nil + if err != nil && !k8serrors.IsNotFound(err) { + return err } if err == nil { - if cspc.Annotations == nil || cspc.Annotations["openebs.io/migrated-from"] != spcName { - return errors.Errorf("cspc with same name %s already exists. Use the flag \"--cspc-name\" to rename the spc during migration", c.CSPCName) + if spcObj.Annotations[types.CStorPoolClusterLabelKey] == c.CSPCName && + cspc.Annotations["openebs.io/migrated-from"] == spcName { + return nil } - return nil + return errors.Errorf( + "failed to validate migration: the spc %s is set to be renamed as %s, but got cspc-name %s instead", + spcName, + spcObj.Annotations[types.CStorPoolClusterLabelKey], + c.CSPCName, + ) } - return err + if k8serrors.IsNotFound(err) { + if spcObj.Annotations == nil || spcObj.Annotations[types.CStorPoolClusterLabelKey] == "" { + return addCSPCAnnotationToSPC(spcObj, c.CSPCName) + } + if spcObj.Annotations[types.CStorPoolClusterLabelKey] != c.CSPCName { + return errors.Errorf( + "failed to validate migration: the spc %s is set to be renamed as %s, but got cspc-name %s instead", + spcName, + spcObj.Annotations[types.CStorPoolClusterLabelKey], + c.CSPCName, + ) + } + } + return nil } // validateSPC determines that if the spc is allowed to migrate or not. From c09a98d80a3ede3eb6ab706d99e0a1f295e50229 Mon Sep 17 00:00:00 2001 From: shubham Date: Tue, 7 Jul 2020 12:37:28 +0530 Subject: [PATCH 6/6] bump VERSION file Signed-off-by: shubham --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1cac385c..0eed1a29 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.11.0 +1.12.0