From 050b97db1c956a1e6444c4069e5d412353d36d92 Mon Sep 17 00:00:00 2001 From: shubham Date: Tue, 19 Nov 2019 16:59:22 +0530 Subject: [PATCH 01/22] feat(migrate): generate CSPC form given SPC Signed-off-by: shubham --- .../openebs.io/v1alpha1/cstorpool_cluster.go | 3 + pkg/cstor/migrate/cspc_generator.go | 102 ++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 pkg/cstor/migrate/cspc_generator.go diff --git a/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go b/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go index 3cc64ef775..9c0197fe53 100644 --- a/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go +++ b/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go @@ -98,6 +98,9 @@ type PoolSpec struct { // PoolConfig is the default pool config that applies to the // pool on node. PoolConfig PoolConfig `json:"poolConfig"` + // OldCSPUID is used to migrate old csp to cspi. This will be the + // old pool name which needs to imported and renamed as new pool + OldCSPUID string `json:"oldCSPUID,omitempty"` } // PoolConfig is the default pool config that applies to the diff --git a/pkg/cstor/migrate/cspc_generator.go b/pkg/cstor/migrate/cspc_generator.go new file mode 100644 index 0000000000..1cd8b8b401 --- /dev/null +++ b/pkg/cstor/migrate/cspc_generator.go @@ -0,0 +1,102 @@ +/* +Copyright 2019 The OpenEBS Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" + csp "github.com/openebs/maya/pkg/cstor/pool/v1alpha3" + cspc "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1" + spc "github.com/openebs/maya/pkg/storagepoolclaim/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + typeMap = map[string]string{ + string(apis.PoolTypeStripedCPV): string(apis.PoolStriped), + string(apis.PoolTypeMirroredCPV): string(apis.PoolMirrored), + string(apis.PoolTypeRaidzCPV): string(apis.PoolRaidz), + string(apis.PoolTypeRaidz2CPV): string(apis.PoolRaidz2), + } +) + +func getBDCList(cspObj apis.CStorPool) []apis.CStorPoolClusterBlockDevice { + list := []apis.CStorPoolClusterBlockDevice{} + for _, bdcObj := range cspObj.Spec.Group[0].Item { + list = append(list, + apis.CStorPoolClusterBlockDevice{ + BlockDeviceName: bdcObj.Name, + }, + ) + } + return list +} + +func getCSPCSpec(spc *apis.StoragePoolClaim) (*apis.CStorPoolCluster, error) { + cspClient := csp.KubeClient() + cspList, err := cspClient.List(metav1.ListOptions{ + LabelSelector: string(apis.StoragePoolClaimCPK) + "=" + spc.Name, + }) + if err != nil { + return nil, err + } + cspcObj := &apis.CStorPoolCluster{} + cspcObj.Name = spc.Name + cspcObj.Annotations = map[string]string{ + // This label will be used to disable reconciliation on the dependants + // In this case that will be CSPI + "reconcile.openebs.io/dependants": "false", + } + for _, cspObj := range cspList.Items { + cspcObj.Spec.Pools = append(cspcObj.Spec.Pools, + apis.PoolSpec{ + NodeSelector: map[string]string{ + string(apis.HostNameCPK): cspObj.Labels[string(apis.HostNameCPK)], + }, + RaidGroups: []apis.RaidGroup{ + apis.RaidGroup{ + Type: typeMap[cspObj.Spec.PoolSpec.PoolType], + BlockDevices: getBDCList(cspObj), + }, + }, + PoolConfig: apis.PoolConfig{ + CacheFile: cspObj.Spec.PoolSpec.CacheFile, + DefaultRaidGroupType: typeMap[cspObj.Spec.PoolSpec.PoolType], + OverProvisioning: cspObj.Spec.PoolSpec.OverProvisioning, + }, + OldCSPUID: string(cspObj.UID), + }, + ) + + } + return cspcObj, nil +} + +func generateCSPC(spcName, openebsNamespace string) error { + spcObj, err := spc.NewKubeClient().Get(spcName, metav1.GetOptions{}) + if err != nil { + return err + } + cspcObj, err := getCSPCSpec(spcObj) + if err != nil { + return err + } + _, err = cspc.NewKubeClient().WithNamespace(openebsNamespace).Create(cspcObj) + if err != nil { + return err + } + return nil +} From 82e8d56be61a1ac653db94566fba51a27fb8c074 Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 28 Nov 2019 16:27:57 +0530 Subject: [PATCH 02/22] feat(cspc): migrate pools from spc to cspc Signed-off-by: shubham --- GNUmakefile | 2 + buildscripts/migrate/Dockerfile | 20 ++ buildscripts/migrate/Makefile.mk | 32 +++ cmd/cspi-mgmt/app/handler.go | 7 +- cmd/cstor-pool-mgmt/pool/v1alpha2/import.go | 5 +- cmd/migrate/executor/env.go | 30 ++ cmd/migrate/executor/error.go | 34 +++ cmd/migrate/executor/options.go | 46 +++ cmd/migrate/executor/pool.go | 80 ++++++ cmd/migrate/executor/setup_job.go | 66 +++++ cmd/migrate/main.go | 31 ++ .../nodeselect/v1alpha2/build_csp.go | 22 +- pkg/cstor/migrate/cspc_generator.go | 64 ++++- pkg/cstor/migrate/migration.go | 270 ++++++++++++++++++ 14 files changed, 694 insertions(+), 15 deletions(-) create mode 100644 buildscripts/migrate/Dockerfile create mode 100644 buildscripts/migrate/Makefile.mk create mode 100644 cmd/migrate/executor/env.go create mode 100644 cmd/migrate/executor/error.go create mode 100644 cmd/migrate/executor/options.go create mode 100644 cmd/migrate/executor/pool.go create mode 100644 cmd/migrate/executor/setup_job.go create mode 100644 cmd/migrate/main.go create mode 100644 pkg/cstor/migrate/migration.go diff --git a/GNUmakefile b/GNUmakefile index f7f33b3d4f..596a3bd0b0 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -75,6 +75,7 @@ HUB_USER?=openebs ADMISSION_SERVER_REPO_NAME?=admission-server M_UPGRADE_REPO_NAME?=m-upgrade CSPC_OPERATOR_REPO_NAME?=cspc-operator +M_MIGRATE_REPO_NAME?=migrate ifeq (${IMAGE_TAG}, ) IMAGE_TAG = ci @@ -122,6 +123,7 @@ include ./buildscripts/cstor-pool-mgmt/Makefile.mk include ./buildscripts/cstor-volume-mgmt/Makefile.mk include ./buildscripts/cspi-mgmt/Makefile.mk include ./buildscripts/cvc-operator/Makefile.mk +include ./buildscripts/migrate/Makefile.mk .PHONY: all all: compile-tests apiserver-image exporter-image pool-mgmt-image volume-mgmt-image \ diff --git a/buildscripts/migrate/Dockerfile b/buildscripts/migrate/Dockerfile new file mode 100644 index 0000000000..bb75294e48 --- /dev/null +++ b/buildscripts/migrate/Dockerfile @@ -0,0 +1,20 @@ +# +# This builds openebs migratte image using +# its latest binary +# + +FROM alpine:3.9 + +# copy the latest binary +COPY migrate /usr/local/bin/migrate + +ARG BUILD_DATE + +LABEL org.label-schema.name="migrate" +LABEL org.label-schema.description="migrates openebs components" +LABEL org.label-schema.url="https://openebs.io/" +LABEL org.label-schema.vcs-url="https://github.com/openebs/maya" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$BUILD_DATE + +ENTRYPOINT ["migrate"] diff --git a/buildscripts/migrate/Makefile.mk b/buildscripts/migrate/Makefile.mk new file mode 100644 index 0000000000..1afafee4f1 --- /dev/null +++ b/buildscripts/migrate/Makefile.mk @@ -0,0 +1,32 @@ + +# Specify the name for the binaries +MIGRATE=migrate + +# build migrate binary +.PHONY: migrate +migrate: + @echo "----------------------------" + @echo "--> ${MIGRATE} " + @echo "----------------------------" + @# PNAME is the sub-folder in ./bin where binary will be placed. + @# CTLNAME indicates the folder/pkg under cmd that needs to be built. + @# The output binary will be: ./bin/${PNAME}//${CTLNAME} + @# A copy of the binary will also be placed under: ./bin/${PNAME}/${CTLNAME} + @PNAME=${MIGRATE} CTLNAME=${MIGRATE} CGO_ENABLED=0 sh -c "'$(PWD)/buildscripts/build.sh'" + +# build migrate image +.PHONY: migrate-image +migrate-image: migrate + @echo "-----------------------------------------------" + @echo "--> ${MIGRATE} image " + @echo "${HUB_USER}/${M_MIGRATE_REPO_NAME}:${IMAGE_TAG}" + @echo "-----------------------------------------------" + @cp bin/${MIGRATE}/${MIGRATE} buildscripts/${MIGRATE}/ + @cd buildscripts/${MIGRATE} && \ + sudo docker build -t "${HUB_USER}/${M_MIGRATE_REPO_NAME}:${IMAGE_TAG}" --build-arg BUILD_DATE=${BUILD_DATE} . + @rm buildscripts/${MIGRATE}/${MIGRATE} + +# cleanup migrate build +.PHONY: cleanup-migrate +cleanup-migrate: + rm -rf ${GOPATH}/bin/${MIGRATE} diff --git a/cmd/cspi-mgmt/app/handler.go b/cmd/cspi-mgmt/app/handler.go index d34ba4528b..8ef6d39832 100644 --- a/cmd/cspi-mgmt/app/handler.go +++ b/cmd/cspi-mgmt/app/handler.go @@ -59,7 +59,11 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { common.SyncResources.Mux.Lock() // try to import pool - isImported, err = zpool.Import(cspi) + if cspi.Annotations["cspuid"] != "" { + isImported, err = zpool.Import(cspi, "cstor-"+cspi.Annotations["cspuid"]) + } else { + isImported, err = zpool.Import(cspi, "") + } if isImported { if err != nil { common.SyncResources.Mux.Unlock() @@ -71,6 +75,7 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { } zpool.CheckImportedPoolVolume() common.SyncResources.Mux.Unlock() + delete(cspi.Annotations, "cspuid") err = c.update(cspi) if err != nil { c.recorder.Event(cspi, diff --git a/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go b/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go index eb4c257777..078689f956 100644 --- a/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go +++ b/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go @@ -31,7 +31,7 @@ import ( // It will return - // - If pool is imported or not // - If any error occurred during import operation -func Import(cspi *apis.CStorPoolInstance) (bool, error) { +func Import(cspi *apis.CStorPoolInstance, oldName string) (bool, error) { if poolExist := checkIfPoolPresent(PoolName(cspi)); poolExist { return true, nil } @@ -54,7 +54,8 @@ func Import(cspi *apis.CStorPoolInstance) (bool, error) { WithCachefile(cspi.Spec.PoolConfig.CacheFile). WithProperty("cachefile", cspi.Spec.PoolConfig.CacheFile). WithDirectory(devID). - WithPool(PoolName(cspi)). + WithPool(oldName). + WithNewPool(PoolName(cspi)). Execute() if err == nil { poolImported = true diff --git a/cmd/migrate/executor/env.go b/cmd/migrate/executor/env.go new file mode 100644 index 0000000000..5d5e3f91c8 --- /dev/null +++ b/cmd/migrate/executor/env.go @@ -0,0 +1,30 @@ +/* +Copyright 2019 The OpenEBS Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + menv "github.com/openebs/maya/pkg/env/v1alpha1" +) + +//This file defines the environement variable names that are specific +// to this provisioner. In addition to the variables defined in this file, +// provisioner also uses the following: +// OPENEBS_NAMESPACE + +func getOpenEBSNamespace() string { + return menv.Get(menv.OpenEBSNamespace) +} diff --git a/cmd/migrate/executor/error.go b/cmd/migrate/executor/error.go new file mode 100644 index 0000000000..ff6d566999 --- /dev/null +++ b/cmd/migrate/executor/error.go @@ -0,0 +1,34 @@ +/* +Copyright 2019 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + "context" + "fmt" + "os" +) + +// CheckError prints err to stderr and exits with code 1 if err is not nil. Otherwise, it is a +// no-op. +func CheckError(err error) { + if err != nil { + if err != context.Canceled { + fmt.Fprintf(os.Stderr, fmt.Sprintf("An error occurred: %v\n", err)) + } + os.Exit(1) + } +} diff --git a/cmd/migrate/executor/options.go b/cmd/migrate/executor/options.go new file mode 100644 index 0000000000..ef881202af --- /dev/null +++ b/cmd/migrate/executor/options.go @@ -0,0 +1,46 @@ +/* +Copyright 2019 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + "strings" + + errors "github.com/pkg/errors" + + "github.com/spf13/cobra" +) + +// MigrateOptions stores information required for migrate +type MigrateOptions struct { + openebsNamespace string + spcName string +} + +var ( + options = &MigrateOptions{ + openebsNamespace: "openebs", + } +) + +// RunPreFlightChecks will ensure the sanity of the common migrate options +func (u *MigrateOptions) RunPreFlightChecks(cmd *cobra.Command) error { + if len(strings.TrimSpace(u.openebsNamespace)) == 0 { + return errors.Errorf("Cannot execute migrate job: namespace is missing") + } + + return nil +} diff --git a/cmd/migrate/executor/pool.go b/cmd/migrate/executor/pool.go new file mode 100644 index 0000000000..707d48f333 --- /dev/null +++ b/cmd/migrate/executor/pool.go @@ -0,0 +1,80 @@ +/* +Copyright 2019 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + "strings" + + migrate "github.com/openebs/maya/pkg/cstor/migrate" + "github.com/openebs/maya/pkg/util" + "github.com/spf13/cobra" + "k8s.io/klog" + + errors "github.com/pkg/errors" +) + +var ( + cstorSPCMigrateCmdHelpText = ` +This command migrates the cStor SPC + +Usage: migrate pool --spc-name +` +) + +// NewMigratePoolJob migrates all the cStor Pools associated with +// a given Storage Pool Claim +func NewMigratePoolJob() *cobra.Command { + cmd := &cobra.Command{ + Use: "pool", + Short: "Migrate cStor SPC", + Long: cstorSPCMigrateCmdHelpText, + Example: `migrate cstor-spc --spc-name `, + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(options.RunPreFlightChecks(cmd), util.Fatal) + util.CheckErr(options.RunCStorSPCMigrateChecks(cmd), util.Fatal) + util.CheckErr(options.RunCStorSPCMigrate(cmd), util.Fatal) + }, + } + + cmd.Flags().StringVarP(&options.spcName, + "spc-name", "", + options.spcName, + "cstor SPC name to be migrated. Run \"kubectl get spc\", to get spc-name") + + return cmd +} + +// RunCStorSPCMigrateChecks will ensure the sanity of the cstor SPC migrate options +func (u *MigrateOptions) RunCStorSPCMigrateChecks(cmd *cobra.Command) error { + if len(strings.TrimSpace(u.spcName)) == 0 { + return errors.Errorf("Cannot execute migrate job: cstor spc name is missing") + } + + return nil +} + +// RunCStorSPCMigrate migrates the given spc. +func (u *MigrateOptions) RunCStorSPCMigrate(cmd *cobra.Command) error { + + err := migrate.Pool(u.spcName, u.openebsNamespace) + if err != nil { + klog.Error(err) + return errors.Errorf("Failed to migrate cStor SPC : %s", u.spcName) + } + + return nil +} diff --git a/cmd/migrate/executor/setup_job.go b/cmd/migrate/executor/setup_job.go new file mode 100644 index 0000000000..52a0edd1c6 --- /dev/null +++ b/cmd/migrate/executor/setup_job.go @@ -0,0 +1,66 @@ +/* +Copyright 2019 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + "flag" + //"fmt" + //"os" + "strings" + + //"k8s.io/klog" + + "github.com/spf13/cobra" +) + +var spcName, openebsNamespace string + +// NewJob will setup a new migrate job +func NewJob() *cobra.Command { + // Create a new command. + cmd := &cobra.Command{ + Use: "migrate", + Short: "OpenEBS Migrate Utility", + Long: `An utility to migrate OpenEBS SPC Pools to CSPC and Non-CSI Volumes to CSI Volumes, + run as a Kubernetes Job`, + PersistentPreRun: PreRun, + } + + cmd.AddCommand( + NewMigratePoolJob(), + ) + + cmd.PersistentFlags().StringVarP(&options.openebsNamespace, + "openebs-namespace", "", + options.openebsNamespace, + "namespace where openebs components are installed.") + + cmd.PersistentFlags().AddGoFlagSet(flag.CommandLine) + + // Hack: Without the following line, the logs will be prefixed with Error + _ = flag.CommandLine.Parse([]string{}) + + return cmd +} + +// PreRun will check for environement variables to be read and intialized. +func PreRun(cmd *cobra.Command, args []string) { + namespace := getOpenEBSNamespace() + if len(strings.TrimSpace(namespace)) != 0 { + openebsNamespace = namespace + } +} diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 0000000000..e806b60638 --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,31 @@ +/* +Copyright 2018 The OpenEBS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/openebs/maya/cmd/migrate/executor" + mlogger "github.com/openebs/maya/pkg/logs" +) + +func main() { + // Init logging + mlogger.InitLogs() + defer mlogger.FlushLogs() + + err := executor.NewJob().Execute() + executor.CheckError(err) +} diff --git a/pkg/algorithm/nodeselect/v1alpha2/build_csp.go b/pkg/algorithm/nodeselect/v1alpha2/build_csp.go index f85a7f2f38..a14b966b33 100644 --- a/pkg/algorithm/nodeselect/v1alpha2/build_csp.go +++ b/pkg/algorithm/nodeselect/v1alpha2/build_csp.go @@ -70,9 +70,25 @@ func (ac *Config) GetCSPSpec() (*apis.CStorPoolInstance, error) { return nil, errors.Wrapf(err, "failed to build CSP object for node selector {%v}", poolSpec.NodeSelector) } - err = ac.ClaimBDsForNode(ac.GetBDListForNode(poolSpec)) - if err != nil { - return nil, errors.Wrapf(err, "failed to claim block devices for node {%s}", nodeName) + annotations := map[string]string{} + + if ac.CSPC.GetAnnotations()["reconcile.openebs.io/dependants"] == "false" { + annotations[string(apis.OpenEBSDisableReconcileKey)] = "true" + } + + if poolSpec.OldCSPUID != "" { + annotations["cspuid"] = poolSpec.OldCSPUID + } + + if len(annotations) != 0 { + cspObj.Object.SetAnnotations(annotations) + } + + if poolSpec.OldCSPUID == "" { + err = ac.ClaimBDsForNode(ac.GetBDListForNode(poolSpec)) + if err != nil { + return nil, errors.Wrapf(err, "failed to claim block devices for node {%s}", nodeName) + } } return cspObj.Object, nil } diff --git a/pkg/cstor/migrate/cspc_generator.go b/pkg/cstor/migrate/cspc_generator.go index 1cd8b8b401..77173538a3 100644 --- a/pkg/cstor/migrate/cspc_generator.go +++ b/pkg/cstor/migrate/cspc_generator.go @@ -17,10 +17,15 @@ limitations under the License. package migrate import ( + "time" + apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" csp "github.com/openebs/maya/pkg/cstor/pool/v1alpha3" cspc "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1" - spc "github.com/openebs/maya/pkg/storagepoolclaim/v1alpha1" + cspi "github.com/openebs/maya/pkg/cstor/poolinstance/v1alpha3" + "github.com/openebs/maya/pkg/util/retry" + "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -85,18 +90,59 @@ func getCSPCSpec(spc *apis.StoragePoolClaim) (*apis.CStorPoolCluster, error) { return cspcObj, nil } -func generateCSPC(spcName, openebsNamespace string) error { - spcObj, err := spc.NewKubeClient().Get(spcName, metav1.GetOptions{}) +func generateCSPC(spcObj *apis.StoragePoolClaim, openebsNamespace string) ( + *apis.CStorPoolCluster, error) { + cspcObj, err := cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Get(spcObj.Name, metav1.GetOptions{}) + if err == nil { + return cspcObj, nil + } + if !k8serrors.IsNotFound(err) { + return nil, err + } + cspcObj, err = getCSPCSpec(spcObj) + if err != nil { + return nil, err + } + cspcObj, err = cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Create(cspcObj) if err != nil { - return err + return nil, err } - cspcObj, err := getCSPCSpec(spcObj) + err = retry. + Times(60). + Wait(5 * time.Second). + Try(func(attempt uint) error { + cspiList, err1 := cspi.NewKubeClient(). + WithNamespace(openebsNamespace).List( + metav1.ListOptions{ + LabelSelector: string(apis.CStorPoolClusterCPK) + "=" + cspcObj.Name, + }) + if err1 != nil { + return err1 + } + if len(cspiList.Items) != len(cspcObj.Spec.Pools) { + return errors.Errorf("failed to verify cspi count expected: %d got: %d", + len(cspcObj.Spec.Pools), + len(cspiList.Items), + ) + } + return nil + }) if err != nil { - return err + return nil, err } - _, err = cspc.NewKubeClient().WithNamespace(openebsNamespace).Create(cspcObj) + cspcObj, err = cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Get(spcObj.Name, metav1.GetOptions{}) if err != nil { - return err + return nil, err } - return nil + delete(cspcObj.Annotations, "reconcile.openebs.io/dependants") + cspcObj, err = cspc.NewKubeClient(). + WithNamespace(openebsNamespace). + Update(cspcObj) + if err != nil { + return nil, err + } + return cspcObj, nil } diff --git a/pkg/cstor/migrate/migration.go b/pkg/cstor/migrate/migration.go new file mode 100644 index 0000000000..ebdc74418e --- /dev/null +++ b/pkg/cstor/migrate/migration.go @@ -0,0 +1,270 @@ +/* +Copyright 2019 The OpenEBS Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + "fmt" + "time" + + "k8s.io/klog" + + apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" + bd "github.com/openebs/maya/pkg/blockdevice/v1alpha2" + bdc "github.com/openebs/maya/pkg/blockdeviceclaim/v1alpha1" + csp "github.com/openebs/maya/pkg/cstor/pool/v1alpha3" + cspc "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1" + cspi "github.com/openebs/maya/pkg/cstor/poolinstance/v1alpha3" + cvr "github.com/openebs/maya/pkg/cstor/volumereplica/v1alpha1" + deploy "github.com/openebs/maya/pkg/kubernetes/deployment/appsv1/v1alpha1" + pod "github.com/openebs/maya/pkg/kubernetes/pod/v1alpha1" + spc "github.com/openebs/maya/pkg/storagepoolclaim/v1alpha1" + "github.com/openebs/maya/pkg/util/retry" + "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" +) + +const replicaPatch = `{ + "spec": { + "replicas": 0 + } +}` + +// Pool ... +func Pool(spcName, openebsNamespace string) error { + klog.Infof("Migrating spc %s to cspc", spcName) + spcObj, err := spc.NewKubeClient(). + Get(spcName, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + klog.Infof("spc %s not found.", spcName) + _, err := cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Get(spcName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to get equivalent cspc for spc %s", spcName) + } + klog.Infof("spc %s is already migrated to cspc", spcName) + return nil + } + if err != nil { + return err + } + klog.Infof("Creating equivalent cspc for spc %s", spcName) + cspcObj, err := generateCSPC(spcObj, openebsNamespace) + if err != nil { + return err + } + + cspiList, err := cspi.NewKubeClient(). + WithNamespace(openebsNamespace). + List(metav1.ListOptions{ + LabelSelector: string(apis.CStorPoolClusterCPK) + "=" + cspcObj.Name, + }) + if err != nil { + return err + } + + for _, cspiObj := range cspiList.Items { + if cspiObj.Status.Phase != "ONLINE" { + err = csptocspi(&cspiObj, cspcObj, openebsNamespace) + if err != nil { + return err + } + cspcObj, err = cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Get(cspcObj.Name, metav1.GetOptions{}) + for i, poolspec := range cspcObj.Spec.Pools { + if poolspec.NodeSelector[string(apis.HostNameCPK)] == + cspiObj.Labels[string(apis.HostNameCPK)] { + cspcObj.Spec.Pools[i].OldCSPUID = "" + } + } + cspcObj, err = cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Update(cspcObj) + if err != nil { + return err + } + } + } + err = spc.NewKubeClient(). + Delete(spcName, &metav1.DeleteOptions{}) + if err != nil { + return err + } + return nil +} + +func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { + hostnameLabel := string(apis.HostNameCPK) + "=" + cspiObj.Labels[string(apis.HostNameCPK)] + spcLabel := string(apis.StoragePoolClaimCPK) + "=" + cspcObj.Name + cspLabel := hostnameLabel + "," + spcLabel + + cspObj, err := getCSP(cspLabel) + if err != nil { + return err + } + klog.Infof("Migrating csp %s to cspi %s", cspiObj.Name, cspObj.Name) + err = scaleDownDeployment(cspObj, openebsNamespace) + if err != nil { + return err + } + for _, bdName := range cspObj.Spec.Group[0].Item { + err = updateBDC(bdName, cspcObj, openebsNamespace) + if err != nil { + return err + } + } + delete(cspiObj.Annotations, string(apis.OpenEBSDisableReconcileKey)) + cspiObj, err = cspi.NewKubeClient(). + WithNamespace(openebsNamespace). + Update(cspiObj) + if err != nil { + return err + } + err = retry. + Times(60). + Wait(5 * time.Second). + Try(func(attempt uint) error { + cspiObj, err1 := cspi.NewKubeClient(). + WithNamespace(openebsNamespace). + Get(cspiObj.Name, metav1.GetOptions{}) + if err1 != nil { + return err1 + } + if cspiObj.Status.Phase != "ONLINE" { + return errors.Errorf("failed to verify cspi phase expected: Healthy got: %s", + cspiObj.Status.Phase) + } + return nil + }) + if err != nil { + return err + } + err = updateCVRsLabels(cspObj.Name, openebsNamespace, cspiObj) + if err != nil { + return err + } + return nil +} + +func getCSP(cspLabel string) (*apis.CStorPool, error) { + cspClient := csp.KubeClient() + cspList, err := cspClient.List(metav1.ListOptions{ + LabelSelector: cspLabel, + }) + if err != nil { + return nil, err + } + if len(cspList.Items) != 1 { + return nil, fmt.Errorf("Invalid number of pools on one node: %d", len(cspList.Items)) + } + cspObj := cspList.Items[0] + if err != nil { + return nil, err + } + return &cspObj, nil +} + +func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error { + klog.Infof("Scaling down deployemnt %s", cspObj.Name) + cspPod, err := pod.NewKubeClient(). + WithNamespace(openebsNamespace).List( + metav1.ListOptions{ + LabelSelector: "openebs.io/cstor-pool=" + cspObj.Name, + }) + if err != nil { + return err + } + _, err = deploy.NewKubeClient(). + WithNamespace(openebsNamespace).Patch( + cspObj.Name, + types.StrategicMergePatchType, + []byte(replicaPatch), + ) + err = retry. + Times(60). + Wait(5 * time.Second). + Try(func(attempt uint) error { + _, err1 := pod.NewKubeClient(). + WithNamespace(openebsNamespace). + Get(cspPod.Items[0].Name, metav1.GetOptions{}) + if !k8serrors.IsNotFound(err1) { + return errors.Errorf("failed to get csp pod because %s", err1) + } + return nil + }) + if err != nil { + return err + } + return nil +} + +func updateBDC(bdName apis.CspBlockDevice, cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { + bdObj, err := bd.NewKubeClient(). + WithNamespace(openebsNamespace). + Get(bdName.Name, metav1.GetOptions{}) + if err != nil { + return err + } + bdcObj, err := bdc.NewKubeClient(). + WithNamespace(openebsNamespace). + Get(bdObj.Spec.ClaimRef.Name, metav1.GetOptions{}) + if err != nil { + return err + } + klog.Infof("Updating bdc %s with cspc %s info.", bdcObj.Name, cspcObj.Name) + delete(bdcObj.Labels, string(apis.StoragePoolClaimCPK)) + bdcObj.Labels[string(apis.CStorPoolClusterCPK)] = cspcObj.Name + for i, finalizer := range bdcObj.Finalizers { + if finalizer == "storagepoolclaim.openebs.io/finalizer" { + bdcObj.Finalizers[i] = "cstorpoolcluster.openebs.io/finalizer" + } + } + bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" + bdcObj.OwnerReferences[0].UID = cspcObj.UID + bdcObj, err = bdc.NewKubeClient(). + WithNamespace(openebsNamespace). + Update(bdcObj) + if err != nil { + return err + } + return nil +} + +func updateCVRsLabels(cspName, openebsNamespace string, cspiObj *apis.CStorPoolInstance) error { + cvrList, err := cvr.NewKubeclient(). + WithNamespace(openebsNamespace).List(metav1.ListOptions{ + LabelSelector: "cstorpool.openebs.io/name=" + cspName, + }) + if err != nil { + return err + } + for _, cvrObj := range cvrList.Items { + klog.Infof("Updating cvr %s with cspi %s info.", cvrObj.Name, cspiObj.Name) + delete(cvrObj.Labels, "cstorpool.openebs.io/name") + delete(cvrObj.Labels, "cstorpool.openebs.io/uid") + cvrObj.Labels["cstorpoolinstance.openebs.io/name"] = cspiObj.Name + cvrObj.Labels["cstorpoolinstance.openebs.io/uid"] = string(cspiObj.UID) + delete(cvrObj.Annotations, "cstorpool.openebs.io/hostname") + cvrObj.Annotations["cstorpoolinstance.openebs.io/hostname"] = cspiObj.Spec.HostName + _, err = cvr.NewKubeclient().WithNamespace(openebsNamespace). + Update(&cvrObj) + if err != nil { + return err + } + } + return nil +} From 5b63cad3b0ad6c3ad399251411bc68ee38a34c27 Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 28 Nov 2019 16:56:10 +0530 Subject: [PATCH 03/22] fix linting issues Signed-off-by: shubham --- cmd/migrate/executor/setup_job.go | 4 +--- pkg/cstor/migrate/migration.go | 29 ++++++++++++++++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/cmd/migrate/executor/setup_job.go b/cmd/migrate/executor/setup_job.go index 52a0edd1c6..dccb58216d 100644 --- a/cmd/migrate/executor/setup_job.go +++ b/cmd/migrate/executor/setup_job.go @@ -27,8 +27,6 @@ import ( "github.com/spf13/cobra" ) -var spcName, openebsNamespace string - // NewJob will setup a new migrate job func NewJob() *cobra.Command { // Create a new command. @@ -61,6 +59,6 @@ func NewJob() *cobra.Command { func PreRun(cmd *cobra.Command, args []string) { namespace := getOpenEBSNamespace() if len(strings.TrimSpace(namespace)) != 0 { - openebsNamespace = namespace + options.openebsNamespace = namespace } } diff --git a/pkg/cstor/migrate/migration.go b/pkg/cstor/migrate/migration.go index ebdc74418e..74680d92c4 100644 --- a/pkg/cstor/migrate/migration.go +++ b/pkg/cstor/migrate/migration.go @@ -52,7 +52,7 @@ func Pool(spcName, openebsNamespace string) error { Get(spcName, metav1.GetOptions{}) if k8serrors.IsNotFound(err) { klog.Infof("spc %s not found.", spcName) - _, err := cspc.NewKubeClient(). + _, err = cspc.NewKubeClient(). WithNamespace(openebsNamespace).Get(spcName, metav1.GetOptions{}) if err != nil { return errors.Wrapf(err, "failed to get equivalent cspc for spc %s", spcName) @@ -86,6 +86,9 @@ func Pool(spcName, openebsNamespace string) error { } cspcObj, err = cspc.NewKubeClient(). WithNamespace(openebsNamespace).Get(cspcObj.Name, metav1.GetOptions{}) + if err != nil { + return err + } for i, poolspec := range cspcObj.Spec.Pools { if poolspec.NodeSelector[string(apis.HostNameCPK)] == cspiObj.Labels[string(apis.HostNameCPK)] { @@ -111,7 +114,7 @@ func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, hostnameLabel := string(apis.HostNameCPK) + "=" + cspiObj.Labels[string(apis.HostNameCPK)] spcLabel := string(apis.StoragePoolClaimCPK) + "=" + cspcObj.Name cspLabel := hostnameLabel + "," + spcLabel - + var err1 error cspObj, err := getCSP(cspLabel) if err != nil { return err @@ -138,7 +141,7 @@ func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, Times(60). Wait(5 * time.Second). Try(func(attempt uint) error { - cspiObj, err1 := cspi.NewKubeClient(). + cspiObj, err1 = cspi.NewKubeClient(). WithNamespace(openebsNamespace). Get(cspiObj.Name, metav1.GetOptions{}) if err1 != nil { @@ -172,9 +175,6 @@ func getCSP(cspLabel string) (*apis.CStorPool, error) { return nil, fmt.Errorf("Invalid number of pools on one node: %d", len(cspList.Items)) } cspObj := cspList.Items[0] - if err != nil { - return nil, err - } return &cspObj, nil } @@ -188,12 +188,15 @@ func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error if err != nil { return err } - _, err = deploy.NewKubeClient(). - WithNamespace(openebsNamespace).Patch( - cspObj.Name, - types.StrategicMergePatchType, - []byte(replicaPatch), - ) + _, err = deploy.NewKubeClient().WithNamespace(openebsNamespace). + Patch( + cspObj.Name, + types.StrategicMergePatchType, + []byte(replicaPatch), + ) + if err != nil { + return err + } err = retry. Times(60). Wait(5 * time.Second). @@ -235,7 +238,7 @@ func updateBDC(bdName apis.CspBlockDevice, cspcObj *apis.CStorPoolCluster, opene } bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" bdcObj.OwnerReferences[0].UID = cspcObj.UID - bdcObj, err = bdc.NewKubeClient(). + _, err = bdc.NewKubeClient(). WithNamespace(openebsNamespace). Update(bdcObj) if err != nil { From 3d62a275147bf99357f1cc2bec8072f233c5df8a Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 28 Nov 2019 18:04:39 +0530 Subject: [PATCH 04/22] renamed one file Signed-off-by: shubham --- cmd/migrate/executor/pool.go | 2 ++ pkg/cstor/migrate/{migration.go => pool.go} | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) rename pkg/cstor/migrate/{migration.go => pool.go} (99%) diff --git a/cmd/migrate/executor/pool.go b/cmd/migrate/executor/pool.go index 707d48f333..48c278c051 100644 --- a/cmd/migrate/executor/pool.go +++ b/cmd/migrate/executor/pool.go @@ -70,11 +70,13 @@ func (u *MigrateOptions) RunCStorSPCMigrateChecks(cmd *cobra.Command) error { // RunCStorSPCMigrate migrates the given spc. func (u *MigrateOptions) RunCStorSPCMigrate(cmd *cobra.Command) error { + klog.Infof("Migrating spc %s to cspc", u.spcName) err := migrate.Pool(u.spcName, u.openebsNamespace) if err != nil { klog.Error(err) return errors.Errorf("Failed to migrate cStor SPC : %s", u.spcName) } + klog.Infof("Successfully migrated spc %s to cspc", u.spcName) return nil } diff --git a/pkg/cstor/migrate/migration.go b/pkg/cstor/migrate/pool.go similarity index 99% rename from pkg/cstor/migrate/migration.go rename to pkg/cstor/migrate/pool.go index 74680d92c4..52bbf0b50f 100644 --- a/pkg/cstor/migrate/migration.go +++ b/pkg/cstor/migrate/pool.go @@ -47,7 +47,7 @@ const replicaPatch = `{ // Pool ... func Pool(spcName, openebsNamespace string) error { - klog.Infof("Migrating spc %s to cspc", spcName) + spcObj, err := spc.NewKubeClient(). Get(spcName, metav1.GetOptions{}) if k8serrors.IsNotFound(err) { From f1e0f7e18d8ef31eda98a42c9a793647d7954cee Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 28 Nov 2019 18:28:09 +0530 Subject: [PATCH 05/22] added idempotent support Signed-off-by: shubham --- pkg/cstor/migrate/pool.go | 48 ++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/pkg/cstor/migrate/pool.go b/pkg/cstor/migrate/pool.go index 52bbf0b50f..325e6ea6f3 100644 --- a/pkg/cstor/migrate/pool.go +++ b/pkg/cstor/migrate/pool.go @@ -188,29 +188,31 @@ func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error if err != nil { return err } - _, err = deploy.NewKubeClient().WithNamespace(openebsNamespace). - Patch( - cspObj.Name, - types.StrategicMergePatchType, - []byte(replicaPatch), - ) - if err != nil { - return err - } - err = retry. - Times(60). - Wait(5 * time.Second). - Try(func(attempt uint) error { - _, err1 := pod.NewKubeClient(). - WithNamespace(openebsNamespace). - Get(cspPod.Items[0].Name, metav1.GetOptions{}) - if !k8serrors.IsNotFound(err1) { - return errors.Errorf("failed to get csp pod because %s", err1) - } - return nil - }) - if err != nil { - return err + if len(cspPod.Items) > 0 { + _, err = deploy.NewKubeClient().WithNamespace(openebsNamespace). + Patch( + cspObj.Name, + types.StrategicMergePatchType, + []byte(replicaPatch), + ) + if err != nil { + return err + } + err = retry. + Times(60). + Wait(5 * time.Second). + Try(func(attempt uint) error { + _, err1 := pod.NewKubeClient(). + WithNamespace(openebsNamespace). + Get(cspPod.Items[0].Name, metav1.GetOptions{}) + if !k8serrors.IsNotFound(err1) { + return errors.Errorf("failed to get csp pod because %s", err1) + } + return nil + }) + if err != nil { + return err + } } return nil } From 18a5e6e1e4c0f1c3c6e6929d74af3fe1f672397e Mon Sep 17 00:00:00 2001 From: shubham Date: Fri, 29 Nov 2019 10:35:14 +0530 Subject: [PATCH 06/22] moved hardcoded strings in function calls to global constants Signed-off-by: shubham --- cmd/cspi-mgmt/app/handler.go | 6 ++-- .../nodeselect/v1alpha2/build_csp.go | 4 +-- pkg/apis/openebs.io/v1alpha1/cas_keys.go | 4 +++ pkg/apis/openebs.io/v1alpha1/cas_pool.go | 2 ++ pkg/cstor/migrate/cspc_generator.go | 4 +-- pkg/cstor/migrate/pool.go | 30 ++++++++++++------- 6 files changed, 33 insertions(+), 17 deletions(-) diff --git a/cmd/cspi-mgmt/app/handler.go b/cmd/cspi-mgmt/app/handler.go index 8ef6d39832..5211860fd0 100644 --- a/cmd/cspi-mgmt/app/handler.go +++ b/cmd/cspi-mgmt/app/handler.go @@ -59,8 +59,8 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { common.SyncResources.Mux.Lock() // try to import pool - if cspi.Annotations["cspuid"] != "" { - isImported, err = zpool.Import(cspi, "cstor-"+cspi.Annotations["cspuid"]) + if cspi.Annotations[string(apis.OldCSPUID)] != "" { + isImported, err = zpool.Import(cspi, "cstor-"+cspi.Annotations[string(apis.OldCSPUID)]) } else { isImported, err = zpool.Import(cspi, "") } @@ -75,7 +75,7 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { } zpool.CheckImportedPoolVolume() common.SyncResources.Mux.Unlock() - delete(cspi.Annotations, "cspuid") + delete(cspi.Annotations, string(apis.OldCSPUID)) err = c.update(cspi) if err != nil { c.recorder.Event(cspi, diff --git a/pkg/algorithm/nodeselect/v1alpha2/build_csp.go b/pkg/algorithm/nodeselect/v1alpha2/build_csp.go index a14b966b33..fcceb5b19c 100644 --- a/pkg/algorithm/nodeselect/v1alpha2/build_csp.go +++ b/pkg/algorithm/nodeselect/v1alpha2/build_csp.go @@ -72,12 +72,12 @@ func (ac *Config) GetCSPSpec() (*apis.CStorPoolInstance, error) { annotations := map[string]string{} - if ac.CSPC.GetAnnotations()["reconcile.openebs.io/dependants"] == "false" { + if ac.CSPC.GetAnnotations()[string(apis.OpenEBSDisableDependantsReconcileKey)] == "false" { annotations[string(apis.OpenEBSDisableReconcileKey)] = "true" } if poolSpec.OldCSPUID != "" { - annotations["cspuid"] = poolSpec.OldCSPUID + annotations[string(apis.OldCSPUID)] = poolSpec.OldCSPUID } if len(annotations) != 0 { diff --git a/pkg/apis/openebs.io/v1alpha1/cas_keys.go b/pkg/apis/openebs.io/v1alpha1/cas_keys.go index 4ccef89e93..8b30777c33 100644 --- a/pkg/apis/openebs.io/v1alpha1/cas_keys.go +++ b/pkg/apis/openebs.io/v1alpha1/cas_keys.go @@ -48,6 +48,10 @@ const ( // OpenEBSDisableReconcileKey is the label key decides to reconcile or not OpenEBSDisableReconcileKey CASKey = "reconcile.openebs.io/disable" + // OpenEBSDisableDependantsReconcileKey is the annotation key that decides to create + // children objects with OpenEBSDisableReconcileKey as true or false + OpenEBSDisableDependantsReconcileKey CASKey = "reconcile.openebs.io/dependants" + // CASConfigKey is the key to fetch configurations w.r.t a CAS entity CASConfigKey CASKey = "cas.openebs.io/config" diff --git a/pkg/apis/openebs.io/v1alpha1/cas_pool.go b/pkg/apis/openebs.io/v1alpha1/cas_pool.go index 9fe10af905..e04bf4c279 100644 --- a/pkg/apis/openebs.io/v1alpha1/cas_pool.go +++ b/pkg/apis/openebs.io/v1alpha1/cas_pool.go @@ -66,6 +66,8 @@ const ( RaidzBlockDeviceCountCPV CasPoolValInt = 3 // Raidz2BlockDeviceCountCPV is the count for raidz2 type pool Raidz2BlockDeviceCountCPV CasPoolValInt = 6 + // OldCSPUID is the uid of the csp that is being migrated to cspi + OldCSPUID CasPoolKey = "cspuid" ) // CasPool is a type which will be utilised by CAS engine to perform diff --git a/pkg/cstor/migrate/cspc_generator.go b/pkg/cstor/migrate/cspc_generator.go index 77173538a3..fcb80f8499 100644 --- a/pkg/cstor/migrate/cspc_generator.go +++ b/pkg/cstor/migrate/cspc_generator.go @@ -63,7 +63,7 @@ func getCSPCSpec(spc *apis.StoragePoolClaim) (*apis.CStorPoolCluster, error) { cspcObj.Annotations = map[string]string{ // This label will be used to disable reconciliation on the dependants // In this case that will be CSPI - "reconcile.openebs.io/dependants": "false", + string(apis.OpenEBSDisableDependantsReconcileKey): "false", } for _, cspObj := range cspList.Items { cspcObj.Spec.Pools = append(cspcObj.Spec.Pools, @@ -137,7 +137,7 @@ func generateCSPC(spcObj *apis.StoragePoolClaim, openebsNamespace string) ( if err != nil { return nil, err } - delete(cspcObj.Annotations, "reconcile.openebs.io/dependants") + delete(cspcObj.Annotations, string(apis.OpenEBSDisableDependantsReconcileKey)) cspcObj, err = cspc.NewKubeClient(). WithNamespace(openebsNamespace). Update(cspcObj) diff --git a/pkg/cstor/migrate/pool.go b/pkg/cstor/migrate/pool.go index 325e6ea6f3..b8294b7ceb 100644 --- a/pkg/cstor/migrate/pool.go +++ b/pkg/cstor/migrate/pool.go @@ -39,11 +39,21 @@ import ( types "k8s.io/apimachinery/pkg/types" ) -const replicaPatch = `{ +const ( + replicaPatch = `{ "spec": { "replicas": 0 } }` + cspNameLabel = "cstorpool.openebs.io/name" + cspUIDLabel = "cstorpool.openebs.io/uid" + cspHostnameAnnotation = "cstorpool.openebs.io/hostname" + cspiNameLabel = "cstorpoolinstance.openebs.io/name" + cspiUIDLabel = "cstorpoolinstance.openebs.io/uid" + cspiHostnameAnnotation = "cstorpoolinstance.openebs.io/hostname" + spcFinalizer = "storagepoolclaim.openebs.io/finalizer" + cspcFinalizer = "cstorpoolcluster.openebs.io/finalizer" +) // Pool ... func Pool(spcName, openebsNamespace string) error { @@ -234,8 +244,8 @@ func updateBDC(bdName apis.CspBlockDevice, cspcObj *apis.CStorPoolCluster, opene delete(bdcObj.Labels, string(apis.StoragePoolClaimCPK)) bdcObj.Labels[string(apis.CStorPoolClusterCPK)] = cspcObj.Name for i, finalizer := range bdcObj.Finalizers { - if finalizer == "storagepoolclaim.openebs.io/finalizer" { - bdcObj.Finalizers[i] = "cstorpoolcluster.openebs.io/finalizer" + if finalizer == spcFinalizer { + bdcObj.Finalizers[i] = cspcFinalizer } } bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" @@ -252,19 +262,19 @@ func updateBDC(bdName apis.CspBlockDevice, cspcObj *apis.CStorPoolCluster, opene func updateCVRsLabels(cspName, openebsNamespace string, cspiObj *apis.CStorPoolInstance) error { cvrList, err := cvr.NewKubeclient(). WithNamespace(openebsNamespace).List(metav1.ListOptions{ - LabelSelector: "cstorpool.openebs.io/name=" + cspName, + LabelSelector: cspNameLabel + "=" + cspName, }) if err != nil { return err } for _, cvrObj := range cvrList.Items { klog.Infof("Updating cvr %s with cspi %s info.", cvrObj.Name, cspiObj.Name) - delete(cvrObj.Labels, "cstorpool.openebs.io/name") - delete(cvrObj.Labels, "cstorpool.openebs.io/uid") - cvrObj.Labels["cstorpoolinstance.openebs.io/name"] = cspiObj.Name - cvrObj.Labels["cstorpoolinstance.openebs.io/uid"] = string(cspiObj.UID) - delete(cvrObj.Annotations, "cstorpool.openebs.io/hostname") - cvrObj.Annotations["cstorpoolinstance.openebs.io/hostname"] = cspiObj.Spec.HostName + delete(cvrObj.Labels, cspNameLabel) + delete(cvrObj.Labels, cspUIDLabel) + cvrObj.Labels[cspiNameLabel] = cspiObj.Name + cvrObj.Labels[cspiUIDLabel] = string(cspiObj.UID) + delete(cvrObj.Annotations, cspHostnameAnnotation) + cvrObj.Annotations[cspiHostnameAnnotation] = cspiObj.Spec.HostName _, err = cvr.NewKubeclient().WithNamespace(openebsNamespace). Update(&cvrObj) if err != nil { From a6e1356e7b251a61ceeff09f9d2beb9e3495b671 Mon Sep 17 00:00:00 2001 From: shubham Date: Fri, 29 Nov 2019 17:32:20 +0530 Subject: [PATCH 07/22] added comments for the function and removed unnecessary checks Signed-off-by: shubham --- cmd/cspi-mgmt/app/handler.go | 4 +++- cmd/cstor-pool-mgmt/pool/v1alpha2/import.go | 2 ++ cmd/migrate/executor/options.go | 1 - cmd/migrate/executor/setup_job.go | 4 ---- .../nodeselect/v1alpha2/build_csp.go | 16 +++++-------- pkg/cstor/migrate/cspc_generator.go | 5 ++++ pkg/cstor/migrate/pool.go | 23 +++++++++++++++++-- 7 files changed, 37 insertions(+), 18 deletions(-) diff --git a/cmd/cspi-mgmt/app/handler.go b/cmd/cspi-mgmt/app/handler.go index 5211860fd0..da1d2d7d3a 100644 --- a/cmd/cspi-mgmt/app/handler.go +++ b/cmd/cspi-mgmt/app/handler.go @@ -58,7 +58,9 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { // take a lock for common package for updating variables common.SyncResources.Mux.Lock() - // try to import pool + // try to import pool, first check for migration case in which the previous + // poolname is CSP uid and should be renamed to CSPC uid. If old CSP uid is + // not present then follow the normal code path. if cspi.Annotations[string(apis.OldCSPUID)] != "" { isImported, err = zpool.Import(cspi, "cstor-"+cspi.Annotations[string(apis.OldCSPUID)]) } else { diff --git a/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go b/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go index 078689f956..35ed0c4f0d 100644 --- a/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go +++ b/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go @@ -31,6 +31,8 @@ import ( // It will return - // - If pool is imported or not // - If any error occurred during import operation +// The oldName argument is used to rename the old CSP +// uid name to the current PoolName(cspi) format func Import(cspi *apis.CStorPoolInstance, oldName string) (bool, error) { if poolExist := checkIfPoolPresent(PoolName(cspi)); poolExist { return true, nil diff --git a/cmd/migrate/executor/options.go b/cmd/migrate/executor/options.go index ef881202af..a5ebdd7422 100644 --- a/cmd/migrate/executor/options.go +++ b/cmd/migrate/executor/options.go @@ -41,6 +41,5 @@ func (u *MigrateOptions) RunPreFlightChecks(cmd *cobra.Command) error { if len(strings.TrimSpace(u.openebsNamespace)) == 0 { return errors.Errorf("Cannot execute migrate job: namespace is missing") } - return nil } diff --git a/cmd/migrate/executor/setup_job.go b/cmd/migrate/executor/setup_job.go index dccb58216d..cadd800aa2 100644 --- a/cmd/migrate/executor/setup_job.go +++ b/cmd/migrate/executor/setup_job.go @@ -18,12 +18,8 @@ package executor import ( "flag" - //"fmt" - //"os" "strings" - //"k8s.io/klog" - "github.com/spf13/cobra" ) diff --git a/pkg/algorithm/nodeselect/v1alpha2/build_csp.go b/pkg/algorithm/nodeselect/v1alpha2/build_csp.go index fcceb5b19c..1a35029bd2 100644 --- a/pkg/algorithm/nodeselect/v1alpha2/build_csp.go +++ b/pkg/algorithm/nodeselect/v1alpha2/build_csp.go @@ -70,20 +70,16 @@ func (ac *Config) GetCSPSpec() (*apis.CStorPoolInstance, error) { return nil, errors.Wrapf(err, "failed to build CSP object for node selector {%v}", poolSpec.NodeSelector) } - annotations := map[string]string{} - + // check for OpenEBSDisableDependantsReconcileKey annotation which implies + // the CSPI should have OpenEBSDisableReconcileKey enabled if ac.CSPC.GetAnnotations()[string(apis.OpenEBSDisableDependantsReconcileKey)] == "false" { - annotations[string(apis.OpenEBSDisableReconcileKey)] = "true" + cspObj.Object.Annotations[string(apis.OpenEBSDisableReconcileKey)] = "true" } - + // if old CSP has the pool with its name add annotation for renaming while importing if poolSpec.OldCSPUID != "" { - annotations[string(apis.OldCSPUID)] = poolSpec.OldCSPUID + cspObj.Object.Annotations[string(apis.OldCSPUID)] = poolSpec.OldCSPUID } - - if len(annotations) != 0 { - cspObj.Object.SetAnnotations(annotations) - } - + // if old CSP is present, it implies the BDs are already claimed for the pool if poolSpec.OldCSPUID == "" { err = ac.ClaimBDsForNode(ac.GetBDListForNode(poolSpec)) if err != nil { diff --git a/pkg/cstor/migrate/cspc_generator.go b/pkg/cstor/migrate/cspc_generator.go index fcb80f8499..d8ae6f0c5d 100644 --- a/pkg/cstor/migrate/cspc_generator.go +++ b/pkg/cstor/migrate/cspc_generator.go @@ -90,6 +90,7 @@ func getCSPCSpec(spc *apis.StoragePoolClaim) (*apis.CStorPoolCluster, error) { return cspcObj, nil } +// generateCSPC creates an equivalent cspc for the given spc object func generateCSPC(spcObj *apis.StoragePoolClaim, openebsNamespace string) ( *apis.CStorPoolCluster, error) { cspcObj, err := cspc.NewKubeClient(). @@ -137,6 +138,10 @@ func generateCSPC(spcObj *apis.StoragePoolClaim, openebsNamespace string) ( if err != nil { return nil, err } + // after all the cspi come up which reconcilation disabled delete the + // OpenEBSDisableDependantsReconcileKey annotation so that in future when + // a cspi is delete and it comes back on reconciliation it should not have + // reconciliation disabled delete(cspcObj.Annotations, string(apis.OpenEBSDisableDependantsReconcileKey)) cspcObj, err = cspc.NewKubeClient(). WithNamespace(openebsNamespace). diff --git a/pkg/cstor/migrate/pool.go b/pkg/cstor/migrate/pool.go index b8294b7ceb..2898f54c50 100644 --- a/pkg/cstor/migrate/pool.go +++ b/pkg/cstor/migrate/pool.go @@ -55,11 +55,12 @@ const ( cspcFinalizer = "cstorpoolcluster.openebs.io/finalizer" ) -// Pool ... +// Pool migrates the pool from SPC schema to CSPC schema func Pool(spcName, openebsNamespace string) error { spcObj, err := spc.NewKubeClient(). Get(spcName, metav1.GetOptions{}) + // verify if the spc is already migrated. if k8serrors.IsNotFound(err) { klog.Infof("spc %s not found.", spcName) _, err = cspc.NewKubeClient(). @@ -78,7 +79,7 @@ func Pool(spcName, openebsNamespace string) error { if err != nil { return err } - + // List all cspi created with reconcile off cspiList, err := cspi.NewKubeClient(). WithNamespace(openebsNamespace). List(metav1.ListOptions{ @@ -88,7 +89,11 @@ func Pool(spcName, openebsNamespace string) error { return err } + // For each cspi perform the migration from csp that present was on + // node on which cspi came up. for _, cspiObj := range cspiList.Items { + // Skip the migration if cspi is already in ONLINE state, + // which implies the migration is done and makes it idempotent if cspiObj.Status.Phase != "ONLINE" { err = csptocspi(&cspiObj, cspcObj, openebsNamespace) if err != nil { @@ -99,6 +104,8 @@ func Pool(spcName, openebsNamespace string) error { if err != nil { return err } + // remove the OldCSPUID name to avoid renaming in case + // cspi is deleted and comes up after reconciliation. for i, poolspec := range cspcObj.Spec.Pools { if poolspec.NodeSelector[string(apis.HostNameCPK)] == cspiObj.Labels[string(apis.HostNameCPK)] { @@ -112,6 +119,7 @@ func Pool(spcName, openebsNamespace string) error { } } } + // Clean up old SPC resources after the migration is complete err = spc.NewKubeClient(). Delete(spcName, &metav1.DeleteOptions{}) if err != nil { @@ -120,6 +128,7 @@ func Pool(spcName, openebsNamespace string) error { return nil } +// csptocspi migrates a CSP to CSPI based on hostname func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { hostnameLabel := string(apis.HostNameCPK) + "=" + cspiObj.Labels[string(apis.HostNameCPK)] spcLabel := string(apis.StoragePoolClaimCPK) + "=" + cspcObj.Name @@ -140,6 +149,8 @@ func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, return err } } + // once the old pool pod is scaled down and bdcs are patched + // bring up the cspi pod so that the old pool can be renamed and imported. delete(cspiObj.Annotations, string(apis.OpenEBSDisableReconcileKey)) cspiObj, err = cspi.NewKubeClient(). WithNamespace(openebsNamespace). @@ -173,6 +184,8 @@ func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, return nil } +// get csp for cspi on the basis of cspLabel, which is the combination of +// hostname label on which cspi came up and the spc label. func getCSP(cspLabel string) (*apis.CStorPool, error) { cspClient := csp.KubeClient() cspList, err := cspClient.List(metav1.ListOptions{ @@ -188,6 +201,8 @@ func getCSP(cspLabel string) (*apis.CStorPool, error) { return &cspObj, nil } +// The old pool pod should be scaled down before the new cspi pod comes up +// to avoid importing the pool at two places at the same time. func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error { klog.Infof("Scaling down deployemnt %s", cspObj.Name) cspPod, err := pod.NewKubeClient(). @@ -227,6 +242,8 @@ func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error return nil } +// Update the bdc with the cspc labels instead of spc labels to allow +// filtering of bds claimed by the migrated cspc. func updateBDC(bdName apis.CspBlockDevice, cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { bdObj, err := bd.NewKubeClient(). WithNamespace(openebsNamespace). @@ -259,6 +276,8 @@ func updateBDC(bdName apis.CspBlockDevice, cspcObj *apis.CStorPoolCluster, opene return nil } +// Update the cvrs on the old csp with the migrated cspi labels and annotations +// to allow backward compatibility with old external provisioned volumes. func updateCVRsLabels(cspName, openebsNamespace string, cspiObj *apis.CStorPoolInstance) error { cvrList, err := cvr.NewKubeclient(). WithNamespace(openebsNamespace).List(metav1.ListOptions{ From 206c214434c6bf3c1951e67ed19dd1cfdadbc895 Mon Sep 17 00:00:00 2001 From: shubham Date: Tue, 10 Dec 2019 16:30:35 +0530 Subject: [PATCH 08/22] This commit chnages the design from schema changes to use of annotations: - removed oldCSPUID schema change - added two generic annotation which can be used in different scenarios - renamed one function according to the functionality Signed-off-by: shubham --- cmd/cspi-mgmt/app/handler.go | 11 +- cmd/cstor-pool-mgmt/pool/v1alpha2/create.go | 2 +- cmd/cstor-pool-mgmt/pool/v1alpha2/import.go | 34 +++-- .../pool/v1alpha2/pool_utils.go | 5 +- .../nodeselect/v1alpha2/build_csp.go | 28 ++-- pkg/apis/openebs.io/v1alpha1/cas_pool.go | 4 +- .../openebs.io/v1alpha1/cstorpool_cluster.go | 3 - pkg/cstor/migrate/cspc_generator.go | 1 - pkg/cstor/migrate/pool.go | 138 ++++++++++-------- 9 files changed, 124 insertions(+), 102 deletions(-) diff --git a/cmd/cspi-mgmt/app/handler.go b/cmd/cspi-mgmt/app/handler.go index da1d2d7d3a..d34ba4528b 100644 --- a/cmd/cspi-mgmt/app/handler.go +++ b/cmd/cspi-mgmt/app/handler.go @@ -58,14 +58,8 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { // take a lock for common package for updating variables common.SyncResources.Mux.Lock() - // try to import pool, first check for migration case in which the previous - // poolname is CSP uid and should be renamed to CSPC uid. If old CSP uid is - // not present then follow the normal code path. - if cspi.Annotations[string(apis.OldCSPUID)] != "" { - isImported, err = zpool.Import(cspi, "cstor-"+cspi.Annotations[string(apis.OldCSPUID)]) - } else { - isImported, err = zpool.Import(cspi, "") - } + // try to import pool + isImported, err = zpool.Import(cspi) if isImported { if err != nil { common.SyncResources.Mux.Unlock() @@ -77,7 +71,6 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { } zpool.CheckImportedPoolVolume() common.SyncResources.Mux.Unlock() - delete(cspi.Annotations, string(apis.OldCSPUID)) err = c.update(cspi) if err != nil { c.recorder.Event(cspi, diff --git a/cmd/cstor-pool-mgmt/pool/v1alpha2/create.go b/cmd/cstor-pool-mgmt/pool/v1alpha2/create.go index 63321d3632..8d1a3fe256 100644 --- a/cmd/cstor-pool-mgmt/pool/v1alpha2/create.go +++ b/cmd/cstor-pool-mgmt/pool/v1alpha2/create.go @@ -29,7 +29,7 @@ func Create(csp *apis.CStorPoolInstance) error { // Let's check if there is any disk having the pool config // If so then we will not create the pool - ret, notImported, err := checkIfPoolNotImported(csp) + ret, notImported, err := checkIfPoolIsImportable(csp) if err != nil { return errors.Errorf("Failed to check not imported pool %s", err.Error()) } diff --git a/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go b/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go index 35ed0c4f0d..649355657a 100644 --- a/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go +++ b/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go @@ -33,7 +33,7 @@ import ( // - If any error occurred during import operation // The oldName argument is used to rename the old CSP // uid name to the current PoolName(cspi) format -func Import(cspi *apis.CStorPoolInstance, oldName string) (bool, error) { +func Import(cspi *apis.CStorPoolInstance) (bool, error) { if poolExist := checkIfPoolPresent(PoolName(cspi)); poolExist { return true, nil } @@ -42,7 +42,14 @@ func Import(cspi *apis.CStorPoolInstance, oldName string) (bool, error) { var cmdOut []byte var err error common.SyncResources.IsImported = false - var poolImported bool + var poolImported, poolNotImported bool + + _, poolNotImported, err = checkIfPoolIsImportable(cspi) + if poolNotImported { + // if the pool is renamed but not imported remove the + // annotation to avoid not found errors + delete(cspi.Annotations, string(apis.OldPoolName)) + } bdPath, err := getPathForBDev(cspi.Spec.RaidGroups[0].BlockDevices[0].BlockDeviceName) if err != nil { @@ -51,14 +58,19 @@ func Import(cspi *apis.CStorPoolInstance, oldName string) (bool, error) { klog.Infof("Importing pool %s %s", string(cspi.GetUID()), PoolName(cspi)) devID := pool.GetDevPathIfNotSlashDev(bdPath[0]) + cmd := zfs.NewPoolImport(). + WithCachefile(cspi.Spec.PoolConfig.CacheFile). + WithProperty("cachefile", cspi.Spec.PoolConfig.CacheFile). + WithDirectory(devID). + WithNewPool(PoolName(cspi)) + // oldName denotes the pool name that may be present + // from previous version and needs to be imported with new name + oldName := cspi.Annotations[string(apis.OldPoolName)] + if oldName != "" { + cmd.WithPool(oldName) + } if len(devID) != 0 { - cmdOut, err = zfs.NewPoolImport(). - WithCachefile(cspi.Spec.PoolConfig.CacheFile). - WithProperty("cachefile", cspi.Spec.PoolConfig.CacheFile). - WithDirectory(devID). - WithPool(oldName). - WithNewPool(PoolName(cspi)). - Execute() + cmdOut, err = cmd.Execute() if err == nil { poolImported = true } else { @@ -83,6 +95,10 @@ func Import(cspi *apis.CStorPoolInstance, oldName string) (bool, error) { } klog.Infof("Pool Import successful: %v", string(PoolName(cspi))) + // after successful import of pool the annotation needs to be deleted + // to avoid renaming of pool that is already renamed which will cause + // pool not found errors + delete(cspi.Annotations, string(apis.OldPoolName)) return true, nil } diff --git a/cmd/cstor-pool-mgmt/pool/v1alpha2/pool_utils.go b/cmd/cstor-pool-mgmt/pool/v1alpha2/pool_utils.go index f3127e3998..3232f7eb14 100644 --- a/cmd/cstor-pool-mgmt/pool/v1alpha2/pool_utils.go +++ b/cmd/cstor-pool-mgmt/pool/v1alpha2/pool_utils.go @@ -143,7 +143,10 @@ func checkIfDeviceUsed(path []string, t zpool.Topology) (string, bool) { return usedPath, isUsed } -func checkIfPoolNotImported(cspi *apis.CStorPoolInstance) (string, bool, error) { +// checkIfPoolIsImportable checks if the pool is imported or not. If the pool +// is present on the disk but not imported it returns true as the pool can be +// imported. It also returns false if pool is not found on the disk. +func checkIfPoolIsImportable(cspi *apis.CStorPoolInstance) (string, bool, error) { var cmdOut []byte var err error diff --git a/pkg/algorithm/nodeselect/v1alpha2/build_csp.go b/pkg/algorithm/nodeselect/v1alpha2/build_csp.go index 1a35029bd2..2c3386cfbd 100644 --- a/pkg/algorithm/nodeselect/v1alpha2/build_csp.go +++ b/pkg/algorithm/nodeselect/v1alpha2/build_csp.go @@ -53,7 +53,7 @@ func (ac *Config) GetCSPSpec() (*apis.CStorPoolInstance, error) { } csplabels := ac.buildLabelsForCSPI(nodeName) - cspObj, err := apiscsp.NewBuilder(). + cspBuilder := apiscsp.NewBuilder(). WithName(ac.CSPC.Name + "-" + rand.String(4)). WithNamespace(ac.Namespace). WithNodeSelectorByReference(poolSpec.NodeSelector). @@ -64,27 +64,21 @@ func (ac *Config) GetCSPSpec() (*apis.CStorPoolInstance, error) { WithLabelsNew(csplabels). WithFinalizer(apicspsc.CSPCFinalizer). WithNewVersion(version.GetVersion()). - WithDependentsUpgraded(). - Build() - if err != nil { - return nil, errors.Wrapf(err, "failed to build CSP object for node selector {%v}", poolSpec.NodeSelector) - } - + WithDependentsUpgraded() // check for OpenEBSDisableDependantsReconcileKey annotation which implies // the CSPI should have OpenEBSDisableReconcileKey enabled if ac.CSPC.GetAnnotations()[string(apis.OpenEBSDisableDependantsReconcileKey)] == "false" { - cspObj.Object.Annotations[string(apis.OpenEBSDisableReconcileKey)] = "true" + cspBuilder.WithAnnotationsNew(map[string]string{ + string(apis.OpenEBSDisableReconcileKey): "true", + }) } - // if old CSP has the pool with its name add annotation for renaming while importing - if poolSpec.OldCSPUID != "" { - cspObj.Object.Annotations[string(apis.OldCSPUID)] = poolSpec.OldCSPUID + cspObj, err := cspBuilder.Build() + if err != nil { + return nil, errors.Wrapf(err, "failed to build CSP object for node selector {%v}", poolSpec.NodeSelector) } - // if old CSP is present, it implies the BDs are already claimed for the pool - if poolSpec.OldCSPUID == "" { - err = ac.ClaimBDsForNode(ac.GetBDListForNode(poolSpec)) - if err != nil { - return nil, errors.Wrapf(err, "failed to claim block devices for node {%s}", nodeName) - } + err = ac.ClaimBDsForNode(ac.GetBDListForNode(poolSpec)) + if err != nil { + return nil, errors.Wrapf(err, "failed to claim block devices for node {%s}", nodeName) } return cspObj.Object, nil } diff --git a/pkg/apis/openebs.io/v1alpha1/cas_pool.go b/pkg/apis/openebs.io/v1alpha1/cas_pool.go index e04bf4c279..a062cd2690 100644 --- a/pkg/apis/openebs.io/v1alpha1/cas_pool.go +++ b/pkg/apis/openebs.io/v1alpha1/cas_pool.go @@ -66,8 +66,8 @@ const ( RaidzBlockDeviceCountCPV CasPoolValInt = 3 // Raidz2BlockDeviceCountCPV is the count for raidz2 type pool Raidz2BlockDeviceCountCPV CasPoolValInt = 6 - // OldCSPUID is the uid of the csp that is being migrated to cspi - OldCSPUID CasPoolKey = "cspuid" + // OldPoolName is the old pool that needs to be imported and renamed + OldPoolName CasPoolKey = "cstorpoolinstance.openebs.io/oldname" ) // CasPool is a type which will be utilised by CAS engine to perform diff --git a/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go b/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go index 9c0197fe53..3cc64ef775 100644 --- a/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go +++ b/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go @@ -98,9 +98,6 @@ type PoolSpec struct { // PoolConfig is the default pool config that applies to the // pool on node. PoolConfig PoolConfig `json:"poolConfig"` - // OldCSPUID is used to migrate old csp to cspi. This will be the - // old pool name which needs to imported and renamed as new pool - OldCSPUID string `json:"oldCSPUID,omitempty"` } // PoolConfig is the default pool config that applies to the diff --git a/pkg/cstor/migrate/cspc_generator.go b/pkg/cstor/migrate/cspc_generator.go index d8ae6f0c5d..67784ab0f2 100644 --- a/pkg/cstor/migrate/cspc_generator.go +++ b/pkg/cstor/migrate/cspc_generator.go @@ -82,7 +82,6 @@ func getCSPCSpec(spc *apis.StoragePoolClaim) (*apis.CStorPoolCluster, error) { DefaultRaidGroupType: typeMap[cspObj.Spec.PoolSpec.PoolType], OverProvisioning: cspObj.Spec.PoolSpec.OverProvisioning, }, - OldCSPUID: string(cspObj.UID), }, ) diff --git a/pkg/cstor/migrate/pool.go b/pkg/cstor/migrate/pool.go index 2898f54c50..59eeccd9c7 100644 --- a/pkg/cstor/migrate/pool.go +++ b/pkg/cstor/migrate/pool.go @@ -23,7 +23,6 @@ import ( "k8s.io/klog" apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" - bd "github.com/openebs/maya/pkg/blockdevice/v1alpha2" bdc "github.com/openebs/maya/pkg/blockdeviceclaim/v1alpha1" csp "github.com/openebs/maya/pkg/cstor/pool/v1alpha3" cspc "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1" @@ -43,7 +42,7 @@ const ( replicaPatch = `{ "spec": { "replicas": 0 - } + } }` cspNameLabel = "cstorpool.openebs.io/name" cspUIDLabel = "cstorpool.openebs.io/uid" @@ -58,27 +57,27 @@ const ( // Pool migrates the pool from SPC schema to CSPC schema func Pool(spcName, openebsNamespace string) error { - spcObj, err := spc.NewKubeClient(). - Get(spcName, metav1.GetOptions{}) - // verify if the spc is already migrated. - if k8serrors.IsNotFound(err) { - klog.Infof("spc %s not found.", spcName) - _, err = cspc.NewKubeClient(). - WithNamespace(openebsNamespace).Get(spcName, metav1.GetOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to get equivalent cspc for spc %s", spcName) - } + spcObj, migrated, err := getSPC(spcName, openebsNamespace) + if migrated { klog.Infof("spc %s is already migrated to cspc", spcName) return nil } if err != nil { return err } + err = updateBDCLabels(spcName, openebsNamespace) + if err != nil { + return err + } klog.Infof("Creating equivalent cspc for spc %s", spcName) cspcObj, err := generateCSPC(spcObj, openebsNamespace) if err != nil { return err } + err = updateBDCOwnerRef(cspcObj, openebsNamespace) + if err != nil { + return err + } // List all cspi created with reconcile off cspiList, err := cspi.NewKubeClient(). WithNamespace(openebsNamespace). @@ -99,24 +98,6 @@ func Pool(spcName, openebsNamespace string) error { if err != nil { return err } - cspcObj, err = cspc.NewKubeClient(). - WithNamespace(openebsNamespace).Get(cspcObj.Name, metav1.GetOptions{}) - if err != nil { - return err - } - // remove the OldCSPUID name to avoid renaming in case - // cspi is deleted and comes up after reconciliation. - for i, poolspec := range cspcObj.Spec.Pools { - if poolspec.NodeSelector[string(apis.HostNameCPK)] == - cspiObj.Labels[string(apis.HostNameCPK)] { - cspcObj.Spec.Pools[i].OldCSPUID = "" - } - } - cspcObj, err = cspc.NewKubeClient(). - WithNamespace(openebsNamespace).Update(cspcObj) - if err != nil { - return err - } } } // Clean up old SPC resources after the migration is complete @@ -128,8 +109,33 @@ func Pool(spcName, openebsNamespace string) error { return nil } +// getSPC gets the spc by name and verifies if the spc is already +// migrated or not +func getSPC(spcName, openebsNamespace string) (*apis.StoragePoolClaim, bool, error) { + spcObj, err := spc.NewKubeClient(). + Get(spcName, metav1.GetOptions{}) + // verify if the spc is already migrated. + if k8serrors.IsNotFound(err) { + klog.Infof("spc %s not found.", spcName) + _, err = cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Get(spcName, metav1.GetOptions{}) + if err != nil { + return nil, false, errors.Wrapf(err, "failed to get equivalent cspc for spc %s", spcName) + } + return nil, true, nil + } + if err != nil { + return nil, false, err + } + return spcObj, false, nil +} + // csptocspi migrates a CSP to CSPI based on hostname -func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { +func csptocspi( + cspiObj *apis.CStorPoolInstance, + cspcObj *apis.CStorPoolCluster, + openebsNamespace string, +) error { hostnameLabel := string(apis.HostNameCPK) + "=" + cspiObj.Labels[string(apis.HostNameCPK)] spcLabel := string(apis.StoragePoolClaimCPK) + "=" + cspcObj.Name cspLabel := hostnameLabel + "," + spcLabel @@ -143,14 +149,9 @@ func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, if err != nil { return err } - for _, bdName := range cspObj.Spec.Group[0].Item { - err = updateBDC(bdName, cspcObj, openebsNamespace) - if err != nil { - return err - } - } // once the old pool pod is scaled down and bdcs are patched // bring up the cspi pod so that the old pool can be renamed and imported. + cspiObj.Annotations[string(apis.OldPoolName)] = "cstor-" + string(cspObj.UID) delete(cspiObj.Annotations, string(apis.OpenEBSDisableReconcileKey)) cspiObj, err = cspi.NewKubeClient(). WithNamespace(openebsNamespace). @@ -244,35 +245,54 @@ func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error // Update the bdc with the cspc labels instead of spc labels to allow // filtering of bds claimed by the migrated cspc. -func updateBDC(bdName apis.CspBlockDevice, cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { - bdObj, err := bd.NewKubeClient(). - WithNamespace(openebsNamespace). - Get(bdName.Name, metav1.GetOptions{}) - if err != nil { - return err - } - bdcObj, err := bdc.NewKubeClient(). - WithNamespace(openebsNamespace). - Get(bdObj.Spec.ClaimRef.Name, metav1.GetOptions{}) +func updateBDCLabels(cspcName, openebsNamespace string) error { + bdcList, err := bdc.NewKubeClient().List(metav1.ListOptions{ + LabelSelector: string(apis.StoragePoolClaimCPK) + "=" + cspcName, + }) if err != nil { return err } - klog.Infof("Updating bdc %s with cspc %s info.", bdcObj.Name, cspcObj.Name) - delete(bdcObj.Labels, string(apis.StoragePoolClaimCPK)) - bdcObj.Labels[string(apis.CStorPoolClusterCPK)] = cspcObj.Name - for i, finalizer := range bdcObj.Finalizers { - if finalizer == spcFinalizer { - bdcObj.Finalizers[i] = cspcFinalizer + for _, bdcObj := range bdcList.Items { + klog.Infof("Updating bdc %s with cspc labels & finalizer.", bdcObj.Name) + delete(bdcObj.Labels, string(apis.StoragePoolClaimCPK)) + bdcObj.Labels[string(apis.CStorPoolClusterCPK)] = cspcName + for i, finalizer := range bdcObj.Finalizers { + if finalizer == spcFinalizer { + bdcObj.Finalizers[i] = cspcFinalizer + } + } + // bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" + // bdcObj.OwnerReferences[0].UID = cspcObj.UID + _, err := bdc.NewKubeClient(). + WithNamespace(openebsNamespace). + Update(&bdcObj) + if err != nil { + return errors.Wrapf(err, "failed to update bdc %s with cspc label & finalizer", bdcObj.Name) } } - bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" - bdcObj.OwnerReferences[0].UID = cspcObj.UID - _, err = bdc.NewKubeClient(). - WithNamespace(openebsNamespace). - Update(bdcObj) + return nil +} + +// Update the bdc with the cspc OwnerReferences instead of spc OwnerReferences +// to allow clean up of bdcs on deletion of cspc. +func updateBDCOwnerRef(cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { + bdcList, err := bdc.NewKubeClient().List(metav1.ListOptions{ + LabelSelector: string(apis.CStorPoolClusterCPK) + "=" + cspcObj.Name, + }) if err != nil { return err } + for _, bdcObj := range bdcList.Items { + klog.Infof("Updating bdc %s with cspc ownerRef.", bdcObj.Name) + bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" + bdcObj.OwnerReferences[0].UID = cspcObj.UID + _, err := bdc.NewKubeClient(). + WithNamespace(openebsNamespace). + Update(&bdcObj) + if err != nil { + return errors.Wrapf(err, "failed to update bdc %s with cspc onwerRef", bdcObj.Name) + } + } return nil } @@ -297,7 +317,7 @@ func updateCVRsLabels(cspName, openebsNamespace string, cspiObj *apis.CStorPoolI _, err = cvr.NewKubeclient().WithNamespace(openebsNamespace). Update(&cvrObj) if err != nil { - return err + return errors.Wrapf(err, "failed to update cvr %s with cspc info", cvrObj.Name) } } return nil From ea82cd88c4807abef163e0ccea92adc68e9cf43e Mon Sep 17 00:00:00 2001 From: shubham Date: Tue, 10 Dec 2019 17:27:25 +0530 Subject: [PATCH 09/22] fixed golang-ci issues Signed-off-by: shubham --- cmd/cstor-pool-mgmt/pool/v1alpha2/import.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go b/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go index 649355657a..558bade9c1 100644 --- a/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go +++ b/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go @@ -44,7 +44,7 @@ func Import(cspi *apis.CStorPoolInstance) (bool, error) { common.SyncResources.IsImported = false var poolImported, poolNotImported bool - _, poolNotImported, err = checkIfPoolIsImportable(cspi) + _, poolNotImported, _ = checkIfPoolIsImportable(cspi) if poolNotImported { // if the pool is renamed but not imported remove the // annotation to avoid not found errors From b590ef3e096a8a0e90d7ab41e5ba557cadbbf0dd Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 26 Dec 2019 18:14:42 +0530 Subject: [PATCH 10/22] addressed review comments for pkg Signed-off-by: shubham --- .../nodeselect/v1alpha2/build_csp.go | 2 +- pkg/apis/openebs.io/v1alpha1/cas_keys.go | 2 +- pkg/cstor/migrate/cspc_generator.go | 80 +++++++++--- pkg/cstor/migrate/pool.go | 121 ++++++++++++------ 4 files changed, 143 insertions(+), 62 deletions(-) diff --git a/pkg/algorithm/nodeselect/v1alpha2/build_csp.go b/pkg/algorithm/nodeselect/v1alpha2/build_csp.go index 2c3386cfbd..f8e0af03aa 100644 --- a/pkg/algorithm/nodeselect/v1alpha2/build_csp.go +++ b/pkg/algorithm/nodeselect/v1alpha2/build_csp.go @@ -67,7 +67,7 @@ func (ac *Config) GetCSPSpec() (*apis.CStorPoolInstance, error) { WithDependentsUpgraded() // check for OpenEBSDisableDependantsReconcileKey annotation which implies // the CSPI should have OpenEBSDisableReconcileKey enabled - if ac.CSPC.GetAnnotations()[string(apis.OpenEBSDisableDependantsReconcileKey)] == "false" { + if ac.CSPC.GetAnnotations()[string(apis.OpenEBSDisableDependantsReconcileKey)] == "true" { cspBuilder.WithAnnotationsNew(map[string]string{ string(apis.OpenEBSDisableReconcileKey): "true", }) diff --git a/pkg/apis/openebs.io/v1alpha1/cas_keys.go b/pkg/apis/openebs.io/v1alpha1/cas_keys.go index 8b30777c33..7cbde53844 100644 --- a/pkg/apis/openebs.io/v1alpha1/cas_keys.go +++ b/pkg/apis/openebs.io/v1alpha1/cas_keys.go @@ -50,7 +50,7 @@ const ( // OpenEBSDisableDependantsReconcileKey is the annotation key that decides to create // children objects with OpenEBSDisableReconcileKey as true or false - OpenEBSDisableDependantsReconcileKey CASKey = "reconcile.openebs.io/dependants" + OpenEBSDisableDependantsReconcileKey CASKey = "reconcile.openebs.io/disable-dependants" // CASConfigKey is the key to fetch configurations w.r.t a CAS entity CASConfigKey CASKey = "cas.openebs.io/config" diff --git a/pkg/cstor/migrate/cspc_generator.go b/pkg/cstor/migrate/cspc_generator.go index 67784ab0f2..783bb65d6f 100644 --- a/pkg/cstor/migrate/cspc_generator.go +++ b/pkg/cstor/migrate/cspc_generator.go @@ -23,8 +23,11 @@ import ( csp "github.com/openebs/maya/pkg/cstor/pool/v1alpha3" cspc "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1" cspi "github.com/openebs/maya/pkg/cstor/poolinstance/v1alpha3" + deploy "github.com/openebs/maya/pkg/kubernetes/deployment/appsv1/v1alpha1" "github.com/openebs/maya/pkg/util/retry" "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -38,7 +41,7 @@ var ( } ) -func getBDCList(cspObj apis.CStorPool) []apis.CStorPoolClusterBlockDevice { +func getBDList(cspObj apis.CStorPool) []apis.CStorPoolClusterBlockDevice { list := []apis.CStorPoolClusterBlockDevice{} for _, bdcObj := range cspObj.Spec.Group[0].Item { list = append(list, @@ -50,7 +53,7 @@ func getBDCList(cspObj apis.CStorPool) []apis.CStorPoolClusterBlockDevice { return list } -func getCSPCSpec(spc *apis.StoragePoolClaim) (*apis.CStorPoolCluster, error) { +func getCSPCSpecForSPC(spc *apis.StoragePoolClaim, openebsNamespace string) (*apis.CStorPoolCluster, error) { cspClient := csp.KubeClient() cspList, err := cspClient.List(metav1.ListOptions{ LabelSelector: string(apis.StoragePoolClaimCPK) + "=" + spc.Name, @@ -63,32 +66,66 @@ func getCSPCSpec(spc *apis.StoragePoolClaim) (*apis.CStorPoolCluster, error) { cspcObj.Annotations = map[string]string{ // This label will be used to disable reconciliation on the dependants // In this case that will be CSPI - string(apis.OpenEBSDisableDependantsReconcileKey): "false", + string(apis.OpenEBSDisableDependantsReconcileKey): "true", } for _, cspObj := range cspList.Items { - cspcObj.Spec.Pools = append(cspcObj.Spec.Pools, - apis.PoolSpec{ - NodeSelector: map[string]string{ - string(apis.HostNameCPK): cspObj.Labels[string(apis.HostNameCPK)], - }, - RaidGroups: []apis.RaidGroup{ - apis.RaidGroup{ - Type: typeMap[cspObj.Spec.PoolSpec.PoolType], - BlockDevices: getBDCList(cspObj), - }, - }, - PoolConfig: apis.PoolConfig{ - CacheFile: cspObj.Spec.PoolSpec.CacheFile, - DefaultRaidGroupType: typeMap[cspObj.Spec.PoolSpec.PoolType], - OverProvisioning: cspObj.Spec.PoolSpec.OverProvisioning, + cspDeployList, err := deploy.NewKubeClient().WithNamespace(openebsNamespace). + List(&metav1.ListOptions{ + LabelSelector: "openebs.io/cstor-pool=" + cspObj.Name, + }) + if err != nil { + return nil, err + } + if len(cspDeployList.Items) != 1 { + return nil, errors.Errorf("invalid number of csp deployment found: %d", len(cspDeployList.Items)) + } + poolSpec := apis.PoolSpec{ + NodeSelector: map[string]string{ + string(apis.HostNameCPK): cspObj.Labels[string(apis.HostNameCPK)], + }, + RaidGroups: []apis.RaidGroup{ + apis.RaidGroup{ + Type: typeMap[cspObj.Spec.PoolSpec.PoolType], + BlockDevices: getBDList(cspObj), }, }, - ) - + PoolConfig: apis.PoolConfig{ + CacheFile: cspObj.Spec.PoolSpec.CacheFile, + DefaultRaidGroupType: typeMap[cspObj.Spec.PoolSpec.PoolType], + OverProvisioning: cspObj.Spec.PoolSpec.OverProvisioning, + Resources: getCSPResources(cspDeployList.Items[0]), + Tolerations: cspDeployList.Items[0].Spec.Template.Spec.Tolerations, + // AuxResources: getCSPAuxResources(cspDeployList.Items[0]), + // PriorityClassName: cspDeployList.Items[0].Spec.Template.Spec.PriorityClassName, + }, + } + // if the csp does not have a cachefile then add cachefile + if poolSpec.PoolConfig.CacheFile == "" { + poolSpec.PoolConfig.CacheFile = "/tmp/pool1.cache" + } + cspcObj.Spec.Pools = append(cspcObj.Spec.Pools, poolSpec) } return cspcObj, nil } +func getCSPResources(cspDeploy appsv1.Deployment) *corev1.ResourceRequirements { + for _, con := range cspDeploy.Spec.Template.Spec.Containers { + if con.Name == "cstor-pool" { + return &con.Resources + } + } + return nil +} + +// func getCSPAuxResources(cspDeploy appsv1.Deployment) *corev1.ResourceRequirements { +// for _, con := range cspDeploy.Spec.Template.Spec.Containers { +// if con.Name == "cstor-pool-mgmt" { +// return &con.Resources +// } +// } +// return nil +// } + // generateCSPC creates an equivalent cspc for the given spc object func generateCSPC(spcObj *apis.StoragePoolClaim, openebsNamespace string) ( *apis.CStorPoolCluster, error) { @@ -100,7 +137,7 @@ func generateCSPC(spcObj *apis.StoragePoolClaim, openebsNamespace string) ( if !k8serrors.IsNotFound(err) { return nil, err } - cspcObj, err = getCSPCSpec(spcObj) + cspcObj, err = getCSPCSpecForSPC(spcObj, openebsNamespace) if err != nil { return nil, err } @@ -109,6 +146,7 @@ func generateCSPC(spcObj *apis.StoragePoolClaim, openebsNamespace string) ( if err != nil { return nil, err } + // verify the number of cspi created is correct err = retry. Times(60). Wait(5 * time.Second). diff --git a/pkg/cstor/migrate/pool.go b/pkg/cstor/migrate/pool.go index 59eeccd9c7..7cf1022348 100644 --- a/pkg/cstor/migrate/pool.go +++ b/pkg/cstor/migrate/pool.go @@ -29,7 +29,6 @@ import ( cspi "github.com/openebs/maya/pkg/cstor/poolinstance/v1alpha3" cvr "github.com/openebs/maya/pkg/cstor/volumereplica/v1alpha1" deploy "github.com/openebs/maya/pkg/kubernetes/deployment/appsv1/v1alpha1" - pod "github.com/openebs/maya/pkg/kubernetes/pod/v1alpha1" spc "github.com/openebs/maya/pkg/storagepoolclaim/v1alpha1" "github.com/openebs/maya/pkg/util/retry" "github.com/pkg/errors" @@ -56,8 +55,7 @@ const ( // Pool migrates the pool from SPC schema to CSPC schema func Pool(spcName, openebsNamespace string) error { - - spcObj, migrated, err := getSPC(spcName, openebsNamespace) + spcObj, migrated, err := getSPCWithMigrationStatus(spcName, openebsNamespace) if migrated { klog.Infof("spc %s is already migrated to cspc", spcName) return nil @@ -65,6 +63,10 @@ func Pool(spcName, openebsNamespace string) error { if err != nil { return err } + err = validateSPC(spcObj) + if err != nil { + return err + } err = updateBDCLabels(spcName, openebsNamespace) if err != nil { return err @@ -90,11 +92,13 @@ func Pool(spcName, openebsNamespace string) error { // For each cspi perform the migration from csp that present was on // node on which cspi came up. - for _, cspiObj := range cspiList.Items { + for _, cspiItem := range cspiList.Items { // Skip the migration if cspi is already in ONLINE state, // which implies the migration is done and makes it idempotent + cspiItem := cspiItem // pin it + cspiObj := &cspiItem if cspiObj.Status.Phase != "ONLINE" { - err = csptocspi(&cspiObj, cspcObj, openebsNamespace) + err = csptocspi(cspiObj, cspcObj, openebsNamespace) if err != nil { return err } @@ -109,12 +113,50 @@ func Pool(spcName, openebsNamespace string) error { return nil } -// getSPC gets the spc by name and verifies if the spc is already -// migrated or not -func getSPC(spcName, openebsNamespace string) (*apis.StoragePoolClaim, bool, error) { +// validateSPC determines that if the spc is allowed to migrate or not. +// If the max pool count does not match the number of csp for auto spc, or +// the bd list in spc does not match bds from the csp migration should not be done. +func validateSPC(spcObj *apis.StoragePoolClaim) error { + cspClient := csp.KubeClient() + cspList, err := cspClient.List(metav1.ListOptions{ + LabelSelector: string(apis.StoragePoolClaimCPK) + "=" + spcObj.Name, + }) + if err != nil { + return err + } + if spcObj.Spec.MaxPools != nil { + if *spcObj.Spec.MaxPools != len(cspList.Items) { + return errors.Errorf("maxpool count does not match csp count") + } + return nil + } + bdMap := map[string]int{} + for _, bdName := range spcObj.Spec.BlockDevices.BlockDeviceList { + bdMap[bdName]++ + } + for _, cspObj := range cspList.Items { + for _, bdObj := range cspObj.Spec.Group[0].Item { + bdMap[bdObj.Name]++ + } + } + for bdName, count := range bdMap { + // if bd is configured properly it should occur exactly twice + // one in spc spec and one in csp spec + if count != 2 { + return errors.Errorf("bd %s is not configured properly", bdName) + } + } + return nil +} + +// getSPCWithMigrationStatus gets the spc by name and verifies if the spc is already +// migrated or not. The spc will not be present in the cluster as the last step +// of migration deletes the spc. +func getSPCWithMigrationStatus(spcName, openebsNamespace string) (*apis.StoragePoolClaim, bool, error) { spcObj, err := spc.NewKubeClient(). Get(spcName, metav1.GetOptions{}) - // verify if the spc is already migrated. + // verify if the spc is already migrated. IF an equivalent cspc exists then + // spc is already migrated as spc is only deleted as last step. if k8serrors.IsNotFound(err) { klog.Infof("spc %s not found.", spcName) _, err = cspc.NewKubeClient(). @@ -202,45 +244,46 @@ func getCSP(cspLabel string) (*apis.CStorPool, error) { return &cspObj, nil } -// The old pool pod should be scaled down before the new cspi pod comes up -// to avoid importing the pool at two places at the same time. +// The old pool pod should be scaled down before the new cspi pod reconcile is +// enabled to avoid importing the pool at two places at the same time. func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error { klog.Infof("Scaling down deployemnt %s", cspObj.Name) - cspPod, err := pod.NewKubeClient(). + cspDeployList, err := deploy.NewKubeClient(). WithNamespace(openebsNamespace).List( - metav1.ListOptions{ + &metav1.ListOptions{ LabelSelector: "openebs.io/cstor-pool=" + cspObj.Name, }) if err != nil { return err } - if len(cspPod.Items) > 0 { - _, err = deploy.NewKubeClient().WithNamespace(openebsNamespace). - Patch( - cspObj.Name, - types.StrategicMergePatchType, - []byte(replicaPatch), - ) - if err != nil { - return err - } - err = retry. - Times(60). - Wait(5 * time.Second). - Try(func(attempt uint) error { - _, err1 := pod.NewKubeClient(). - WithNamespace(openebsNamespace). - Get(cspPod.Items[0].Name, metav1.GetOptions{}) - if !k8serrors.IsNotFound(err1) { - return errors.Errorf("failed to get csp pod because %s", err1) - } - return nil - }) - if err != nil { - return err - } + if len(cspDeployList.Items) != 1 { + return errors.Errorf("invalid number of csp deployment found: %d", len(cspDeployList.Items)) } - return nil + _, err = deploy.NewKubeClient().WithNamespace(openebsNamespace). + Patch( + cspDeployList.Items[0].Name, + types.StrategicMergePatchType, + []byte(replicaPatch), + ) + if err != nil { + return err + } + err = retry. + Times(60). + Wait(5 * time.Second). + Try(func(attempt uint) error { + cspDeploy, err1 := deploy.NewKubeClient(). + WithNamespace(openebsNamespace). + Get(cspDeployList.Items[0].Name) + if err1 != nil { + return errors.Wrapf(err1, "failed to get csp deploy") + } + if cspDeploy.Status.ReadyReplicas != 0 { + return errors.Errorf("failed to scale down csp deployment") + } + return nil + }) + return err } // Update the bdc with the cspc labels instead of spc labels to allow From 976f30190ff7f2ed38993e9916978f9dcaed8af3 Mon Sep 17 00:00:00 2001 From: shubham Date: Fri, 27 Dec 2019 13:15:56 +0530 Subject: [PATCH 11/22] fix scopelint issue and add priorityclass Signed-off-by: shubham --- GNUmakefile | 4 ++++ pkg/cstor/migrate/cspc_generator.go | 4 ++-- pkg/cstor/migrate/pool.go | 18 ++++++++++++------ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 596a3bd0b0..baeaec9502 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -124,6 +124,10 @@ include ./buildscripts/cstor-volume-mgmt/Makefile.mk include ./buildscripts/cspi-mgmt/Makefile.mk include ./buildscripts/cvc-operator/Makefile.mk include ./buildscripts/migrate/Makefile.mk +include ./buildscripts/exporter/Makefile.mk +include ./buildscripts/cstor-pool-mgmt/Makefile.mk +include ./buildscripts/cstor-volume-mgmt/Makefile.mk +include ./buildscripts/cspi-mgmt/Makefile.mk .PHONY: all all: compile-tests apiserver-image exporter-image pool-mgmt-image volume-mgmt-image \ diff --git a/pkg/cstor/migrate/cspc_generator.go b/pkg/cstor/migrate/cspc_generator.go index 783bb65d6f..feb975c5c2 100644 --- a/pkg/cstor/migrate/cspc_generator.go +++ b/pkg/cstor/migrate/cspc_generator.go @@ -64,7 +64,7 @@ func getCSPCSpecForSPC(spc *apis.StoragePoolClaim, openebsNamespace string) (*ap cspcObj := &apis.CStorPoolCluster{} cspcObj.Name = spc.Name cspcObj.Annotations = map[string]string{ - // This label will be used to disable reconciliation on the dependants + // This label will be used to disable reconciliation on the dependants. // In this case that will be CSPI string(apis.OpenEBSDisableDependantsReconcileKey): "true", } @@ -95,8 +95,8 @@ func getCSPCSpecForSPC(spc *apis.StoragePoolClaim, openebsNamespace string) (*ap OverProvisioning: cspObj.Spec.PoolSpec.OverProvisioning, Resources: getCSPResources(cspDeployList.Items[0]), Tolerations: cspDeployList.Items[0].Spec.Template.Spec.Tolerations, + PriorityClassName: cspDeployList.Items[0].Spec.Template.Spec.PriorityClassName, // AuxResources: getCSPAuxResources(cspDeployList.Items[0]), - // PriorityClassName: cspDeployList.Items[0].Spec.Template.Spec.PriorityClassName, }, } // if the csp does not have a cachefile then add cachefile diff --git a/pkg/cstor/migrate/pool.go b/pkg/cstor/migrate/pool.go index 7cf1022348..f60112ed50 100644 --- a/pkg/cstor/migrate/pool.go +++ b/pkg/cstor/migrate/pool.go @@ -295,7 +295,9 @@ func updateBDCLabels(cspcName, openebsNamespace string) error { if err != nil { return err } - for _, bdcObj := range bdcList.Items { + for _, bdcItem := range bdcList.Items { + bdcItem := bdcItem // pin it + bdcObj := &bdcItem klog.Infof("Updating bdc %s with cspc labels & finalizer.", bdcObj.Name) delete(bdcObj.Labels, string(apis.StoragePoolClaimCPK)) bdcObj.Labels[string(apis.CStorPoolClusterCPK)] = cspcName @@ -308,7 +310,7 @@ func updateBDCLabels(cspcName, openebsNamespace string) error { // bdcObj.OwnerReferences[0].UID = cspcObj.UID _, err := bdc.NewKubeClient(). WithNamespace(openebsNamespace). - Update(&bdcObj) + Update(bdcObj) if err != nil { return errors.Wrapf(err, "failed to update bdc %s with cspc label & finalizer", bdcObj.Name) } @@ -325,13 +327,15 @@ func updateBDCOwnerRef(cspcObj *apis.CStorPoolCluster, openebsNamespace string) if err != nil { return err } - for _, bdcObj := range bdcList.Items { + for _, bdcItem := range bdcList.Items { + bdcItem := bdcItem // pin it + bdcObj := &bdcItem klog.Infof("Updating bdc %s with cspc ownerRef.", bdcObj.Name) bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" bdcObj.OwnerReferences[0].UID = cspcObj.UID _, err := bdc.NewKubeClient(). WithNamespace(openebsNamespace). - Update(&bdcObj) + Update(bdcObj) if err != nil { return errors.Wrapf(err, "failed to update bdc %s with cspc onwerRef", bdcObj.Name) } @@ -349,7 +353,9 @@ func updateCVRsLabels(cspName, openebsNamespace string, cspiObj *apis.CStorPoolI if err != nil { return err } - for _, cvrObj := range cvrList.Items { + for _, cvrItem := range cvrList.Items { + cvrItem := cvrItem // pin it + cvrObj := &cvrItem klog.Infof("Updating cvr %s with cspi %s info.", cvrObj.Name, cspiObj.Name) delete(cvrObj.Labels, cspNameLabel) delete(cvrObj.Labels, cspUIDLabel) @@ -358,7 +364,7 @@ func updateCVRsLabels(cspName, openebsNamespace string, cspiObj *apis.CStorPoolI delete(cvrObj.Annotations, cspHostnameAnnotation) cvrObj.Annotations[cspiHostnameAnnotation] = cspiObj.Spec.HostName _, err = cvr.NewKubeclient().WithNamespace(openebsNamespace). - Update(&cvrObj) + Update(cvrObj) if err != nil { return errors.Wrapf(err, "failed to update cvr %s with cspc info", cvrObj.Name) } From 99b37f0d515570dccf0b351a188461787747ba25 Mon Sep 17 00:00:00 2001 From: shubham Date: Tue, 19 Nov 2019 16:59:22 +0530 Subject: [PATCH 12/22] feat(migrate): generate CSPC form given SPC Signed-off-by: shubham --- pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go b/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go index 3cc64ef775..9c0197fe53 100644 --- a/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go +++ b/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go @@ -98,6 +98,9 @@ type PoolSpec struct { // PoolConfig is the default pool config that applies to the // pool on node. PoolConfig PoolConfig `json:"poolConfig"` + // OldCSPUID is used to migrate old csp to cspi. This will be the + // old pool name which needs to imported and renamed as new pool + OldCSPUID string `json:"oldCSPUID,omitempty"` } // PoolConfig is the default pool config that applies to the From 2559bc37d028a674296f8f6f48117d5be3f2ab49 Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 28 Nov 2019 16:27:57 +0530 Subject: [PATCH 13/22] feat(cspc): migrate pools from spc to cspc Signed-off-by: shubham --- GNUmakefile | 4 - cmd/cspi-mgmt/app/handler.go | 7 +- pkg/cstor/migrate/migration.go | 270 +++++++++++++++++++++++++++++++++ 3 files changed, 276 insertions(+), 5 deletions(-) create mode 100644 pkg/cstor/migrate/migration.go diff --git a/GNUmakefile b/GNUmakefile index baeaec9502..596a3bd0b0 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -124,10 +124,6 @@ include ./buildscripts/cstor-volume-mgmt/Makefile.mk include ./buildscripts/cspi-mgmt/Makefile.mk include ./buildscripts/cvc-operator/Makefile.mk include ./buildscripts/migrate/Makefile.mk -include ./buildscripts/exporter/Makefile.mk -include ./buildscripts/cstor-pool-mgmt/Makefile.mk -include ./buildscripts/cstor-volume-mgmt/Makefile.mk -include ./buildscripts/cspi-mgmt/Makefile.mk .PHONY: all all: compile-tests apiserver-image exporter-image pool-mgmt-image volume-mgmt-image \ diff --git a/cmd/cspi-mgmt/app/handler.go b/cmd/cspi-mgmt/app/handler.go index d34ba4528b..8ef6d39832 100644 --- a/cmd/cspi-mgmt/app/handler.go +++ b/cmd/cspi-mgmt/app/handler.go @@ -59,7 +59,11 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { common.SyncResources.Mux.Lock() // try to import pool - isImported, err = zpool.Import(cspi) + if cspi.Annotations["cspuid"] != "" { + isImported, err = zpool.Import(cspi, "cstor-"+cspi.Annotations["cspuid"]) + } else { + isImported, err = zpool.Import(cspi, "") + } if isImported { if err != nil { common.SyncResources.Mux.Unlock() @@ -71,6 +75,7 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { } zpool.CheckImportedPoolVolume() common.SyncResources.Mux.Unlock() + delete(cspi.Annotations, "cspuid") err = c.update(cspi) if err != nil { c.recorder.Event(cspi, diff --git a/pkg/cstor/migrate/migration.go b/pkg/cstor/migrate/migration.go new file mode 100644 index 0000000000..ebdc74418e --- /dev/null +++ b/pkg/cstor/migrate/migration.go @@ -0,0 +1,270 @@ +/* +Copyright 2019 The OpenEBS Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrate + +import ( + "fmt" + "time" + + "k8s.io/klog" + + apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" + bd "github.com/openebs/maya/pkg/blockdevice/v1alpha2" + bdc "github.com/openebs/maya/pkg/blockdeviceclaim/v1alpha1" + csp "github.com/openebs/maya/pkg/cstor/pool/v1alpha3" + cspc "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1" + cspi "github.com/openebs/maya/pkg/cstor/poolinstance/v1alpha3" + cvr "github.com/openebs/maya/pkg/cstor/volumereplica/v1alpha1" + deploy "github.com/openebs/maya/pkg/kubernetes/deployment/appsv1/v1alpha1" + pod "github.com/openebs/maya/pkg/kubernetes/pod/v1alpha1" + spc "github.com/openebs/maya/pkg/storagepoolclaim/v1alpha1" + "github.com/openebs/maya/pkg/util/retry" + "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" +) + +const replicaPatch = `{ + "spec": { + "replicas": 0 + } +}` + +// Pool ... +func Pool(spcName, openebsNamespace string) error { + klog.Infof("Migrating spc %s to cspc", spcName) + spcObj, err := spc.NewKubeClient(). + Get(spcName, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + klog.Infof("spc %s not found.", spcName) + _, err := cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Get(spcName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to get equivalent cspc for spc %s", spcName) + } + klog.Infof("spc %s is already migrated to cspc", spcName) + return nil + } + if err != nil { + return err + } + klog.Infof("Creating equivalent cspc for spc %s", spcName) + cspcObj, err := generateCSPC(spcObj, openebsNamespace) + if err != nil { + return err + } + + cspiList, err := cspi.NewKubeClient(). + WithNamespace(openebsNamespace). + List(metav1.ListOptions{ + LabelSelector: string(apis.CStorPoolClusterCPK) + "=" + cspcObj.Name, + }) + if err != nil { + return err + } + + for _, cspiObj := range cspiList.Items { + if cspiObj.Status.Phase != "ONLINE" { + err = csptocspi(&cspiObj, cspcObj, openebsNamespace) + if err != nil { + return err + } + cspcObj, err = cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Get(cspcObj.Name, metav1.GetOptions{}) + for i, poolspec := range cspcObj.Spec.Pools { + if poolspec.NodeSelector[string(apis.HostNameCPK)] == + cspiObj.Labels[string(apis.HostNameCPK)] { + cspcObj.Spec.Pools[i].OldCSPUID = "" + } + } + cspcObj, err = cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Update(cspcObj) + if err != nil { + return err + } + } + } + err = spc.NewKubeClient(). + Delete(spcName, &metav1.DeleteOptions{}) + if err != nil { + return err + } + return nil +} + +func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { + hostnameLabel := string(apis.HostNameCPK) + "=" + cspiObj.Labels[string(apis.HostNameCPK)] + spcLabel := string(apis.StoragePoolClaimCPK) + "=" + cspcObj.Name + cspLabel := hostnameLabel + "," + spcLabel + + cspObj, err := getCSP(cspLabel) + if err != nil { + return err + } + klog.Infof("Migrating csp %s to cspi %s", cspiObj.Name, cspObj.Name) + err = scaleDownDeployment(cspObj, openebsNamespace) + if err != nil { + return err + } + for _, bdName := range cspObj.Spec.Group[0].Item { + err = updateBDC(bdName, cspcObj, openebsNamespace) + if err != nil { + return err + } + } + delete(cspiObj.Annotations, string(apis.OpenEBSDisableReconcileKey)) + cspiObj, err = cspi.NewKubeClient(). + WithNamespace(openebsNamespace). + Update(cspiObj) + if err != nil { + return err + } + err = retry. + Times(60). + Wait(5 * time.Second). + Try(func(attempt uint) error { + cspiObj, err1 := cspi.NewKubeClient(). + WithNamespace(openebsNamespace). + Get(cspiObj.Name, metav1.GetOptions{}) + if err1 != nil { + return err1 + } + if cspiObj.Status.Phase != "ONLINE" { + return errors.Errorf("failed to verify cspi phase expected: Healthy got: %s", + cspiObj.Status.Phase) + } + return nil + }) + if err != nil { + return err + } + err = updateCVRsLabels(cspObj.Name, openebsNamespace, cspiObj) + if err != nil { + return err + } + return nil +} + +func getCSP(cspLabel string) (*apis.CStorPool, error) { + cspClient := csp.KubeClient() + cspList, err := cspClient.List(metav1.ListOptions{ + LabelSelector: cspLabel, + }) + if err != nil { + return nil, err + } + if len(cspList.Items) != 1 { + return nil, fmt.Errorf("Invalid number of pools on one node: %d", len(cspList.Items)) + } + cspObj := cspList.Items[0] + if err != nil { + return nil, err + } + return &cspObj, nil +} + +func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error { + klog.Infof("Scaling down deployemnt %s", cspObj.Name) + cspPod, err := pod.NewKubeClient(). + WithNamespace(openebsNamespace).List( + metav1.ListOptions{ + LabelSelector: "openebs.io/cstor-pool=" + cspObj.Name, + }) + if err != nil { + return err + } + _, err = deploy.NewKubeClient(). + WithNamespace(openebsNamespace).Patch( + cspObj.Name, + types.StrategicMergePatchType, + []byte(replicaPatch), + ) + err = retry. + Times(60). + Wait(5 * time.Second). + Try(func(attempt uint) error { + _, err1 := pod.NewKubeClient(). + WithNamespace(openebsNamespace). + Get(cspPod.Items[0].Name, metav1.GetOptions{}) + if !k8serrors.IsNotFound(err1) { + return errors.Errorf("failed to get csp pod because %s", err1) + } + return nil + }) + if err != nil { + return err + } + return nil +} + +func updateBDC(bdName apis.CspBlockDevice, cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { + bdObj, err := bd.NewKubeClient(). + WithNamespace(openebsNamespace). + Get(bdName.Name, metav1.GetOptions{}) + if err != nil { + return err + } + bdcObj, err := bdc.NewKubeClient(). + WithNamespace(openebsNamespace). + Get(bdObj.Spec.ClaimRef.Name, metav1.GetOptions{}) + if err != nil { + return err + } + klog.Infof("Updating bdc %s with cspc %s info.", bdcObj.Name, cspcObj.Name) + delete(bdcObj.Labels, string(apis.StoragePoolClaimCPK)) + bdcObj.Labels[string(apis.CStorPoolClusterCPK)] = cspcObj.Name + for i, finalizer := range bdcObj.Finalizers { + if finalizer == "storagepoolclaim.openebs.io/finalizer" { + bdcObj.Finalizers[i] = "cstorpoolcluster.openebs.io/finalizer" + } + } + bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" + bdcObj.OwnerReferences[0].UID = cspcObj.UID + bdcObj, err = bdc.NewKubeClient(). + WithNamespace(openebsNamespace). + Update(bdcObj) + if err != nil { + return err + } + return nil +} + +func updateCVRsLabels(cspName, openebsNamespace string, cspiObj *apis.CStorPoolInstance) error { + cvrList, err := cvr.NewKubeclient(). + WithNamespace(openebsNamespace).List(metav1.ListOptions{ + LabelSelector: "cstorpool.openebs.io/name=" + cspName, + }) + if err != nil { + return err + } + for _, cvrObj := range cvrList.Items { + klog.Infof("Updating cvr %s with cspi %s info.", cvrObj.Name, cspiObj.Name) + delete(cvrObj.Labels, "cstorpool.openebs.io/name") + delete(cvrObj.Labels, "cstorpool.openebs.io/uid") + cvrObj.Labels["cstorpoolinstance.openebs.io/name"] = cspiObj.Name + cvrObj.Labels["cstorpoolinstance.openebs.io/uid"] = string(cspiObj.UID) + delete(cvrObj.Annotations, "cstorpool.openebs.io/hostname") + cvrObj.Annotations["cstorpoolinstance.openebs.io/hostname"] = cspiObj.Spec.HostName + _, err = cvr.NewKubeclient().WithNamespace(openebsNamespace). + Update(&cvrObj) + if err != nil { + return err + } + } + return nil +} From 627c12b86257a86c19833def64f8859b28701acd Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 28 Nov 2019 16:56:10 +0530 Subject: [PATCH 14/22] fix linting issues Signed-off-by: shubham --- pkg/cstor/migrate/migration.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pkg/cstor/migrate/migration.go b/pkg/cstor/migrate/migration.go index ebdc74418e..74680d92c4 100644 --- a/pkg/cstor/migrate/migration.go +++ b/pkg/cstor/migrate/migration.go @@ -52,7 +52,7 @@ func Pool(spcName, openebsNamespace string) error { Get(spcName, metav1.GetOptions{}) if k8serrors.IsNotFound(err) { klog.Infof("spc %s not found.", spcName) - _, err := cspc.NewKubeClient(). + _, err = cspc.NewKubeClient(). WithNamespace(openebsNamespace).Get(spcName, metav1.GetOptions{}) if err != nil { return errors.Wrapf(err, "failed to get equivalent cspc for spc %s", spcName) @@ -86,6 +86,9 @@ func Pool(spcName, openebsNamespace string) error { } cspcObj, err = cspc.NewKubeClient(). WithNamespace(openebsNamespace).Get(cspcObj.Name, metav1.GetOptions{}) + if err != nil { + return err + } for i, poolspec := range cspcObj.Spec.Pools { if poolspec.NodeSelector[string(apis.HostNameCPK)] == cspiObj.Labels[string(apis.HostNameCPK)] { @@ -111,7 +114,7 @@ func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, hostnameLabel := string(apis.HostNameCPK) + "=" + cspiObj.Labels[string(apis.HostNameCPK)] spcLabel := string(apis.StoragePoolClaimCPK) + "=" + cspcObj.Name cspLabel := hostnameLabel + "," + spcLabel - + var err1 error cspObj, err := getCSP(cspLabel) if err != nil { return err @@ -138,7 +141,7 @@ func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, Times(60). Wait(5 * time.Second). Try(func(attempt uint) error { - cspiObj, err1 := cspi.NewKubeClient(). + cspiObj, err1 = cspi.NewKubeClient(). WithNamespace(openebsNamespace). Get(cspiObj.Name, metav1.GetOptions{}) if err1 != nil { @@ -172,9 +175,6 @@ func getCSP(cspLabel string) (*apis.CStorPool, error) { return nil, fmt.Errorf("Invalid number of pools on one node: %d", len(cspList.Items)) } cspObj := cspList.Items[0] - if err != nil { - return nil, err - } return &cspObj, nil } @@ -188,12 +188,15 @@ func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error if err != nil { return err } - _, err = deploy.NewKubeClient(). - WithNamespace(openebsNamespace).Patch( - cspObj.Name, - types.StrategicMergePatchType, - []byte(replicaPatch), - ) + _, err = deploy.NewKubeClient().WithNamespace(openebsNamespace). + Patch( + cspObj.Name, + types.StrategicMergePatchType, + []byte(replicaPatch), + ) + if err != nil { + return err + } err = retry. Times(60). Wait(5 * time.Second). @@ -235,7 +238,7 @@ func updateBDC(bdName apis.CspBlockDevice, cspcObj *apis.CStorPoolCluster, opene } bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" bdcObj.OwnerReferences[0].UID = cspcObj.UID - bdcObj, err = bdc.NewKubeClient(). + _, err = bdc.NewKubeClient(). WithNamespace(openebsNamespace). Update(bdcObj) if err != nil { From 33f4d9c567ef9f59d505bf35c5e4f6cc618bde42 Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 28 Nov 2019 18:04:39 +0530 Subject: [PATCH 15/22] renamed one file Signed-off-by: shubham --- pkg/cstor/migrate/migration.go | 273 --------------------------------- 1 file changed, 273 deletions(-) delete mode 100644 pkg/cstor/migrate/migration.go diff --git a/pkg/cstor/migrate/migration.go b/pkg/cstor/migrate/migration.go deleted file mode 100644 index 74680d92c4..0000000000 --- a/pkg/cstor/migrate/migration.go +++ /dev/null @@ -1,273 +0,0 @@ -/* -Copyright 2019 The OpenEBS Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package migrate - -import ( - "fmt" - "time" - - "k8s.io/klog" - - apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" - bd "github.com/openebs/maya/pkg/blockdevice/v1alpha2" - bdc "github.com/openebs/maya/pkg/blockdeviceclaim/v1alpha1" - csp "github.com/openebs/maya/pkg/cstor/pool/v1alpha3" - cspc "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1" - cspi "github.com/openebs/maya/pkg/cstor/poolinstance/v1alpha3" - cvr "github.com/openebs/maya/pkg/cstor/volumereplica/v1alpha1" - deploy "github.com/openebs/maya/pkg/kubernetes/deployment/appsv1/v1alpha1" - pod "github.com/openebs/maya/pkg/kubernetes/pod/v1alpha1" - spc "github.com/openebs/maya/pkg/storagepoolclaim/v1alpha1" - "github.com/openebs/maya/pkg/util/retry" - "github.com/pkg/errors" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" -) - -const replicaPatch = `{ - "spec": { - "replicas": 0 - } -}` - -// Pool ... -func Pool(spcName, openebsNamespace string) error { - klog.Infof("Migrating spc %s to cspc", spcName) - spcObj, err := spc.NewKubeClient(). - Get(spcName, metav1.GetOptions{}) - if k8serrors.IsNotFound(err) { - klog.Infof("spc %s not found.", spcName) - _, err = cspc.NewKubeClient(). - WithNamespace(openebsNamespace).Get(spcName, metav1.GetOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to get equivalent cspc for spc %s", spcName) - } - klog.Infof("spc %s is already migrated to cspc", spcName) - return nil - } - if err != nil { - return err - } - klog.Infof("Creating equivalent cspc for spc %s", spcName) - cspcObj, err := generateCSPC(spcObj, openebsNamespace) - if err != nil { - return err - } - - cspiList, err := cspi.NewKubeClient(). - WithNamespace(openebsNamespace). - List(metav1.ListOptions{ - LabelSelector: string(apis.CStorPoolClusterCPK) + "=" + cspcObj.Name, - }) - if err != nil { - return err - } - - for _, cspiObj := range cspiList.Items { - if cspiObj.Status.Phase != "ONLINE" { - err = csptocspi(&cspiObj, cspcObj, openebsNamespace) - if err != nil { - return err - } - cspcObj, err = cspc.NewKubeClient(). - WithNamespace(openebsNamespace).Get(cspcObj.Name, metav1.GetOptions{}) - if err != nil { - return err - } - for i, poolspec := range cspcObj.Spec.Pools { - if poolspec.NodeSelector[string(apis.HostNameCPK)] == - cspiObj.Labels[string(apis.HostNameCPK)] { - cspcObj.Spec.Pools[i].OldCSPUID = "" - } - } - cspcObj, err = cspc.NewKubeClient(). - WithNamespace(openebsNamespace).Update(cspcObj) - if err != nil { - return err - } - } - } - err = spc.NewKubeClient(). - Delete(spcName, &metav1.DeleteOptions{}) - if err != nil { - return err - } - return nil -} - -func csptocspi(cspiObj *apis.CStorPoolInstance, cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { - hostnameLabel := string(apis.HostNameCPK) + "=" + cspiObj.Labels[string(apis.HostNameCPK)] - spcLabel := string(apis.StoragePoolClaimCPK) + "=" + cspcObj.Name - cspLabel := hostnameLabel + "," + spcLabel - var err1 error - cspObj, err := getCSP(cspLabel) - if err != nil { - return err - } - klog.Infof("Migrating csp %s to cspi %s", cspiObj.Name, cspObj.Name) - err = scaleDownDeployment(cspObj, openebsNamespace) - if err != nil { - return err - } - for _, bdName := range cspObj.Spec.Group[0].Item { - err = updateBDC(bdName, cspcObj, openebsNamespace) - if err != nil { - return err - } - } - delete(cspiObj.Annotations, string(apis.OpenEBSDisableReconcileKey)) - cspiObj, err = cspi.NewKubeClient(). - WithNamespace(openebsNamespace). - Update(cspiObj) - if err != nil { - return err - } - err = retry. - Times(60). - Wait(5 * time.Second). - Try(func(attempt uint) error { - cspiObj, err1 = cspi.NewKubeClient(). - WithNamespace(openebsNamespace). - Get(cspiObj.Name, metav1.GetOptions{}) - if err1 != nil { - return err1 - } - if cspiObj.Status.Phase != "ONLINE" { - return errors.Errorf("failed to verify cspi phase expected: Healthy got: %s", - cspiObj.Status.Phase) - } - return nil - }) - if err != nil { - return err - } - err = updateCVRsLabels(cspObj.Name, openebsNamespace, cspiObj) - if err != nil { - return err - } - return nil -} - -func getCSP(cspLabel string) (*apis.CStorPool, error) { - cspClient := csp.KubeClient() - cspList, err := cspClient.List(metav1.ListOptions{ - LabelSelector: cspLabel, - }) - if err != nil { - return nil, err - } - if len(cspList.Items) != 1 { - return nil, fmt.Errorf("Invalid number of pools on one node: %d", len(cspList.Items)) - } - cspObj := cspList.Items[0] - return &cspObj, nil -} - -func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error { - klog.Infof("Scaling down deployemnt %s", cspObj.Name) - cspPod, err := pod.NewKubeClient(). - WithNamespace(openebsNamespace).List( - metav1.ListOptions{ - LabelSelector: "openebs.io/cstor-pool=" + cspObj.Name, - }) - if err != nil { - return err - } - _, err = deploy.NewKubeClient().WithNamespace(openebsNamespace). - Patch( - cspObj.Name, - types.StrategicMergePatchType, - []byte(replicaPatch), - ) - if err != nil { - return err - } - err = retry. - Times(60). - Wait(5 * time.Second). - Try(func(attempt uint) error { - _, err1 := pod.NewKubeClient(). - WithNamespace(openebsNamespace). - Get(cspPod.Items[0].Name, metav1.GetOptions{}) - if !k8serrors.IsNotFound(err1) { - return errors.Errorf("failed to get csp pod because %s", err1) - } - return nil - }) - if err != nil { - return err - } - return nil -} - -func updateBDC(bdName apis.CspBlockDevice, cspcObj *apis.CStorPoolCluster, openebsNamespace string) error { - bdObj, err := bd.NewKubeClient(). - WithNamespace(openebsNamespace). - Get(bdName.Name, metav1.GetOptions{}) - if err != nil { - return err - } - bdcObj, err := bdc.NewKubeClient(). - WithNamespace(openebsNamespace). - Get(bdObj.Spec.ClaimRef.Name, metav1.GetOptions{}) - if err != nil { - return err - } - klog.Infof("Updating bdc %s with cspc %s info.", bdcObj.Name, cspcObj.Name) - delete(bdcObj.Labels, string(apis.StoragePoolClaimCPK)) - bdcObj.Labels[string(apis.CStorPoolClusterCPK)] = cspcObj.Name - for i, finalizer := range bdcObj.Finalizers { - if finalizer == "storagepoolclaim.openebs.io/finalizer" { - bdcObj.Finalizers[i] = "cstorpoolcluster.openebs.io/finalizer" - } - } - bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" - bdcObj.OwnerReferences[0].UID = cspcObj.UID - _, err = bdc.NewKubeClient(). - WithNamespace(openebsNamespace). - Update(bdcObj) - if err != nil { - return err - } - return nil -} - -func updateCVRsLabels(cspName, openebsNamespace string, cspiObj *apis.CStorPoolInstance) error { - cvrList, err := cvr.NewKubeclient(). - WithNamespace(openebsNamespace).List(metav1.ListOptions{ - LabelSelector: "cstorpool.openebs.io/name=" + cspName, - }) - if err != nil { - return err - } - for _, cvrObj := range cvrList.Items { - klog.Infof("Updating cvr %s with cspi %s info.", cvrObj.Name, cspiObj.Name) - delete(cvrObj.Labels, "cstorpool.openebs.io/name") - delete(cvrObj.Labels, "cstorpool.openebs.io/uid") - cvrObj.Labels["cstorpoolinstance.openebs.io/name"] = cspiObj.Name - cvrObj.Labels["cstorpoolinstance.openebs.io/uid"] = string(cspiObj.UID) - delete(cvrObj.Annotations, "cstorpool.openebs.io/hostname") - cvrObj.Annotations["cstorpoolinstance.openebs.io/hostname"] = cspiObj.Spec.HostName - _, err = cvr.NewKubeclient().WithNamespace(openebsNamespace). - Update(&cvrObj) - if err != nil { - return err - } - } - return nil -} From edc729d2e6b57cdeec84768b7e0833aef52898f2 Mon Sep 17 00:00:00 2001 From: shubham Date: Fri, 29 Nov 2019 10:35:14 +0530 Subject: [PATCH 16/22] moved hardcoded strings in function calls to global constants Signed-off-by: shubham --- cmd/cspi-mgmt/app/handler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/cspi-mgmt/app/handler.go b/cmd/cspi-mgmt/app/handler.go index 8ef6d39832..5211860fd0 100644 --- a/cmd/cspi-mgmt/app/handler.go +++ b/cmd/cspi-mgmt/app/handler.go @@ -59,8 +59,8 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { common.SyncResources.Mux.Lock() // try to import pool - if cspi.Annotations["cspuid"] != "" { - isImported, err = zpool.Import(cspi, "cstor-"+cspi.Annotations["cspuid"]) + if cspi.Annotations[string(apis.OldCSPUID)] != "" { + isImported, err = zpool.Import(cspi, "cstor-"+cspi.Annotations[string(apis.OldCSPUID)]) } else { isImported, err = zpool.Import(cspi, "") } @@ -75,7 +75,7 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { } zpool.CheckImportedPoolVolume() common.SyncResources.Mux.Unlock() - delete(cspi.Annotations, "cspuid") + delete(cspi.Annotations, string(apis.OldCSPUID)) err = c.update(cspi) if err != nil { c.recorder.Event(cspi, From 4e771b179ade0b1ca972aa0af45e482181e29871 Mon Sep 17 00:00:00 2001 From: shubham Date: Fri, 29 Nov 2019 17:32:20 +0530 Subject: [PATCH 17/22] added comments for the function and removed unnecessary checks Signed-off-by: shubham --- cmd/cspi-mgmt/app/handler.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/cspi-mgmt/app/handler.go b/cmd/cspi-mgmt/app/handler.go index 5211860fd0..da1d2d7d3a 100644 --- a/cmd/cspi-mgmt/app/handler.go +++ b/cmd/cspi-mgmt/app/handler.go @@ -58,7 +58,9 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { // take a lock for common package for updating variables common.SyncResources.Mux.Lock() - // try to import pool + // try to import pool, first check for migration case in which the previous + // poolname is CSP uid and should be renamed to CSPC uid. If old CSP uid is + // not present then follow the normal code path. if cspi.Annotations[string(apis.OldCSPUID)] != "" { isImported, err = zpool.Import(cspi, "cstor-"+cspi.Annotations[string(apis.OldCSPUID)]) } else { From 2a24183520aafcbfd00ef34a72647bd6c55cef3a Mon Sep 17 00:00:00 2001 From: shubham Date: Tue, 31 Dec 2019 17:08:28 +0530 Subject: [PATCH 18/22] added aux resources to migrated pool Signed-off-by: shubham --- pkg/cstor/migrate/cspc_generator.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/cstor/migrate/cspc_generator.go b/pkg/cstor/migrate/cspc_generator.go index feb975c5c2..79957d5242 100644 --- a/pkg/cstor/migrate/cspc_generator.go +++ b/pkg/cstor/migrate/cspc_generator.go @@ -96,7 +96,7 @@ func getCSPCSpecForSPC(spc *apis.StoragePoolClaim, openebsNamespace string) (*ap Resources: getCSPResources(cspDeployList.Items[0]), Tolerations: cspDeployList.Items[0].Spec.Template.Spec.Tolerations, PriorityClassName: cspDeployList.Items[0].Spec.Template.Spec.PriorityClassName, - // AuxResources: getCSPAuxResources(cspDeployList.Items[0]), + AuxResources: getCSPAuxResources(cspDeployList.Items[0]), }, } // if the csp does not have a cachefile then add cachefile @@ -117,14 +117,14 @@ func getCSPResources(cspDeploy appsv1.Deployment) *corev1.ResourceRequirements { return nil } -// func getCSPAuxResources(cspDeploy appsv1.Deployment) *corev1.ResourceRequirements { -// for _, con := range cspDeploy.Spec.Template.Spec.Containers { -// if con.Name == "cstor-pool-mgmt" { -// return &con.Resources -// } -// } -// return nil -// } +func getCSPAuxResources(cspDeploy appsv1.Deployment) *corev1.ResourceRequirements { + for _, con := range cspDeploy.Spec.Template.Spec.Containers { + if con.Name == "cstor-pool-mgmt" { + return &con.Resources + } + } + return nil +} // generateCSPC creates an equivalent cspc for the given spc object func generateCSPC(spcObj *apis.StoragePoolClaim, openebsNamespace string) ( From d81cf5cfe7e23377fd22e430da440310efcb1d2b Mon Sep 17 00:00:00 2001 From: shubham Date: Tue, 31 Dec 2019 17:47:44 +0530 Subject: [PATCH 19/22] resolved conflicts Signed-off-by: shubham --- cmd/cspi-mgmt/app/handler.go | 11 ++--------- pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go | 3 --- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/cmd/cspi-mgmt/app/handler.go b/cmd/cspi-mgmt/app/handler.go index da1d2d7d3a..d34ba4528b 100644 --- a/cmd/cspi-mgmt/app/handler.go +++ b/cmd/cspi-mgmt/app/handler.go @@ -58,14 +58,8 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { // take a lock for common package for updating variables common.SyncResources.Mux.Lock() - // try to import pool, first check for migration case in which the previous - // poolname is CSP uid and should be renamed to CSPC uid. If old CSP uid is - // not present then follow the normal code path. - if cspi.Annotations[string(apis.OldCSPUID)] != "" { - isImported, err = zpool.Import(cspi, "cstor-"+cspi.Annotations[string(apis.OldCSPUID)]) - } else { - isImported, err = zpool.Import(cspi, "") - } + // try to import pool + isImported, err = zpool.Import(cspi) if isImported { if err != nil { common.SyncResources.Mux.Unlock() @@ -77,7 +71,6 @@ func (c *CStorPoolInstanceController) reconcile(key string) error { } zpool.CheckImportedPoolVolume() common.SyncResources.Mux.Unlock() - delete(cspi.Annotations, string(apis.OldCSPUID)) err = c.update(cspi) if err != nil { c.recorder.Event(cspi, diff --git a/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go b/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go index 9c0197fe53..3cc64ef775 100644 --- a/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go +++ b/pkg/apis/openebs.io/v1alpha1/cstorpool_cluster.go @@ -98,9 +98,6 @@ type PoolSpec struct { // PoolConfig is the default pool config that applies to the // pool on node. PoolConfig PoolConfig `json:"poolConfig"` - // OldCSPUID is used to migrate old csp to cspi. This will be the - // old pool name which needs to imported and renamed as new pool - OldCSPUID string `json:"oldCSPUID,omitempty"` } // PoolConfig is the default pool config that applies to the From 524897a44c7a1cfb8b3ad2e0931ed6470965b990 Mon Sep 17 00:00:00 2001 From: shubham Date: Tue, 7 Jan 2020 13:05:42 +0530 Subject: [PATCH 20/22] addressed review comments Signed-off-by: shubham --- pkg/cstor/migrate/cspc_generator.go | 22 +++++----- pkg/cstor/migrate/pool.go | 62 ++++++++++++++++------------- 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/pkg/cstor/migrate/cspc_generator.go b/pkg/cstor/migrate/cspc_generator.go index 79957d5242..8c05e7bd3d 100644 --- a/pkg/cstor/migrate/cspc_generator.go +++ b/pkg/cstor/migrate/cspc_generator.go @@ -77,7 +77,7 @@ func getCSPCSpecForSPC(spc *apis.StoragePoolClaim, openebsNamespace string) (*ap return nil, err } if len(cspDeployList.Items) != 1 { - return nil, errors.Errorf("invalid number of csp deployment found: %d", len(cspDeployList.Items)) + return nil, errors.Errorf("invalid number of deployments found for csp %s: %d", cspObj.Name, len(cspDeployList.Items)) } poolSpec := apis.PoolSpec{ NodeSelector: map[string]string{ @@ -131,20 +131,22 @@ func generateCSPC(spcObj *apis.StoragePoolClaim, openebsNamespace string) ( *apis.CStorPoolCluster, error) { cspcObj, err := cspc.NewKubeClient(). WithNamespace(openebsNamespace).Get(spcObj.Name, metav1.GetOptions{}) - if err == nil { - return cspcObj, nil - } if !k8serrors.IsNotFound(err) { return nil, err } - cspcObj, err = getCSPCSpecForSPC(spcObj, openebsNamespace) if err != nil { - return nil, err + cspcObj, err = getCSPCSpecForSPC(spcObj, openebsNamespace) + if err != nil { + return nil, err + } + cspcObj, err = cspc.NewKubeClient(). + WithNamespace(openebsNamespace).Create(cspcObj) + if err != nil { + return nil, err + } } - cspcObj, err = cspc.NewKubeClient(). - WithNamespace(openebsNamespace).Create(cspcObj) - if err != nil { - return nil, err + if cspcObj.Annotations[string(apis.OpenEBSDisableDependantsReconcileKey)] == "" { + return cspcObj, nil } // verify the number of cspi created is correct err = retry. diff --git a/pkg/cstor/migrate/pool.go b/pkg/cstor/migrate/pool.go index f60112ed50..d52cb7f50b 100644 --- a/pkg/cstor/migrate/pool.go +++ b/pkg/cstor/migrate/pool.go @@ -124,9 +124,13 @@ func validateSPC(spcObj *apis.StoragePoolClaim) error { if err != nil { return err } - if spcObj.Spec.MaxPools != nil { + if spcObj.Spec.BlockDevices.BlockDeviceList == nil { + if spcObj.Spec.MaxPools == nil { + return errors.Errorf("invalid spc %s neither has bdc list nor maxpools", spcObj.Name) + } if *spcObj.Spec.MaxPools != len(cspList.Items) { - return errors.Errorf("maxpool count does not match csp count") + return errors.Errorf("maxpool count does not match csp count expected: %d got: %d", + *spcObj.Spec.MaxPools, len(cspList.Items)) } return nil } @@ -238,7 +242,7 @@ func getCSP(cspLabel string) (*apis.CStorPool, error) { return nil, err } if len(cspList.Items) != 1 { - return nil, fmt.Errorf("Invalid number of pools on one node: %d", len(cspList.Items)) + return nil, fmt.Errorf("Invalid number of pools on one node: %v", cspList.Items) } cspObj := cspList.Items[0] return &cspObj, nil @@ -289,7 +293,7 @@ func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error // Update the bdc with the cspc labels instead of spc labels to allow // filtering of bds claimed by the migrated cspc. func updateBDCLabels(cspcName, openebsNamespace string) error { - bdcList, err := bdc.NewKubeClient().List(metav1.ListOptions{ + bdcList, err := bdc.NewKubeClient().WithNamespace(openebsNamespace).List(metav1.ListOptions{ LabelSelector: string(apis.StoragePoolClaimCPK) + "=" + cspcName, }) if err != nil { @@ -328,16 +332,18 @@ func updateBDCOwnerRef(cspcObj *apis.CStorPoolCluster, openebsNamespace string) return err } for _, bdcItem := range bdcList.Items { - bdcItem := bdcItem // pin it - bdcObj := &bdcItem - klog.Infof("Updating bdc %s with cspc ownerRef.", bdcObj.Name) - bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" - bdcObj.OwnerReferences[0].UID = cspcObj.UID - _, err := bdc.NewKubeClient(). - WithNamespace(openebsNamespace). - Update(bdcObj) - if err != nil { - return errors.Wrapf(err, "failed to update bdc %s with cspc onwerRef", bdcObj.Name) + if bdcItem.OwnerReferences[0].Kind != "CStorPoolCluster" { + bdcItem := bdcItem // pin it + bdcObj := &bdcItem + klog.Infof("Updating bdc %s with cspc ownerRef.", bdcObj.Name) + bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" + bdcObj.OwnerReferences[0].UID = cspcObj.UID + _, err := bdc.NewKubeClient(). + WithNamespace(openebsNamespace). + Update(bdcObj) + if err != nil { + return errors.Wrapf(err, "failed to update bdc %s with cspc onwerRef", bdcObj.Name) + } } } return nil @@ -354,19 +360,21 @@ func updateCVRsLabels(cspName, openebsNamespace string, cspiObj *apis.CStorPoolI return err } for _, cvrItem := range cvrList.Items { - cvrItem := cvrItem // pin it - cvrObj := &cvrItem - klog.Infof("Updating cvr %s with cspi %s info.", cvrObj.Name, cspiObj.Name) - delete(cvrObj.Labels, cspNameLabel) - delete(cvrObj.Labels, cspUIDLabel) - cvrObj.Labels[cspiNameLabel] = cspiObj.Name - cvrObj.Labels[cspiUIDLabel] = string(cspiObj.UID) - delete(cvrObj.Annotations, cspHostnameAnnotation) - cvrObj.Annotations[cspiHostnameAnnotation] = cspiObj.Spec.HostName - _, err = cvr.NewKubeclient().WithNamespace(openebsNamespace). - Update(cvrObj) - if err != nil { - return errors.Wrapf(err, "failed to update cvr %s with cspc info", cvrObj.Name) + if cvrItem.Labels[cspiNameLabel] == "" { + cvrItem := cvrItem // pin it + cvrObj := &cvrItem + klog.Infof("Updating cvr %s with cspi %s info.", cvrObj.Name, cspiObj.Name) + delete(cvrObj.Labels, cspNameLabel) + delete(cvrObj.Labels, cspUIDLabel) + cvrObj.Labels[cspiNameLabel] = cspiObj.Name + cvrObj.Labels[cspiUIDLabel] = string(cspiObj.UID) + delete(cvrObj.Annotations, cspHostnameAnnotation) + cvrObj.Annotations[cspiHostnameAnnotation] = cspiObj.Spec.HostName + _, err = cvr.NewKubeclient().WithNamespace(openebsNamespace). + Update(cvrObj) + if err != nil { + return errors.Wrapf(err, "failed to update cvr %s with cspc info", cvrObj.Name) + } } } return nil From aab7caa4a610a82c6cbb93ce9319a389bc46e5d7 Mon Sep 17 00:00:00 2001 From: shubham Date: Fri, 17 Jan 2020 12:27:48 +0530 Subject: [PATCH 21/22] added object name to logs and fixed annotation deletion Signed-off-by: shubham --- cmd/cstor-pool-mgmt/pool/v1alpha2/import.go | 22 +++++++----- pkg/cstor/migrate/pool.go | 38 ++++++++++----------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go b/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go index 558bade9c1..ec3e28d339 100644 --- a/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go +++ b/cmd/cstor-pool-mgmt/pool/v1alpha2/import.go @@ -35,6 +35,9 @@ import ( // uid name to the current PoolName(cspi) format func Import(cspi *apis.CStorPoolInstance) (bool, error) { if poolExist := checkIfPoolPresent(PoolName(cspi)); poolExist { + // If the pool is renamed and imported but the pool-mgmt restarts + // for some reason then the annotation should be removed. + delete(cspi.Annotations, string(apis.OldPoolName)) return true, nil } @@ -46,8 +49,8 @@ func Import(cspi *apis.CStorPoolInstance) (bool, error) { _, poolNotImported, _ = checkIfPoolIsImportable(cspi) if poolNotImported { - // if the pool is renamed but not imported remove the - // annotation to avoid not found errors + // If the pool is renamed but not imported, remove the + // annotation to avoid not found errors. delete(cspi.Annotations, string(apis.OldPoolName)) } @@ -56,19 +59,19 @@ func Import(cspi *apis.CStorPoolInstance) (bool, error) { return false, err } + // oldName denotes the pool name that may be present + // from previous version and needs to be imported with new name + oldName := cspi.Annotations[string(apis.OldPoolName)] + klog.Infof("Importing pool %s %s", string(cspi.GetUID()), PoolName(cspi)) devID := pool.GetDevPathIfNotSlashDev(bdPath[0]) cmd := zfs.NewPoolImport(). WithCachefile(cspi.Spec.PoolConfig.CacheFile). WithProperty("cachefile", cspi.Spec.PoolConfig.CacheFile). WithDirectory(devID). + WithPool(oldName). WithNewPool(PoolName(cspi)) - // oldName denotes the pool name that may be present - // from previous version and needs to be imported with new name - oldName := cspi.Annotations[string(apis.OldPoolName)] - if oldName != "" { - cmd.WithPool(oldName) - } + if len(devID) != 0 { cmdOut, err = cmd.Execute() if err == nil { @@ -84,7 +87,8 @@ func Import(cspi *apis.CStorPoolInstance) (bool, error) { cmdOut, err = zfs.NewPoolImport(). WithCachefile(cspi.Spec.PoolConfig.CacheFile). WithProperty("cachefile", cspi.Spec.PoolConfig.CacheFile). - WithPool(PoolName(cspi)). + WithNewPool(PoolName(cspi)). + WithPool(oldName). Execute() } diff --git a/pkg/cstor/migrate/pool.go b/pkg/cstor/migrate/pool.go index d52cb7f50b..51b04e5ecc 100644 --- a/pkg/cstor/migrate/pool.go +++ b/pkg/cstor/migrate/pool.go @@ -216,8 +216,8 @@ func csptocspi( return err1 } if cspiObj.Status.Phase != "ONLINE" { - return errors.Errorf("failed to verify cspi phase expected: Healthy got: %s", - cspiObj.Status.Phase) + return errors.Errorf("failed to verify cspi %s phase expected: Healthy got: %s", + cspiObj.Name, cspiObj.Status.Phase) } return nil }) @@ -261,7 +261,7 @@ func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error return err } if len(cspDeployList.Items) != 1 { - return errors.Errorf("invalid number of csp deployment found: %d", len(cspDeployList.Items)) + return errors.Errorf("invalid number of csp deployment found for %s: %d", len(cspDeployList.Items), cspObj.Name) } _, err = deploy.NewKubeClient().WithNamespace(openebsNamespace). Patch( @@ -300,23 +300,23 @@ func updateBDCLabels(cspcName, openebsNamespace string) error { return err } for _, bdcItem := range bdcList.Items { - bdcItem := bdcItem // pin it - bdcObj := &bdcItem - klog.Infof("Updating bdc %s with cspc labels & finalizer.", bdcObj.Name) - delete(bdcObj.Labels, string(apis.StoragePoolClaimCPK)) - bdcObj.Labels[string(apis.CStorPoolClusterCPK)] = cspcName - for i, finalizer := range bdcObj.Finalizers { - if finalizer == spcFinalizer { - bdcObj.Finalizers[i] = cspcFinalizer + if bdcItem.Labels[string(apis.StoragePoolClaimCPK)] != "" { + bdcItem := bdcItem // pin it + bdcObj := &bdcItem + klog.Infof("Updating bdc %s with cspc labels & finalizer.", bdcObj.Name) + delete(bdcObj.Labels, string(apis.StoragePoolClaimCPK)) + bdcObj.Labels[string(apis.CStorPoolClusterCPK)] = cspcName + for i, finalizer := range bdcObj.Finalizers { + if finalizer == spcFinalizer { + bdcObj.Finalizers[i] = cspcFinalizer + } + } + _, err := bdc.NewKubeClient(). + WithNamespace(openebsNamespace). + Update(bdcObj) + if err != nil { + return errors.Wrapf(err, "failed to update bdc %s with cspc label & finalizer", bdcObj.Name) } - } - // bdcObj.OwnerReferences[0].Kind = "CStorPoolCluster" - // bdcObj.OwnerReferences[0].UID = cspcObj.UID - _, err := bdc.NewKubeClient(). - WithNamespace(openebsNamespace). - Update(bdcObj) - if err != nil { - return errors.Wrapf(err, "failed to update bdc %s with cspc label & finalizer", bdcObj.Name) } } return nil From e5f6235f716bbefe719f6d346b9125941bc1ca01 Mon Sep 17 00:00:00 2001 From: shubham Date: Fri, 17 Jan 2020 12:34:11 +0530 Subject: [PATCH 22/22] fixed golang issue Signed-off-by: shubham --- pkg/cstor/migrate/pool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cstor/migrate/pool.go b/pkg/cstor/migrate/pool.go index 51b04e5ecc..122c7dcf75 100644 --- a/pkg/cstor/migrate/pool.go +++ b/pkg/cstor/migrate/pool.go @@ -261,7 +261,7 @@ func scaleDownDeployment(cspObj *apis.CStorPool, openebsNamespace string) error return err } if len(cspDeployList.Items) != 1 { - return errors.Errorf("invalid number of csp deployment found for %s: %d", len(cspDeployList.Items), cspObj.Name) + return errors.Errorf("invalid number of csp deployment found for %s: %d", cspObj.Name, len(cspDeployList.Items)) } _, err = deploy.NewKubeClient().WithNamespace(openebsNamespace). Patch(