feat(provisioning): add support for multiple vg to use for provisioning

Signed-off-by: Yashpal Choudhary <[email protected]>

deploy/lvm-operator.yaml

@@ -118,14 +118,18 @@ spec:
                 - "yes"
                 - "no"
                 type: string
+              vgPattern:
+                description: VgPattern specifies the regex to choose volume groups
+                  where volume needs to be created.
+                type: string
               volGroup:
                 description: VolGroup specifies the name of the volume group where
                   the volume has been created.
-                minLength: 1
                 type: string
             required:
             - capacity
             - ownerNodeID
+            - vgPattern
             - volGroup
             type: object
           status:
@@ -244,14 +248,18 @@ spec:
                 - "yes"
                 - "no"
                 type: string
+              vgPattern:
+                description: VgPattern specifies the regex to choose volume groups
+                  where volume needs to be created.
+                type: string
               volGroup:
                 description: VolGroup specifies the name of the volume group where
                   the volume has been created.
-                minLength: 1
                 type: string
             required:
             - capacity
             - ownerNodeID
+            - vgPattern
             - volGroup
             type: object
           status:

deploy/yamls/lvmsnapshot-crd.yaml

@@ -74,14 +74,18 @@ spec:
                 - "yes"
                 - "no"
                 type: string
+              vgPattern:
+                description: VgPattern specifies the regex to choose volume groups
+                  where volume needs to be created.
+                type: string
               volGroup:
                 description: VolGroup specifies the name of the volume group where
                   the volume has been created.
-                minLength: 1
                 type: string
             required:
             - capacity
             - ownerNodeID
+            - vgPattern
             - volGroup
             type: object
           status:

deploy/yamls/lvmvolume-crd.yaml

@@ -97,14 +97,18 @@ spec:
                 - "yes"
                 - "no"
                 type: string
+              vgPattern:
+                description: VgPattern specifies the regex to choose volume groups
+                  where volume needs to be created.
+                type: string
               volGroup:
                 description: VolGroup specifies the name of the volume group where
                   the volume has been created.
-                minLength: 1
                 type: string
             required:
             - capacity
             - ownerNodeID
+            - vgPattern
             - volGroup
             type: object
           status:

e2e-tests/experiments/functional/lvm-volume-resize/README.md

@@ -46,4 +46,4 @@ then update the needed test specific values in run_e2e_test.yml file and create
 ```
 kubectl create -f run_e2e_test.yml
 ```
-All the env variables description is provided with the comments in the same file.
+All the env variables description is provided with the comments in the same file.

pkg/apis/openebs.io/lvm/v1alpha1/lvmvolume.go

@@ -62,9 +62,13 @@ type VolumeInfo struct {
 
 	// VolGroup specifies the name of the volume group where the volume has been created.
 	// +kubebuilder:validation:Required
-	// +kubebuilder:validation:MinLength=1
 	VolGroup string `json:"volGroup"`
 
+	// VgPattern specifies the regex to choose volume groups where volume
+	// needs to be created.
+	// +kubebuilder:validation:Required
+	VgPattern string `json:"vgPattern"`
+
 	// Capacity of the volume
 	// +kubebuilder:validation:Required
 	// +kubebuilder:validation:MinLength=1

pkg/builder/volbuilder/volume.go

@@ -154,6 +154,21 @@ func (b *Builder) WithVolGroup(vg string) *Builder {
 	return b
 }
 
+// WithVgPattern sets volume group regex pattern.
+func (b *Builder) WithVgPattern(pattern string) *Builder {
+	if pattern == "" {
+		b.errs = append(
+			b.errs,
+			errors.New(
+				"failed to build lvm volume object: missing vg name",
+			),
+		)
+		return b
+	}
+	b.volume.Object.Spec.VgPattern = pattern
+	return b
+}
+
 // WithNodeName sets NodeID for creating the volume
 func (b *Builder) WithNodeName(name string) *Builder {
 	if name == "" {

pkg/driver/controller.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	k8sapi "github.com/openebs/lib-csi/pkg/client/k8s"
-	"github.com/openebs/lib-csi/pkg/common/helpers"
 	clientset "github.com/openebs/lvm-localpv/pkg/generated/clientset/internalclientset"
 	informers "github.com/openebs/lvm-localpv/pkg/generated/informer/externalversions"
 	kubeinformers "k8s.io/client-go/informers"
@@ -250,7 +249,7 @@ func CreateLVMVolume(ctx context.Context, req *csi.CreateVolumeRequest,
 		}
 	}
 
-	nmap, err := getNodeMap(params.Scheduler, params.VolumeGroup)
+	nmap, err := getNodeMap(params.Scheduler, params.VgPattern)
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "get node map failed : %s", err.Error())
 	}
@@ -263,12 +262,13 @@ func CreateLVMVolume(ctx context.Context, req *csi.CreateVolumeRequest,
 	}
 
 	owner := selected[0]
-	klog.Infof("scheduling the volume %s/%s on node %s", params.VolumeGroup, volName, owner)
+	klog.Infof("scheduling the volume %s/%s on node %s",
+		params.VgPattern.String(), volName, owner)
 
 	volObj, err := volbuilder.NewBuilder().
 		WithName(volName).
 		WithCapacity(capacity).
-		WithVolGroup(params.VolumeGroup).
+		WithVgPattern(params.VgPattern.String()).
 		WithOwnerNode(owner).
 		WithVolumeStatus(lvm.LVMStatusPending).
 		WithShared(params.Shared).
@@ -323,7 +323,7 @@ func (cs *controller) CreateVolume(
 		"lvm-localpv", analytics.VolumeProvision)
 
 	topology := map[string]string{lvm.LVMTopologyKey: vol.Spec.OwnerNodeID}
-	cntx := map[string]string{lvm.VolGroupKey: params.VolumeGroup}
+	cntx := map[string]string{lvm.VolGroupKey: vol.Spec.VolGroup}
 
 	return csipayload.NewCreateVolumeResponseBuilder().
 		WithName(volName).
@@ -704,8 +704,12 @@ func (cs *controller) GetCapacity(
 	}
 
 	lvmNodesCache := cs.lvmNodeInformer.GetIndexer()
-	params := req.GetParameters()
-	vgParam := helpers.GetInsensitiveParameter(&params, "volgroup")
+
+	params, err := NewVolumeParams(req.GetParameters())
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument,
+			"failed to parse csi volume params: %v", err)
+	}
 
 	var availableCapacity int64
 	for _, nodeName := range nodeNames {
@@ -723,7 +727,7 @@ func (cs *controller) GetCapacity(
 		// See https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/1472-storage-capacity-tracking#available-capacity-vs-maximum-volume-size &
 		// https://github.com/container-storage-interface/spec/issues/432 for more details
 		for _, vg := range lvmNode.VolumeGroups {
-			if vg.Name != vgParam {
+			if !params.VgPattern.MatchString(vg.Name) {
 				continue
 			}
 			freeCapacity := vg.Free.Value()

pkg/driver/params.go

@@ -17,15 +17,18 @@
 package driver
 
 import (
+	"fmt"
+	"regexp"
+
 	"github.com/openebs/lib-csi/pkg/common/helpers"
 )
 
 // VolumeParams holds collection of supported settings that can
 // be configured in storage class.
 type VolumeParams struct {
-	// VolumeGroup specifies vg name to use for
+	// VgPattern specifies vg regex to use for
 	// provisioning logical volumes.
-	VolumeGroup string
+	VgPattern *regexp.Regexp
 
 	Scheduler     string
 	Shared        string
@@ -50,7 +53,18 @@ func NewVolumeParams(m map[string]string) (*VolumeParams, error) {
 	// parameter keys (not values!) are all lowercase, keys may safely be forced
 	// to the lower case.
 	m = helpers.GetCaseInsensitiveMap(&m)
-	params.VolumeGroup = m["volgroup"]
+
+	// for ensuring backward compatibility, we first check if
+	// there is any volgroup param exists for storage class.
+	vgPattern, ok := m["volgroup"]
+	if !ok {
+		vgPattern = m["vgpattern"]
+	}
+
+	var err error
+	if params.VgPattern, err = regexp.Compile(vgPattern); err != nil {
+		return nil, fmt.Errorf("invalid volgroup/vgpattern param %v: %v", vgPattern, err)
+	}
 
 	// parse string params
 	stringParams := map[string]*string{

pkg/driver/schd_helper.go

@@ -17,6 +17,7 @@ limitations under the License.
 package driver
 
 import (
+	"regexp"
 	"strconv"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -39,7 +40,7 @@ const (
 // and creates the node mapping of the volume for all the nodes.
 // It returns a map which has nodes as key and volumes present
 // on the nodes as corresponding value.
-func getVolumeWeightedMap(vg string) (map[string]int64, error) {
+func getVolumeWeightedMap(re *regexp.Regexp) (map[string]int64, error) {
 	nmap := map[string]int64{}
 
 	vollist, err := volbuilder.NewKubeclient().
@@ -53,7 +54,7 @@ func getVolumeWeightedMap(vg string) (map[string]int64, error) {
 	// create the map of the volume count
 	// for the given vg
 	for _, vol := range vollist.Items {
-		if vol.Spec.VolGroup == vg {
+		if re.MatchString(vol.Spec.VolGroup) {
 			nmap[vol.Spec.OwnerNodeID]++
 		}
 	}
@@ -66,7 +67,7 @@ func getVolumeWeightedMap(vg string) (map[string]int64, error) {
 // It returns a map which has nodes as key and capacity provisioned
 // on the nodes as corresponding value. The scheduler will use this map
 // and picks the node which is less weighted.
-func getCapacityWeightedMap(vg string) (map[string]int64, error) {
+func getCapacityWeightedMap(re *regexp.Regexp) (map[string]int64, error) {
 	nmap := map[string]int64{}
 
 	volList, err := volbuilder.NewKubeclient().
@@ -80,7 +81,7 @@ func getCapacityWeightedMap(vg string) (map[string]int64, error) {
 	// create the map of the volume capacity
 	// for the given volume group
 	for _, vol := range volList.Items {
-		if vol.Spec.VolGroup == vg {
+		if re.MatchString(vol.Spec.VolGroup) {
 			volSize, err := strconv.ParseInt(vol.Spec.Capacity, 10, 64)
 			if err == nil {
 				nmap[vol.Spec.OwnerNodeID] += volSize
@@ -92,13 +93,13 @@ func getCapacityWeightedMap(vg string) (map[string]int64, error) {
 }
 
 // getNodeMap returns the node mapping for the given scheduling algorithm
-func getNodeMap(schd string, vg string) (map[string]int64, error) {
+func getNodeMap(schd string, vgPattern *regexp.Regexp) (map[string]int64, error) {
 	switch schd {
 	case VolumeWeighted:
-		return getVolumeWeightedMap(vg)
+		return getVolumeWeightedMap(vgPattern)
 	case CapacityWeighted:
-		return getCapacityWeightedMap(vg)
+		return getCapacityWeightedMap(vgPattern)
 	}
 	// return CapacityWeighted(default) if not specified
-	return getCapacityWeightedMap(vg)
+	return getCapacityWeightedMap(vgPattern)
 }

pkg/lvm/lvm_util.go

@@ -144,6 +144,11 @@ func CreateVolume(vol *apis.LVMVolume) error {
 
 // DestroyVolume deletes the lvm volume
 func DestroyVolume(vol *apis.LVMVolume) error {
+	if vol.Spec.VolGroup == "" {
+		klog.Infof("volGroup not set for lvm volume %v, skipping its deletion", vol.Name)
+		return nil
+	}
+
 	volume := vol.Spec.VolGroup + "/" + vol.Name
 
 	volExists, err := CheckVolumeExists(vol)

pkg/lvm/volume.go

@@ -196,6 +196,16 @@ func UpdateVolInfo(vol *apis.LVMVolume, state string) error {
 	return err
 }
 
+// UpdateVolGroup updates LVMVolume CR with volGroup name.
+func UpdateVolGroup(vol *apis.LVMVolume, vgName string) (*apis.LVMVolume, error) {
+	newVol, err := volbuilder.BuildFrom(vol).
+		WithVolGroup(vgName).Build()
+	if err != nil {
+		return nil, err
+	}
+	return volbuilder.NewKubeclient().WithNamespace(LvmNamespace).Update(newVol)
+}
+
 // RemoveVolFinalizer adds finalizer to LVMVolume CR
 func RemoveVolFinalizer(vol *apis.LVMVolume) error {
 	vol.Finalizers = nil