Skip to content

Commit

Permalink
provisioner: add lvm provisioner
Browse files Browse the repository at this point in the history 
    - Now we could use the following fields to provision the volume group

    ```
    Spec:
      provisioner:
        lvm:
          vgName: vg001
    ```

    - Introduce the new field to trigger provision

    ```
    Spec:
      provision: true
    ```

    - The new CRD lvmvgs is introduced, this will contain the node info
      and the volume group information.

Signed-off-by: Vicente Cheng <[email protected]>
  • Loading branch information
@Vicente-Cheng @tserong
Vicente-Cheng authored and tserong committed Sep 12, 2024
1 parent d478cf9 commit a6b3584
Show file tree
Hide file tree
Showing 6 changed files with 647 additions and 4 deletions.
7 changes: 7 additions & 0 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/harvester/node-disk-manager/pkg/block"
blockdevicev1 "github.com/harvester/node-disk-manager/pkg/controller/blockdevice"
nodev1 "github.com/harvester/node-disk-manager/pkg/controller/node"
volumegroupv1 "github.com/harvester/node-disk-manager/pkg/controller/volumegroup"
"github.com/harvester/node-disk-manager/pkg/filter"
ctldisk "github.com/harvester/node-disk-manager/pkg/generated/controllers/harvesterhci.io"
ctllonghorn "github.com/harvester/node-disk-manager/pkg/generated/controllers/longhorn.io"
Expand Down Expand Up @@ -219,6 +220,7 @@ func run(opt *option.Option) error {
locker := &sync.Mutex{}
cond := sync.NewCond(locker)
bds := disks.Harvesterhci().V1beta1().BlockDevice()
lvmVGs := disks.Harvesterhci().V1beta1().LVMVolumeGroup()
nodes := lhs.Longhorn().V1beta2().Node()
scanner := blockdevicev1.NewScanner(
opt.NodeName,
Expand All @@ -237,6 +239,7 @@ func run(opt *option.Option) error {
ctx,
nodes,
bds,
lvmVGs,
block,
opt,
scanner,
Expand All @@ -248,6 +251,10 @@ func run(opt *option.Option) error {
logrus.Fatalf("failed to register ndm node controller, %s", err.Error())
}

if err := volumegroupv1.Register(ctx, lvmVGs, opt); err != nil {
logrus.Fatalf("failed to register ndm volume group controller, %s", err.Error())
}

if err := start.All(ctx, opt.Threadiness, disks, lhs); err != nil {
logrus.Fatalf("error starting, %s", err.Error())
}
Expand Down
40 changes: 36 additions & 4 deletions pkg/controller/blockdevice/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ type Controller struct {
BlockdeviceCache ctldiskv1.BlockDeviceCache
BlockInfo block.Info

LVMVgClient ctldiskv1.LVMVolumeGroupController

scanner *Scanner
semaphore *provisioner.Semaphore
}
Expand All @@ -61,6 +63,7 @@ func Register(
ctx context.Context,
nodes ctllonghornv1.NodeController,
bds ctldiskv1.BlockDeviceController,
lvmVGs ctldiskv1.LVMVolumeGroupController,
block block.Info,
opt *option.Option,
scanner *Scanner,
Expand All @@ -74,6 +77,7 @@ func Register(
Nodes: nodes,
Blockdevices: bds,
BlockdeviceCache: bds.Cache(),
LVMVgClient: lvmVGs,
BlockInfo: block,
scanner: scanner,
semaphore: semaphoreObj,
Expand Down Expand Up @@ -107,6 +111,10 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) (
logrus.Warnf("Failed to generate provisioner for device %s: %v", device.Name, err)
return nil, err
}
if provisionerInst == nil {
logrus.Infof("Skip device %s as no provisioner found or not configured", device.Name)
return nil, nil
}

// handle remove device no matter inactive or corrupted, we will set `device.Spec.FileSystem.Provisioned` to false
if needProvisionerUnprovision(device) {
Expand All @@ -133,6 +141,7 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) (
return nil, fmt.Errorf("failed to resolve persistent dev path for block device %s", device.Name)
}

logrus.Debugf("Checking to format device %s", device.Name)
if formatted, requeue, err := provisionerInst.Format(devPath); !formatted {
if requeue {
c.Blockdevices.EnqueueAfter(c.Namespace, device.Name, jitterEnqueueDelay())
Expand All @@ -153,6 +162,7 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) (
* 2. Spec.Filesystem.Provisioned = true, Status.ProvisionPhase = ProvisionPhaseUnprovisioned
* -> Provision the device
*/
logrus.Debugf("Checking to provision/update device %s", device.Name)
if needProvisionerUpdate(device, deviceCpy) {
logrus.Infof("Prepare to check the new device tags %v with device: %s", deviceCpy.Spec.Tags, device.Name)
requeue, err := provisionerInst.Update()
Expand Down Expand Up @@ -200,12 +210,26 @@ func (c *Controller) finalizeBlockDevice(oldBd, newBd *diskv1.BlockDevice, devPa
}

func (c *Controller) generateProvisioner(device *diskv1.BlockDevice) (provisioner.Provisioner, error) {
if device.Spec.Provisioner == nil && device.Status.ProvisionPhase != diskv1.ProvisionPhaseProvisioned {
return nil, nil
}
logrus.Infof("Generate provisioner from device %s, content: %v", device.Name, device.Spec.Provisioner)
// set default
provisionerType := provisioner.TypeLonghornV1
if device.Spec.Provisioner != nil {
// **TODO**: we should use webhook to validate the provisioner type (and number)
numProvisioner := 0
if device.Spec.Provisioner.Longhorn != nil {
numProvisioner++
provisionerType = device.Spec.Provisioner.Longhorn.EngineVersion
}
if device.Spec.Provisioner.LVM != nil {
numProvisioner++
provisionerType = provisioner.TypeLVM
}
if numProvisioner > 1 {
return nil, fmt.Errorf("multiple provisioner types found for block device %s", device.Name)
}
}
switch provisionerType {
case provisioner.TypeLonghornV1:
Expand All @@ -220,7 +244,7 @@ func (c *Controller) generateProvisioner(device *diskv1.BlockDevice) (provisione
case provisioner.TypeLonghornV2:
return nil, fmt.Errorf("TBD type %s", provisionerType)
case provisioner.TypeLVM:
return nil, fmt.Errorf("TBD type %s", provisionerType)
return c.generateLVMProvisioner(device)
default:
return nil, fmt.Errorf("unsupported provisioner type %s", provisionerType)
}
Expand All @@ -237,6 +261,11 @@ func (c *Controller) generateLHv1Provisioner(device *diskv1.BlockDevice) (provis
return provisioner.NewLHV1Provisioner(device, c.BlockInfo, node, c.Nodes, c.NodeCache, CacheDiskTags, c.semaphore)
}

func (c *Controller) generateLVMProvisioner(device *diskv1.BlockDevice) (provisioner.Provisioner, error) {
vgName := device.Spec.Provisioner.LVM.VgName
return provisioner.NewLVMProvisioner(vgName, c.NodeName, c.LVMVgClient, device, c.BlockInfo)
}

func (c *Controller) updateDeviceStatus(device *diskv1.BlockDevice, devPath string) error {
var newStatus diskv1.DeviceStatus
var needAutoProvision bool
Expand Down Expand Up @@ -365,13 +394,16 @@ func canSkipBlockDeviceChange(device *diskv1.BlockDevice, nodeName string) bool
}

func needProvisionerUnprovision(device *diskv1.BlockDevice) bool {
return !device.Spec.FileSystem.Provisioned && device.Status.ProvisionPhase != diskv1.ProvisionPhaseUnprovisioned
return (!device.Spec.FileSystem.Provisioned && !device.Spec.Provision) &&
device.Status.ProvisionPhase != diskv1.ProvisionPhaseUnprovisioned
}

func needProvisionerUpdate(oldBd, newBd *diskv1.BlockDevice) bool {
return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseProvisioned && newBd.Spec.FileSystem.Provisioned
return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseProvisioned &&
(newBd.Spec.FileSystem.Provisioned || newBd.Spec.Provision)
}

func needProvisionerProvision(oldBd, newBd *diskv1.BlockDevice) bool {
return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseUnprovisioned && newBd.Spec.FileSystem.Provisioned
return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseUnprovisioned &&
(newBd.Spec.FileSystem.Provisioned || newBd.Spec.Provision)
}
Loading

0 comments on commit a6b3584

Please sign in to comment.