feat(localpv-hostpath): support provisioning on nodes with taints (#1560

)

Provisioning Local PV with hostpath requires launching of helper pods to create or delete the PV directory. This feature support launching helper pods on nodes that have taints. 

The provisioner will get the taints applied on the node and apply them as tolerations to the helper pods. 

Refer: openebs/openebs#2860

This commit does the following:
- Adds - Taints information to the podOpts
- Extends the Pod Builder with a new method that will convert taints to tolerations
- A node object is passed to the PV creation, so taints are easily extracted. 
- During PV deletion, fetch the node object from hostname - used during delete of the volume

Signed-off-by: Rahul M Chheda <[email protected]>

cmd/provisioner-localpv/app/config.go

@@ -227,3 +227,9 @@ func GetNodeHostname(n *v1.Node) string {
 	}
 	return hostname
 }
+
+// GetTaints extracts the Taints from the Spec on the node
+// If Taints are empty, it just returns empty structure of corev1.Taints
+func GetTaints(n *v1.Node) []v1.Taint {
+	return n.Spec.Taints
+}

cmd/provisioner-localpv/app/helper_hostpath.go

@@ -38,6 +38,7 @@ import (
 type podConfig struct {
 	pOpts                         *HelperPodOptions
 	parentDir, volumeDir, podName string
+	taints                        []corev1.Taint
 }
 
 var (
@@ -66,6 +67,8 @@ type HelperPodOptions struct {
 
 	// serviceAccountName is the service account with which the pod should be launched
 	serviceAccountName string
+
+	selectedNodeTaints []corev1.Taint
 }
 
 // validate checks that the required fields to launch
@@ -104,6 +107,9 @@ func (p *Provisioner) createInitPod(pOpts *HelperPodOptions) error {
 		return vErr
 	}
 
+	//Pass on the taints, to create tolerations.
+	config.taints = pOpts.selectedNodeTaints
+
 	iPod, err := p.launchPod(config)
 	if err != nil {
 		return err
@@ -129,6 +135,7 @@ func (p *Provisioner) createCleanupPod(pOpts *HelperPodOptions) error {
 		return err
 	}
 
+	config.taints = pOpts.selectedNodeTaints
 	// Initialize HostPath builder and validate that
 	// volume directory is not directly under root.
 	// Extract the base path and the volume unique path.
@@ -157,11 +164,12 @@ func (p *Provisioner) launchPod(config podConfig) (*corev1.Pod, error) {
 	// Helper pods need to create and delete directories on the host.
 	privileged := true
 
-	helperPod, _ := pod.NewBuilder().
+	helperPod, err := pod.NewBuilder().
 		WithName(config.podName + "-" + config.pOpts.name).
 		WithRestartPolicy(corev1.RestartPolicyNever).
 		WithNodeSelectorHostnameNew(config.pOpts.nodeHostname).
 		WithServiceAccountName(config.pOpts.serviceAccountName).
+		WithTolerationsForTaints(config.taints...).
 		WithContainerBuilder(
 			container.NewBuilder().
 				WithName("local-path-" + config.podName).

cmd/provisioner-localpv/app/provisioner_hostpath.go

@@ -20,6 +20,8 @@ import (
 	"github.com/openebs/maya/pkg/alertlog"
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/klog"
 
 	pvController "sigs.k8s.io/sig-storage-lib-external-provisioner/controller"
@@ -33,6 +35,7 @@ import (
 func (p *Provisioner) ProvisionHostPath(opts pvController.VolumeOptions, volumeConfig *VolumeConfig) (*v1.PersistentVolume, error) {
 	pvc := opts.PVC
 	nodeHostname := GetNodeHostname(opts.SelectedNode)
+	taints := GetTaints(opts.SelectedNode)
 	name := opts.PVName
 	stgType := volumeConfig.GetStorageType()
 	saName := getOpenEBSServiceAccountName()
@@ -59,8 +62,8 @@ func (p *Provisioner) ProvisionHostPath(opts pvController.VolumeOptions, volumeC
 		path:               path,
 		nodeHostname:       nodeHostname,
 		serviceAccountName: saName,
+		selectedNodeTaints: taints,
 	}
-
 	iErr := p.createInitPod(podOpts)
 	if iErr != nil {
 		klog.Infof("Initialize volume %v failed: %v", name, iErr)
@@ -123,6 +126,21 @@ func (p *Provisioner) ProvisionHostPath(opts pvController.VolumeOptions, volumeC
 	return pvObj, nil
 }
 
+// GetNodeObjectFromHostName returns the Node Object with matching NodeHostName.
+func (p *Provisioner) GetNodeObjectFromHostName(hostName string) (*v1.Node, error) {
+	labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{persistentvolume.KeyNode: hostName}}
+	listOptions := metav1.ListOptions{
+		LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
+		Limit:         1,
+	}
+	nodeList, err := p.kubeClient.CoreV1().Nodes().List(listOptions)
+	if err != nil {
+		return nil, errors.Errorf("Unable to get the Node with the NodeHostName")
+	}
+	return &nodeList.Items[0], nil
+
+}
+
 // DeleteHostPath is invoked by the PVC controller to perform clean-up
 //  activities before deleteing the PV object. If reclaim policy is
 //  set to not-retain, then this function will create a helper pod
@@ -133,7 +151,6 @@ func (p *Provisioner) DeleteHostPath(pv *v1.PersistentVolume) (err error) {
 	}()
 
 	saName := getOpenEBSServiceAccountName()
-
 	//Determine the path and node of the Local PV.
 	pvObj := persistentvolume.NewForAPIObject(pv)
 	path := pvObj.GetPath()
@@ -145,7 +162,14 @@ func (p *Provisioner) DeleteHostPath(pv *v1.PersistentVolume) (err error) {
 	if hostname == "" {
 		return errors.Errorf("cannot find affinited node hostname")
 	}
+	alertlog.Logger.Infof("Get the Node Object from hostName: %v", hostname)
 
+	//Get the node Object once again to get updated Taints.
+	nodeObject, err := p.GetNodeObjectFromHostName(hostname)
+	if err != nil {
+		return err
+	}
+	taints := GetTaints(nodeObject)
 	//Initiate clean up only when reclaim policy is not retain.
 	klog.Infof("Deleting volume %v at %v:%v", pv.Name, hostname, path)
 	cleanupCmdsForPath := []string{"rm", "-rf"}
@@ -155,6 +179,7 @@ func (p *Provisioner) DeleteHostPath(pv *v1.PersistentVolume) (err error) {
 		path:               path,
 		nodeHostname:       hostname,
 		serviceAccountName: saName,
+		selectedNodeTaints: taints,
 	}
 
 	if err := p.createCleanupPod(podOpts); err != nil {

pkg/kubernetes/pod/v1alpha1/build.go

@@ -40,6 +40,30 @@ func NewBuilder() *Builder {
 	return &Builder{pod: &Pod{object: &corev1.Pod{}}}
 }
 
+// WithTolerationsForTaints sets the Spec.Tolerations with provided taints.
+func (b *Builder) WithTolerationsForTaints(taints ...corev1.Taint) *Builder {
+
+	tolerations := []corev1.Toleration{}
+	for i := range taints {
+		var toleration corev1.Toleration
+		toleration.Key = taints[i].Key
+		toleration.Effect = taints[i].Effect
+		if len(taints[i].Value) == 0 {
+			toleration.Operator = corev1.TolerationOpExists
+		} else {
+			toleration.Value = taints[i].Value
+			toleration.Operator = corev1.TolerationOpEqual
+		}
+		tolerations = append(tolerations, toleration)
+	}
+
+	b.pod.object.Spec.Tolerations = append(
+		b.pod.object.Spec.Tolerations,
+		tolerations...,
+	)
+	return b
+}
+
 // WithName sets the Name field of Pod with provided value.
 func (b *Builder) WithName(name string) *Builder {
 	if len(name) == 0 {