From 5228dc2ed837f0f22f7bed7451c6e54270206586 Mon Sep 17 00:00:00 2001 From: Rahul M Chheda <53308066+rahulchheda@users.noreply.github.com> Date: Tue, 7 Jan 2020 18:03:31 +0530 Subject: [PATCH] feat(localpv-hostpath): support provisioning on nodes with taints (#1560) Provisioning Local PV with hostpath requires launching of helper pods to create or delete the PV directory. This feature support launching helper pods on nodes that have taints. The provisioner will get the taints applied on the node and apply them as tolerations to the helper pods. Refer: openebs/openebs#2860 This commit does the following: - Adds - Taints information to the podOpts - Extends the Pod Builder with a new method that will convert taints to tolerations - A node object is passed to the PV creation, so taints are easily extracted. - During PV deletion, fetch the node object from hostname - used during delete of the volume Signed-off-by: Rahul M Chheda --- cmd/provisioner-localpv/app/config.go | 6 ++++ .../app/helper_hostpath.go | 10 ++++++- .../app/provisioner_hostpath.go | 29 +++++++++++++++++-- pkg/kubernetes/pod/v1alpha1/build.go | 24 +++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/cmd/provisioner-localpv/app/config.go b/cmd/provisioner-localpv/app/config.go index b070c99cef..129d90a8b8 100644 --- a/cmd/provisioner-localpv/app/config.go +++ b/cmd/provisioner-localpv/app/config.go @@ -227,3 +227,9 @@ func GetNodeHostname(n *v1.Node) string { } return hostname } + +// GetTaints extracts the Taints from the Spec on the node +// If Taints are empty, it just returns empty structure of corev1.Taints +func GetTaints(n *v1.Node) []v1.Taint { + return n.Spec.Taints +} diff --git a/cmd/provisioner-localpv/app/helper_hostpath.go b/cmd/provisioner-localpv/app/helper_hostpath.go index 10c5b18a06..5737e675b9 100644 --- a/cmd/provisioner-localpv/app/helper_hostpath.go +++ b/cmd/provisioner-localpv/app/helper_hostpath.go @@ -38,6 +38,7 @@ import ( type podConfig struct { pOpts *HelperPodOptions parentDir, volumeDir, podName string + taints []corev1.Taint } var ( @@ -66,6 +67,8 @@ type HelperPodOptions struct { // serviceAccountName is the service account with which the pod should be launched serviceAccountName string + + selectedNodeTaints []corev1.Taint } // validate checks that the required fields to launch @@ -104,6 +107,9 @@ func (p *Provisioner) createInitPod(pOpts *HelperPodOptions) error { return vErr } + //Pass on the taints, to create tolerations. + config.taints = pOpts.selectedNodeTaints + iPod, err := p.launchPod(config) if err != nil { return err @@ -129,6 +135,7 @@ func (p *Provisioner) createCleanupPod(pOpts *HelperPodOptions) error { return err } + config.taints = pOpts.selectedNodeTaints // Initialize HostPath builder and validate that // volume directory is not directly under root. // Extract the base path and the volume unique path. @@ -157,11 +164,12 @@ func (p *Provisioner) launchPod(config podConfig) (*corev1.Pod, error) { // Helper pods need to create and delete directories on the host. privileged := true - helperPod, _ := pod.NewBuilder(). + helperPod, err := pod.NewBuilder(). WithName(config.podName + "-" + config.pOpts.name). WithRestartPolicy(corev1.RestartPolicyNever). WithNodeSelectorHostnameNew(config.pOpts.nodeHostname). WithServiceAccountName(config.pOpts.serviceAccountName). + WithTolerationsForTaints(config.taints...). WithContainerBuilder( container.NewBuilder(). WithName("local-path-" + config.podName). diff --git a/cmd/provisioner-localpv/app/provisioner_hostpath.go b/cmd/provisioner-localpv/app/provisioner_hostpath.go index 9dca6f26fd..affedc49f9 100644 --- a/cmd/provisioner-localpv/app/provisioner_hostpath.go +++ b/cmd/provisioner-localpv/app/provisioner_hostpath.go @@ -20,6 +20,8 @@ import ( "github.com/openebs/maya/pkg/alertlog" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/klog" pvController "sigs.k8s.io/sig-storage-lib-external-provisioner/controller" @@ -33,6 +35,7 @@ import ( func (p *Provisioner) ProvisionHostPath(opts pvController.VolumeOptions, volumeConfig *VolumeConfig) (*v1.PersistentVolume, error) { pvc := opts.PVC nodeHostname := GetNodeHostname(opts.SelectedNode) + taints := GetTaints(opts.SelectedNode) name := opts.PVName stgType := volumeConfig.GetStorageType() saName := getOpenEBSServiceAccountName() @@ -59,8 +62,8 @@ func (p *Provisioner) ProvisionHostPath(opts pvController.VolumeOptions, volumeC path: path, nodeHostname: nodeHostname, serviceAccountName: saName, + selectedNodeTaints: taints, } - iErr := p.createInitPod(podOpts) if iErr != nil { klog.Infof("Initialize volume %v failed: %v", name, iErr) @@ -123,6 +126,21 @@ func (p *Provisioner) ProvisionHostPath(opts pvController.VolumeOptions, volumeC return pvObj, nil } +// GetNodeObjectFromHostName returns the Node Object with matching NodeHostName. +func (p *Provisioner) GetNodeObjectFromHostName(hostName string) (*v1.Node, error) { + labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{persistentvolume.KeyNode: hostName}} + listOptions := metav1.ListOptions{ + LabelSelector: labels.Set(labelSelector.MatchLabels).String(), + Limit: 1, + } + nodeList, err := p.kubeClient.CoreV1().Nodes().List(listOptions) + if err != nil { + return nil, errors.Errorf("Unable to get the Node with the NodeHostName") + } + return &nodeList.Items[0], nil + +} + // DeleteHostPath is invoked by the PVC controller to perform clean-up // activities before deleteing the PV object. If reclaim policy is // set to not-retain, then this function will create a helper pod @@ -133,7 +151,6 @@ func (p *Provisioner) DeleteHostPath(pv *v1.PersistentVolume) (err error) { }() saName := getOpenEBSServiceAccountName() - //Determine the path and node of the Local PV. pvObj := persistentvolume.NewForAPIObject(pv) path := pvObj.GetPath() @@ -145,7 +162,14 @@ func (p *Provisioner) DeleteHostPath(pv *v1.PersistentVolume) (err error) { if hostname == "" { return errors.Errorf("cannot find affinited node hostname") } + alertlog.Logger.Infof("Get the Node Object from hostName: %v", hostname) + //Get the node Object once again to get updated Taints. + nodeObject, err := p.GetNodeObjectFromHostName(hostname) + if err != nil { + return err + } + taints := GetTaints(nodeObject) //Initiate clean up only when reclaim policy is not retain. klog.Infof("Deleting volume %v at %v:%v", pv.Name, hostname, path) cleanupCmdsForPath := []string{"rm", "-rf"} @@ -155,6 +179,7 @@ func (p *Provisioner) DeleteHostPath(pv *v1.PersistentVolume) (err error) { path: path, nodeHostname: hostname, serviceAccountName: saName, + selectedNodeTaints: taints, } if err := p.createCleanupPod(podOpts); err != nil { diff --git a/pkg/kubernetes/pod/v1alpha1/build.go b/pkg/kubernetes/pod/v1alpha1/build.go index 9c258ad226..e638393886 100644 --- a/pkg/kubernetes/pod/v1alpha1/build.go +++ b/pkg/kubernetes/pod/v1alpha1/build.go @@ -40,6 +40,30 @@ func NewBuilder() *Builder { return &Builder{pod: &Pod{object: &corev1.Pod{}}} } +// WithTolerationsForTaints sets the Spec.Tolerations with provided taints. +func (b *Builder) WithTolerationsForTaints(taints ...corev1.Taint) *Builder { + + tolerations := []corev1.Toleration{} + for i := range taints { + var toleration corev1.Toleration + toleration.Key = taints[i].Key + toleration.Effect = taints[i].Effect + if len(taints[i].Value) == 0 { + toleration.Operator = corev1.TolerationOpExists + } else { + toleration.Value = taints[i].Value + toleration.Operator = corev1.TolerationOpEqual + } + tolerations = append(tolerations, toleration) + } + + b.pod.object.Spec.Tolerations = append( + b.pod.object.Spec.Tolerations, + tolerations..., + ) + return b +} + // WithName sets the Name field of Pod with provided value. func (b *Builder) WithName(name string) *Builder { if len(name) == 0 {