Skip to content

Commit

Permalink
Store taints in node annotation
Browse files Browse the repository at this point in the history 
Storing taints in annotation helps to identify if the taint set on
node is owned by NFD or not. When user deletes the taint entry from
NodeFeatureRule CR, NFD will remove the taint from the node. But
to avoid accidental deletion of taints not owned by the NFD, it
needs to know the owner. Keeping track of NFD set taints in the
annotation can be used during the filtering of the owner.

Signed-off-by: Feruzjon Muyassarov <[email protected]>
  • Loading branch information
@fmuyassarov
fmuyassarov committed Oct 14, 2022
1 parent 6425372 commit a990e1b
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 21 deletions.
3 changes: 3 additions & 0 deletions pkg/apis/nfd/v1alpha1/annotations_labels.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,7 @@ const (

// WorkerVersionAnnotation is the annotation that holds the version of nfd-worker running on the node
WorkerVersionAnnotation = AnnotationNs + "/worker.version"

// NodeTaintsAnnotation is the annotation that holds the taints that nfd-master set on the node
NodeTaintsAnnotation = AnnotationNs + "/taints"
)
121 changes: 100 additions & 21 deletions pkg/nfd-master/nfd-master.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,7 @@ func (m *nfdMaster) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*pb.Se
rawLabels = r.Labels
}
crLabels, mustTaint := m.crLabels(r)

for k, v := range crLabels {
rawLabels[k] = v
}
Expand All @@ -436,11 +437,17 @@ func (m *nfdMaster) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*pb.Se
klog.Errorf("failed to advertise labels: %v", err)
return &pb.SetLabelsReply{}, err
}

// ensure we delete the old stale taints
labelReply, err := m.deleteTaints(c, r)
if err != nil {
return labelReply, err
}

if !m.args.NoTaint {
// Set NodeFeatureRule provided taints
if mustTaint {
klog.Infof("nfd tainting feature is enabled")
labelReply, err := m.SetTaints(c, r)
labelReply, err := m.setTaints(c, r)
if err != nil {
return labelReply, err
}
Expand All @@ -450,11 +457,12 @@ func (m *nfdMaster) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*pb.Se
return &pb.SetLabelsReply{}, nil
}

// SetTaints sets taint(s) on the node matching rule(s) in the NodeFeatureRule CRD
func (m *nfdMaster) SetTaints(c context.Context, r *pb.SetLabelsRequest) (*pb.SetLabelsReply, error) {
err := authorizeClient(c, m.args.VerifyNodeName, r.NodeName)
taints := []corev1.Taint{}
// setTaints sets taint(s) on the node matching rule(s) in the NodeFeatureRule CRD
func (m *nfdMaster) setTaints(c context.Context, r *pb.SetLabelsRequest) (*pb.SetLabelsReply, error) {
var nodeTaints, crt []api.Taint
taintStrs := []string{}

err := authorizeClient(c, m.args.VerifyNodeName, r.NodeName)
if err != nil {
return &pb.SetLabelsReply{}, err
}
Expand All @@ -467,10 +475,6 @@ func (m *nfdMaster) SetTaints(c context.Context, r *pb.SetLabelsRequest) (*pb.Se
return &pb.SetLabelsReply{}, err
}

sort.Slice(nfrs, func(i, j int) bool {
return nfrs[i].Name < nfrs[j].Name
})

cli, err := m.apihelper.GetClient()
if err != nil {
return &pb.SetLabelsReply{}, err
Expand All @@ -483,27 +487,35 @@ func (m *nfdMaster) SetTaints(c context.Context, r *pb.SetLabelsRequest) (*pb.Se
}

if !m.args.NoPublish {

// check if the taint already exists
nodeTaints := node.Spec.Taints
// don't re-add the taint if already exists
annotation := node.GetAnnotations()

for _, nfr := range nfrs {
for _, rule := range nfr.Spec.Rules {
for _, crTaint := range rule.Taints {
if !taintutils.TaintExists(nodeTaints, &crTaint) {
taints = append(taints, crTaint)
}
}
crt = append(crt, rule.Taints...)
}
}
node.Spec.Taints = append(node.Spec.Taints, taints...)

// Update the node
for _, taint := range crt {
if !taintutils.TaintExists(node.Spec.Taints, &taint) {
nodeTaints = append(nodeTaints, taint)
}
}
if len(nodeTaints) > 0 {
node.Spec.Taints = append(node.Spec.Taints, nodeTaints...)
}

// Store taints in an annotation
for _, taint := range crt {
taintStrs = append(taintStrs, taint.ToString())
val := strings.Join(taintStrs, ",")
annotation[nfdv1alpha1.NodeTaintsAnnotation] = val
}

err = m.apihelper.UpdateNode(cli, node)
if err != nil {
return &pb.SetLabelsReply{}, fmt.Errorf("failed to taint the node %q: %v", node.Name, err)
}

}
return &pb.SetLabelsReply{}, nil
}
Expand Down Expand Up @@ -833,3 +845,70 @@ func (m *nfdMaster) instanceAnnotation(name string) string {
}
return m.args.Instance + "." + name
}

// deleteTaints deletes node taint(s) and taints annotation
func (m *nfdMaster) deleteTaints(c context.Context, r *pb.SetLabelsRequest) (*pb.SetLabelsReply, error) {
var annotationTaints, crt []api.Taint
cli, err := m.apihelper.GetClient()
if err != nil {
return &pb.SetLabelsReply{}, err
}

// fetch the node object
node, err := m.apihelper.GetNode(cli, r.NodeName)
if err != nil {
return &pb.SetLabelsReply{}, err
}

annotation := node.GetAnnotations()
if val, ok := annotation[nfdv1alpha1.NodeTaintsAnnotation]; ok {
sts := strings.Split(val, ",")
corev1Taints, _, err := taintutils.ParseTaints(sts)
if err != nil {
return &pb.SetLabelsReply{}, err
}
for _, ct := range corev1Taints {
annotationTaints = append(annotationTaints,
api.Taint{
Key: ct.Key,
Value: ct.Value,
Effect: ct.Effect})
}

nfrs, err := m.nfdController.ruleLister.List(labels.Everything())
if err != nil {
klog.Errorf("failed to list NodeFeatureRule resources: %v", err)
return &pb.SetLabelsReply{}, err
}

// delete the old taint that was set by the NFD
for _, nodeTaint := range node.Spec.Taints {
if len(nfrs) > 0 {
for _, nfr := range nfrs {
for _, rule := range nfr.Spec.Rules {
crt = append(crt, rule.Taints...)
}
}

if !m.args.NoTaint && !taintutils.TaintExists(crt, &nodeTaint) && taintutils.TaintExists(annotationTaints, &nodeTaint) {
taintsNew, _ := taintutils.DeleteTaint(node.Spec.Taints, &nodeTaint)
node.Spec.Taints = taintsNew
} else if taintutils.TaintExists(annotationTaints, &nodeTaint) {
taintsNew, _ := taintutils.DeleteTaint(node.Spec.Taints, &nodeTaint)
node.Spec.Taints = taintsNew
}
}
}

// delete the NFD set annotation
delete(annotation, nfdv1alpha1.NodeTaintsAnnotation)
node.SetAnnotations(annotation)

err = m.apihelper.UpdateNode(cli, node)
if err != nil {
return &pb.SetLabelsReply{}, fmt.Errorf("failed to untaint the node %q: %v", node.Name, err)
}
}

return &pb.SetLabelsReply{}, err
}

0 comments on commit a990e1b

Please sign in to comment.