From 54fa55bca905f64ba925ec75f3b96cb9413780fe Mon Sep 17 00:00:00 2001 From: Yashvardhan Kukreja Date: Fri, 13 Nov 2020 16:20:54 +0530 Subject: [PATCH 1/2] Added: support for adding labels from the config itself --- pkg/apis/config/v1alpha4/types.go | 3 + .../config/v1alpha4/zz_generated.deepcopy.go | 7 + .../postcreationconfiguration.go | 125 ++++++++++++++++++ pkg/cluster/internal/create/create.go | 2 + pkg/internal/apis/config/convert_v1alpha4.go | 1 + pkg/internal/apis/config/types.go | 3 + .../apis/config/zz_generated.deepcopy.go | 7 + 7 files changed, 148 insertions(+) create mode 100644 pkg/cluster/internal/create/actions/postcreationconfiguration/postcreationconfiguration.go diff --git a/pkg/apis/config/v1alpha4/types.go b/pkg/apis/config/v1alpha4/types.go index 9722c3d995..99c4181fa3 100644 --- a/pkg/apis/config/v1alpha4/types.go +++ b/pkg/apis/config/v1alpha4/types.go @@ -104,6 +104,9 @@ type Node struct { // If unset a default image will be used, see defaults.Image Image string `yaml:"image,omitempty"` + // Labels are the labels with which the respective node will be labeled + Labels map[string]string `yaml:"labels,omitempty"` + /* Advanced fields */ // TODO: cri-like types should be inline instead diff --git a/pkg/apis/config/v1alpha4/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha4/zz_generated.deepcopy.go index 6fa61d4e34..8aab0fa122 100644 --- a/pkg/apis/config/v1alpha4/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha4/zz_generated.deepcopy.go @@ -114,6 +114,13 @@ func (in *Networking) DeepCopy() *Networking { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Node) DeepCopyInto(out *Node) { *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.ExtraMounts != nil { in, out := &in.ExtraMounts, &out.ExtraMounts *out = make([]Mount, len(*in)) diff --git a/pkg/cluster/internal/create/actions/postcreationconfiguration/postcreationconfiguration.go b/pkg/cluster/internal/create/actions/postcreationconfiguration/postcreationconfiguration.go new file mode 100644 index 0000000000..2c588532d5 --- /dev/null +++ b/pkg/cluster/internal/create/actions/postcreationconfiguration/postcreationconfiguration.go @@ -0,0 +1,125 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package postcreationconfiguration implements the action for configuring cluster after cluster creation +// like labelling, tainting, etc the nodes. +package postcreationconfiguration + +import ( + "fmt" + "strings" + + "sigs.k8s.io/kind/pkg/cluster/internal/create/actions" + "sigs.k8s.io/kind/pkg/cluster/nodes" + "sigs.k8s.io/kind/pkg/cluster/nodeutils" + "sigs.k8s.io/kind/pkg/internal/apis/config" +) + +// Action implements and action for configuring the cluster after cluster creation +type Action struct{} + +// NewAction returns a new Action for configuring the cluster after cluster creation +func NewAction() actions.Action { + return &Action{} +} + +// Execute runs the action +func (a *Action) Execute(ctx *actions.ActionContext) error { + + // Label the control-plane and worker nodes as per the config file + if err := labelNodes(ctx); err != nil { + return err + } + + return nil +} + +// labelNodes labels the control-plane and worker nodes +// as per the labels provided in the config file +func labelNodes(ctx *actions.ActionContext) error { + allNodes, err := ctx.Nodes() + if err != nil { + return err + } + + // Get one of the cluster nodes for the execution of labelling action/command. + // The labelling action/command for a control-plane node will be executed by that respective node only. (label will be self-applied for the control-plane node) + // The labelling action/command for a worker node will be executed by the below 'firstControlPlane' node. + controlPlanes, err := nodeutils.ControlPlaneNodes(allNodes) + if err != nil { + return err + } + firstControlPlaneNode := controlPlanes[0] + + // Populate the list of control-plane node labels and the list of worker node labels respectively. + // controlPlaneLabels is an array of maps (labels, read from config) associated with all the control-plane nodes. + // workerLabels is an array of maps (labels, read from config) associated with all the worker nodes. + controlPlaneLabels := []map[string]string{} + workerLabels := []map[string]string{} + for _, node := range ctx.Config.Nodes { + if node.Role == config.ControlPlaneRole { + controlPlaneLabels = append(controlPlaneLabels, node.Labels) + } else if node.Role == config.WorkerRole { + workerLabels = append(workerLabels, node.Labels) + } else { + continue + } + } + + // Label the control-plane and worker nodes accordingly. + controlPlaneLabelsCounter := 0 + workerLabelsCounter := 0 + for _, node := range allNodes { + nodeName := node.String() + nodeRole, err := node.Role() + if err != nil { + return err + } + + var currentNodeLabels map[string]string + var cmdExecutorNode nodes.Node + if nodeRole == string(config.ControlPlaneRole) { + cmdExecutorNode = node + currentNodeLabels = controlPlaneLabels[controlPlaneLabelsCounter] + controlPlaneLabelsCounter++ + } else if nodeRole == string(config.WorkerRole) { + cmdExecutorNode = firstControlPlaneNode + currentNodeLabels = workerLabels[workerLabelsCounter] + workerLabelsCounter++ + } else { + continue + } + if len(currentNodeLabels) == 0 { + continue + } + + // Construct the `kubectl label` command as per the current labels for the current node in the loop. + labelCommandArgsStr := fmt.Sprintf("--kubeconfig=/etc/kubernetes/admin.conf label node %s ", nodeName) + for key, value := range currentNodeLabels { + labelCommandArgsStr += fmt.Sprintf("%s=%s ", key, value) + } + labelCommandArgs := strings.Split(strings.TrimSpace(labelCommandArgsStr), " ") + + // Execute the constructed `kubectl load` command via the rightful node + // If the labels are being applied to a control-plane node, then, they will be self-applied by that node + // If the labels are being applied to a worker node, then, they will be applied by a control-plane node ('firstControlPlaneNode') + if err := cmdExecutorNode.Command("kubectl", labelCommandArgs...).Run(); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cluster/internal/create/create.go b/pkg/cluster/internal/create/create.go index 9d96b7ebec..4afdbd6ea5 100644 --- a/pkg/cluster/internal/create/create.go +++ b/pkg/cluster/internal/create/create.go @@ -39,6 +39,7 @@ import ( "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/kubeadminit" "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/kubeadmjoin" "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/loadbalancer" + "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/postcreationconfiguration" "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/waitforready" "sigs.k8s.io/kind/pkg/cluster/internal/kubeconfig" ) @@ -136,6 +137,7 @@ func Cluster(logger log.Logger, p providers.Provider, opts *ClusterOptions) erro installstorage.NewAction(), // install StorageClass kubeadmjoin.NewAction(), // run kubeadm join waitforready.NewAction(opts.WaitForReady), // wait for cluster readiness + postcreationconfiguration.NewAction(), // perform post-cluster-creation configurations like custom labelling, tainting, etc. ) } diff --git a/pkg/internal/apis/config/convert_v1alpha4.go b/pkg/internal/apis/config/convert_v1alpha4.go index 96e7aa8d4c..f37fe6c16e 100644 --- a/pkg/internal/apis/config/convert_v1alpha4.go +++ b/pkg/internal/apis/config/convert_v1alpha4.go @@ -51,6 +51,7 @@ func convertv1alpha4Node(in *v1alpha4.Node, out *Node) { out.Role = NodeRole(in.Role) out.Image = in.Image + out.Labels = in.Labels out.KubeadmConfigPatches = in.KubeadmConfigPatches out.ExtraMounts = make([]Mount, len(in.ExtraMounts)) out.ExtraPortMappings = make([]PortMapping, len(in.ExtraPortMappings)) diff --git a/pkg/internal/apis/config/types.go b/pkg/internal/apis/config/types.go index 25b364044b..3bc360fef4 100644 --- a/pkg/internal/apis/config/types.go +++ b/pkg/internal/apis/config/types.go @@ -85,6 +85,9 @@ type Node struct { // If unset a default image will be used, see defaults.Image Image string + // Labels are the labels with which the respective node will be labeled + Labels map[string]string + /* Advanced fields */ // ExtraMounts describes additional mount points for the node container diff --git a/pkg/internal/apis/config/zz_generated.deepcopy.go b/pkg/internal/apis/config/zz_generated.deepcopy.go index 74e0c9d5f5..3595f0ae36 100644 --- a/pkg/internal/apis/config/zz_generated.deepcopy.go +++ b/pkg/internal/apis/config/zz_generated.deepcopy.go @@ -113,6 +113,13 @@ func (in *Networking) DeepCopy() *Networking { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Node) DeepCopyInto(out *Node) { *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.ExtraMounts != nil { in, out := &in.ExtraMounts, &out.ExtraMounts *out = make([]Mount, len(*in)) From 4b5c1bb34422afeeaa027601b08a6ba8516b8492 Mon Sep 17 00:00:00 2001 From: Yashvardhan Kukreja Date: Wed, 18 Nov 2020 12:34:55 +0530 Subject: [PATCH 2/2] Added: support for populating labels via kubeadm's kubeletExtraArgs config now --- .../internal/create/actions/config/config.go | 34 ++++- .../postcreationconfiguration.go | 125 ------------------ pkg/cluster/internal/create/create.go | 2 - pkg/cluster/internal/kubeadm/config.go | 5 + 4 files changed, 37 insertions(+), 129 deletions(-) delete mode 100644 pkg/cluster/internal/create/actions/postcreationconfiguration/postcreationconfiguration.go diff --git a/pkg/cluster/internal/create/actions/config/config.go b/pkg/cluster/internal/create/actions/config/config.go index 98eba4c80c..cfc8bf42dc 100644 --- a/pkg/cluster/internal/create/actions/config/config.go +++ b/pkg/cluster/internal/create/actions/config/config.go @@ -90,15 +90,42 @@ func (a *Action) Execute(ctx *actions.ActionContext) error { } } + // Populate the list of control-plane node labels and the list of worker node labels respectively. + // controlPlaneLabels is an array of maps (labels, read from config) associated with all the control-plane nodes. + // workerLabels is an array of maps (labels, read from config) associated with all the worker nodes. + controlPlaneLabels := []map[string]string{} + workerLabels := []map[string]string{} + for _, node := range ctx.Config.Nodes { + if node.Role == config.ControlPlaneRole { + controlPlaneLabels = append(controlPlaneLabels, node.Labels) + } else if node.Role == config.WorkerRole { + workerLabels = append(workerLabels, node.Labels) + } else { + continue + } + } + + // hashMapLabelsToCommaSeparatedLabels converts labels in hashmap form to labels in a comma-separated string form like "key1=value1,key2=value2" + hashMapLabelsToCommaSeparatedLabels := func(labels map[string]string) string { + output := "" + for key, value := range labels { + output += fmt.Sprintf("%s=%s,", key, value) + } + return strings.TrimSuffix(output, ",") // remove the last character (comma) in the output string + } + // create the kubeadm join configuration for control plane nodes controlPlanes, err := nodeutils.ControlPlaneNodes(allNodes) if err != nil { return err } - for _, node := range controlPlanes { + for i, node := range controlPlanes { node := node // capture loop variable configData := configData // copy config data + if len(controlPlaneLabels[i]) > 0 { + configData.NodeLabels = hashMapLabelsToCommaSeparatedLabels(controlPlaneLabels[i]) // updating the config with the respective labels to be written over the current control-plane node in consideration + } fns = append(fns, kubeadmConfigPlusPatches(node, configData)) } @@ -109,10 +136,13 @@ func (a *Action) Execute(ctx *actions.ActionContext) error { } if len(workers) > 0 { // create the workers concurrently - for _, node := range workers { + for i, node := range workers { node := node // capture loop variable configData := configData // copy config data configData.ControlPlane = false + if len(workerLabels[i]) > 0 { + configData.NodeLabels = hashMapLabelsToCommaSeparatedLabels(workerLabels[i]) // updating the config with the respective labels to be written over the current worker node in consideration + } fns = append(fns, kubeadmConfigPlusPatches(node, configData)) } } diff --git a/pkg/cluster/internal/create/actions/postcreationconfiguration/postcreationconfiguration.go b/pkg/cluster/internal/create/actions/postcreationconfiguration/postcreationconfiguration.go deleted file mode 100644 index 2c588532d5..0000000000 --- a/pkg/cluster/internal/create/actions/postcreationconfiguration/postcreationconfiguration.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package postcreationconfiguration implements the action for configuring cluster after cluster creation -// like labelling, tainting, etc the nodes. -package postcreationconfiguration - -import ( - "fmt" - "strings" - - "sigs.k8s.io/kind/pkg/cluster/internal/create/actions" - "sigs.k8s.io/kind/pkg/cluster/nodes" - "sigs.k8s.io/kind/pkg/cluster/nodeutils" - "sigs.k8s.io/kind/pkg/internal/apis/config" -) - -// Action implements and action for configuring the cluster after cluster creation -type Action struct{} - -// NewAction returns a new Action for configuring the cluster after cluster creation -func NewAction() actions.Action { - return &Action{} -} - -// Execute runs the action -func (a *Action) Execute(ctx *actions.ActionContext) error { - - // Label the control-plane and worker nodes as per the config file - if err := labelNodes(ctx); err != nil { - return err - } - - return nil -} - -// labelNodes labels the control-plane and worker nodes -// as per the labels provided in the config file -func labelNodes(ctx *actions.ActionContext) error { - allNodes, err := ctx.Nodes() - if err != nil { - return err - } - - // Get one of the cluster nodes for the execution of labelling action/command. - // The labelling action/command for a control-plane node will be executed by that respective node only. (label will be self-applied for the control-plane node) - // The labelling action/command for a worker node will be executed by the below 'firstControlPlane' node. - controlPlanes, err := nodeutils.ControlPlaneNodes(allNodes) - if err != nil { - return err - } - firstControlPlaneNode := controlPlanes[0] - - // Populate the list of control-plane node labels and the list of worker node labels respectively. - // controlPlaneLabels is an array of maps (labels, read from config) associated with all the control-plane nodes. - // workerLabels is an array of maps (labels, read from config) associated with all the worker nodes. - controlPlaneLabels := []map[string]string{} - workerLabels := []map[string]string{} - for _, node := range ctx.Config.Nodes { - if node.Role == config.ControlPlaneRole { - controlPlaneLabels = append(controlPlaneLabels, node.Labels) - } else if node.Role == config.WorkerRole { - workerLabels = append(workerLabels, node.Labels) - } else { - continue - } - } - - // Label the control-plane and worker nodes accordingly. - controlPlaneLabelsCounter := 0 - workerLabelsCounter := 0 - for _, node := range allNodes { - nodeName := node.String() - nodeRole, err := node.Role() - if err != nil { - return err - } - - var currentNodeLabels map[string]string - var cmdExecutorNode nodes.Node - if nodeRole == string(config.ControlPlaneRole) { - cmdExecutorNode = node - currentNodeLabels = controlPlaneLabels[controlPlaneLabelsCounter] - controlPlaneLabelsCounter++ - } else if nodeRole == string(config.WorkerRole) { - cmdExecutorNode = firstControlPlaneNode - currentNodeLabels = workerLabels[workerLabelsCounter] - workerLabelsCounter++ - } else { - continue - } - if len(currentNodeLabels) == 0 { - continue - } - - // Construct the `kubectl label` command as per the current labels for the current node in the loop. - labelCommandArgsStr := fmt.Sprintf("--kubeconfig=/etc/kubernetes/admin.conf label node %s ", nodeName) - for key, value := range currentNodeLabels { - labelCommandArgsStr += fmt.Sprintf("%s=%s ", key, value) - } - labelCommandArgs := strings.Split(strings.TrimSpace(labelCommandArgsStr), " ") - - // Execute the constructed `kubectl load` command via the rightful node - // If the labels are being applied to a control-plane node, then, they will be self-applied by that node - // If the labels are being applied to a worker node, then, they will be applied by a control-plane node ('firstControlPlaneNode') - if err := cmdExecutorNode.Command("kubectl", labelCommandArgs...).Run(); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/cluster/internal/create/create.go b/pkg/cluster/internal/create/create.go index 4afdbd6ea5..9d96b7ebec 100644 --- a/pkg/cluster/internal/create/create.go +++ b/pkg/cluster/internal/create/create.go @@ -39,7 +39,6 @@ import ( "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/kubeadminit" "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/kubeadmjoin" "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/loadbalancer" - "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/postcreationconfiguration" "sigs.k8s.io/kind/pkg/cluster/internal/create/actions/waitforready" "sigs.k8s.io/kind/pkg/cluster/internal/kubeconfig" ) @@ -137,7 +136,6 @@ func Cluster(logger log.Logger, p providers.Provider, opts *ClusterOptions) erro installstorage.NewAction(), // install StorageClass kubeadmjoin.NewAction(), // run kubeadm join waitforready.NewAction(opts.WaitForReady), // wait for cluster readiness - postcreationconfiguration.NewAction(), // perform post-cluster-creation configurations like custom labelling, tainting, etc. ) } diff --git a/pkg/cluster/internal/kubeadm/config.go b/pkg/cluster/internal/kubeadm/config.go index 4eeb8a4bfd..7adbe42cc6 100644 --- a/pkg/cluster/internal/kubeadm/config.go +++ b/pkg/cluster/internal/kubeadm/config.go @@ -70,6 +70,9 @@ type ConfigData struct { // IPv4 values take precedence over IPv6 by default, if true set IPv6 default values IPv6 bool + // Labels are the labels, in the format "key1=val1,key2=val2", with which the respective node will be labeled + NodeLabels string + // DerivedConfigData is populated by Derive() // These auto-generated fields are available to Config templates, // but not meant to be set by hand @@ -315,6 +318,7 @@ nodeRegistration: fail-swap-on: "false" node-ip: "{{ .NodeAddress }}" provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}" + node-labels: "{{ .NodeLabels }}" --- # no-op entry that exists solely so it can be patched apiVersion: kubeadm.k8s.io/v1beta2 @@ -333,6 +337,7 @@ nodeRegistration: fail-swap-on: "false" node-ip: "{{ .NodeAddress }}" provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}" + node-labels: "{{ .NodeLabels }}" discovery: bootstrapToken: apiServerEndpoint: "{{ .ControlPlaneEndpoint }}"