Support new control plane label and taint

Signed-off-by: Stefan Büringer [email protected]
kubernetes-sigs · Jan 17, 2022 · c2a843c · c2a843c
1 parent 5d5dc7a
commit c2a843c
Show file tree

Hide file tree

Showing 10 changed files with 129 additions and 17 deletions.
diff --git a/bootstrap/kubeadm/config/manager/manager.yaml b/bootstrap/kubeadm/config/manager/manager.yaml
@@ -41,3 +41,5 @@ spec:
       tolerations:
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -42,3 +42,5 @@ spec:
       tolerations:
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
diff --git a/controlplane/kubeadm/config/manager/manager.yaml b/controlplane/kubeadm/config/manager/manager.yaml
@@ -41,3 +41,5 @@ spec:
       tolerations:
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
diff --git a/controlplane/kubeadm/internal/controllers/controller_test.go b/controlplane/kubeadm/internal/controllers/controller_test.go
@@ -1516,7 +1516,7 @@ func createMachineNodePair(name string, cluster *clusterv1.Cluster, kcp *control
 	node := &corev1.Node{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   name,
-			Labels: map[string]string{"node-role.kubernetes.io/master": ""},
+			Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""},
 		},
 	}
 

diff --git a/controlplane/kubeadm/internal/workload_cluster.go b/controlplane/kubeadm/internal/workload_cluster.go
@@ -51,13 +51,14 @@ import (
 )
 
 const (
-	kubeProxyKey              = "kube-proxy"
-	kubeadmConfigKey          = "kubeadm-config"
-	kubeletConfigKey          = "kubelet"
-	cgroupDriverKey           = "cgroupDriver"
-	labelNodeRoleControlPlane = "node-role.kubernetes.io/master"
-	clusterStatusKey          = "ClusterStatus"
-	clusterConfigurationKey   = "ClusterConfiguration"
+	kubeProxyKey                 = "kube-proxy"
+	kubeadmConfigKey             = "kubeadm-config"
+	kubeletConfigKey             = "kubelet"
+	cgroupDriverKey              = "cgroupDriver"
+	labelNodeRoleOldControlPlane = "node-role.kubernetes.io/master" // Deprecated: https://github.com/kubernetes/kubeadm/issues/2200
+	labelNodeRoleControlPlane    = "node-role.kubernetes.io/control-plane"
+	clusterStatusKey             = "ClusterStatus"
+	clusterConfigurationKey      = "ClusterConfiguration"
 )
 
 var (
@@ -121,14 +122,32 @@ type Workload struct {
 var _ WorkloadCluster = &Workload{}
 
 func (w *Workload) getControlPlaneNodes(ctx context.Context) (*corev1.NodeList, error) {
-	nodes := &corev1.NodeList{}
-	labels := map[string]string{
-		labelNodeRoleControlPlane: "",
-	}
-	if err := w.Client.List(ctx, nodes, ctrlclient.MatchingLabels(labels)); err != nil {
-		return nil, err
+	controlPlaneNodes := &corev1.NodeList{}
+	for {
+		allNodes := corev1.NodeList{}
+		if err := w.Client.List(ctx, &allNodes, ctrlclient.Continue(allNodes.Continue)); err != nil {
+			return nil, errors.Wrapf(err, "failed to list nodes")
+		}
+
+		for i := range allNodes.Items {
+			node := allNodes.Items[i]
+			if isControlPlaneNode(node) {
+				controlPlaneNodes.Items = append(controlPlaneNodes.Items, node)
+			}
+		}
+
+		if allNodes.Continue == "" {
+			break
+		}
 	}
-	return nodes, nil
+
+	return controlPlaneNodes, nil
+}
+
+func isControlPlaneNode(node corev1.Node) bool {
+	_, hasLabelNodeRoleControlPlane := node.Labels[labelNodeRoleControlPlane]
+	_, hasLabelNodeRoleOldControlPlane := node.Labels[labelNodeRoleOldControlPlane]
+	return hasLabelNodeRoleControlPlane || hasLabelNodeRoleOldControlPlane
 }
 
 func (w *Workload) getConfigMap(ctx context.Context, configMap ctrlclient.ObjectKey) (*corev1.ConfigMap, error) {

diff --git a/controlplane/kubeadm/internal/workload_cluster_conditions_test.go b/controlplane/kubeadm/internal/workload_cluster_conditions_test.go
@@ -980,6 +980,9 @@ func fakeNode(name string, options ...fakeNodeOption) *corev1.Node {
 	p := &corev1.Node{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
+			Labels: map[string]string{
+				labelNodeRoleControlPlane: "",
+			},
 		},
 	}
 	for _, opt := range options {

diff --git a/controlplane/kubeadm/internal/workload_cluster_test.go b/controlplane/kubeadm/internal/workload_cluster_test.go
@@ -38,6 +38,66 @@ import (
 	"sigs.k8s.io/cluster-api/util/yaml"
 )
 
+func TestIsControlPlaneNode(t *testing.T) {
+	tests := []struct {
+		name               string
+		node               corev1.Node
+		isControlPlaneNode bool
+	}{
+		{
+			name: "control plane node with old label",
+			node: corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						labelNodeRoleOldControlPlane: "",
+					},
+				},
+			},
+			isControlPlaneNode: true,
+		},
+		{
+			name: "control plane node with new label",
+			node: corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						labelNodeRoleControlPlane: "",
+					},
+				},
+			},
+			isControlPlaneNode: true,
+		},
+		{
+			name: "control plane node with both label",
+			node: corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						labelNodeRoleControlPlane:    "",
+						labelNodeRoleOldControlPlane: "",
+					},
+				},
+			},
+			isControlPlaneNode: true,
+		},
+		{
+			name: "worker node",
+			node: corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{},
+				},
+			},
+			isControlPlaneNode: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			g.Expect(isControlPlaneNode(tt.node)).To(Equal(tt.isControlPlaneNode))
+		})
+	}
+}
+
 func TestUpdateKubeProxyImageInfo(t *testing.T) {
 	tests := []struct {
 		name        string

diff --git a/docs/book/src/user/troubleshooting.md b/docs/book/src/user/troubleshooting.md
@@ -35,7 +35,10 @@ kubectl label nodes <name> node-role.kubernetes.io/worker=""
 For convenience, here is an example one-liner to do this post installation
 
 ```
+# Kubernetes 1.19 (kubeadm 1.19 sets only the node-role.kubernetes.io/master label)
 kubectl get nodes --no-headers -l '!node-role.kubernetes.io/master' -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}' | xargs -I{} kubectl label node {} node-role.kubernetes.io/worker=''
+# Kubernetes >= 1.20 (kubeadm >= 1.20 sets the node-role.kubernetes.io/control-plane label) 
+kubectl get nodes --no-headers -l '!node-role.kubernetes.io/control-plane' -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}' | xargs -I{} kubectl label node {} node-role.kubernetes.io/worker=''
 ```                  
 
 ## Cluster API with Docker

diff --git a/test/framework/deployment_helpers.go b/test/framework/deployment_helpers.go
@@ -34,6 +34,7 @@ import (
 	"k8s.io/api/policy/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	utilversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -42,6 +43,11 @@ import (
 	"sigs.k8s.io/cluster-api/test/framework/internal/log"
 )
 
+const (
+	nodeRoleOldControlPlane = "node-role.kubernetes.io/master" // Deprecated: https://github.com/kubernetes/kubeadm/issues/2200
+	nodeRoleControlPlane    = "node-role.kubernetes.io/control-plane"
+)
+
 // WaitForDeploymentsAvailableInput is the input for WaitForDeploymentsAvailable.
 type WaitForDeploymentsAvailableInput struct {
 	Getter     Getter
@@ -306,10 +312,23 @@ func DeployUnevictablePod(ctx context.Context, input DeployUnevictablePodInput)
 		},
 	}
 	if input.ControlPlane != nil {
-		workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{"node-role.kubernetes.io/master": ""}
+		serverVersion, err := workloadClient.ServerVersion()
+		Expect(err).ToNot(HaveOccurred())
+
+		// Use the control-plane label for Kubernetes version >= v1.20.0.
+		if utilversion.MustParseGeneric(serverVersion.String()).AtLeast(utilversion.MustParseGeneric("v1.20.0")) {
+			workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{nodeRoleControlPlane: ""}
+		} else {
+			workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{nodeRoleOldControlPlane: ""}
+		}
+
 		workloadDeployment.Spec.Template.Spec.Tolerations = []corev1.Toleration{
 			{
-				Key:    "node-role.kubernetes.io/master",
+				Key:    nodeRoleOldControlPlane,
+				Effect: "NoSchedule",
+			},
+			{
+				Key:    nodeRoleControlPlane,
 				Effect: "NoSchedule",
 			},
 		}

diff --git a/test/infrastructure/docker/config/manager/manager.yaml b/test/infrastructure/docker/config/manager/manager.yaml
@@ -44,6 +44,8 @@ spec:
       tolerations:
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
       volumes:
         - name: dockersock
           hostPath: