Add taint toleration for unitialized nodes and prefer scheduling on c…

…ontrol-plane nodes. (#198)
giantswarm · Apr 17, 2024 · 34b5d0d · 34b5d0d
1 parent 158acc6
commit 34b5d0d
Show file tree

Hide file tree

Showing 8 changed files with 80 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Change container image registry values name to use values from `config` repo.
+- Add toleration for `node.cluster.x-k8s.io/uninitialized` taint.
+- Remove toleration for old `node-role.kubernetes.io/master` taint.
+- Add node affinity to prefer scheduling CAPI pods to control-plane nodes.
 
 ## [1.15.2] - 2024-01-22
 

diff --git a/Makefile.custom.mk b/Makefile.custom.mk
@@ -5,9 +5,9 @@ generate:
 	hack/fetch-manifest.sh
 
 	# Kustomize templates.
-	rm helm/cluster-api/templates/*.yaml
+	rm -f helm/cluster-api/templates/*.yaml
 	kubectl kustomize config/helm --output helm/cluster-api/templates
-	rm helm/cluster-api/templates/v1_configmap_watch-filter.yaml
+	rm -f helm/cluster-api/templates/v1_configmap_watch-filter.yaml
 
 	# Move CRDs.
 	hack/move-crds.sh

diff --git a/config/helm/patches/deployments/capi-controller-manager.yaml b/config/helm/patches/deployments/capi-controller-manager.yaml
@@ -8,6 +8,14 @@ metadata:
 spec:
   template:
     spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            weight: 10
       containers:
       - name: manager
         args:
@@ -19,3 +27,9 @@ spec:
         - name: metrics
           protocol: TCP
           containerPort: 8080
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: "node.cluster.x-k8s.io/uninitialized"
+        operator: "Exists"
diff --git a/config/helm/patches/deployments/capi-kubeadm-bootstrap-controller-manager.yaml b/config/helm/patches/deployments/capi-kubeadm-bootstrap-controller-manager.yaml
@@ -8,6 +8,14 @@ metadata:
 spec:
   template:
     spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            weight: 10
       containers:
       - name: manager
         args:
@@ -19,3 +27,9 @@ spec:
         - name: metrics
           protocol: TCP
           containerPort: 8080
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: "node.cluster.x-k8s.io/uninitialized"
+        operator: "Exists"
diff --git a/config/helm/patches/deployments/capi-kubeadm-control-plane-controller-manager.yaml b/config/helm/patches/deployments/capi-kubeadm-control-plane-controller-manager.yaml
@@ -8,6 +8,14 @@ metadata:
 spec:
   template:
     spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            weight: 10
       containers:
       - name: manager
         args:
@@ -19,3 +27,9 @@ spec:
         - name: metrics
           protocol: TCP
           containerPort: 8080
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: "node.cluster.x-k8s.io/uninitialized"
+        operator: "Exists"
diff --git a/helm/cluster-api/templates/apps_v1_deployment_capi-controller-manager.yaml b/helm/cluster-api/templates/apps_v1_deployment_capi-controller-manager.yaml
@@ -33,6 +33,14 @@ spec:
         control-plane: controller-manager
         helm.sh/chart: cluster-api
     spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            weight: 10
       containers:
       - args:
         - --metrics-bind-addr=0.0.0.0:8080
@@ -95,10 +103,11 @@ spec:
       serviceAccountName: capi-manager
       terminationGracePeriodSeconds: 10
       tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
       - effect: NoSchedule
         key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: node.cluster.x-k8s.io/uninitialized
+        operator: Exists
       volumes:
       - name: cert
         secret:

diff --git a/helm/cluster-api/templates/apps_v1_deployment_capi-kubeadm-bootstrap-controller-manager.yaml b/helm/cluster-api/templates/apps_v1_deployment_capi-kubeadm-bootstrap-controller-manager.yaml
@@ -33,6 +33,14 @@ spec:
         control-plane: controller-manager
         helm.sh/chart: cluster-api
     spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            weight: 10
       containers:
       - args:
         - --metrics-bind-addr=0.0.0.0:8080
@@ -82,10 +90,11 @@ spec:
       serviceAccountName: capi-kubeadm-bootstrap-manager
       terminationGracePeriodSeconds: 10
       tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
       - effect: NoSchedule
         key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: node.cluster.x-k8s.io/uninitialized
+        operator: Exists
       volumes:
       - name: cert
         secret:

diff --git a/...uster-api/templates/apps_v1_deployment_capi-kubeadm-control-plane-controller-manager.yaml b/...uster-api/templates/apps_v1_deployment_capi-kubeadm-control-plane-controller-manager.yaml
@@ -33,6 +33,14 @@ spec:
         control-plane: controller-manager
         helm.sh/chart: cluster-api
     spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            weight: 10
       containers:
       - args:
         - --metrics-bind-addr=0.0.0.0:8080
@@ -95,10 +103,11 @@ spec:
       serviceAccountName: capi-kubeadm-control-plane-manager
       terminationGracePeriodSeconds: 10
       tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
       - effect: NoSchedule
         key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: node.cluster.x-k8s.io/uninitialized
+        operator: Exists
       volumes:
       - name: cert
         secret: