From 34b5d0d0fc849565c2a4d108e659de93017f2f06 Mon Sep 17 00:00:00 2001 From: calvix Date: Wed, 17 Apr 2024 09:52:06 +0200 Subject: [PATCH] Add taint toleration for unitialized nodes and prefer scheduling on control-plane nodes. (#198) --- CHANGELOG.md | 3 +++ Makefile.custom.mk | 4 ++-- .../deployments/capi-controller-manager.yaml | 14 ++++++++++++++ .../capi-kubeadm-bootstrap-controller-manager.yaml | 14 ++++++++++++++ ...i-kubeadm-control-plane-controller-manager.yaml | 14 ++++++++++++++ ...apps_v1_deployment_capi-controller-manager.yaml | 13 +++++++++++-- ..._capi-kubeadm-bootstrap-controller-manager.yaml | 13 +++++++++++-- ...i-kubeadm-control-plane-controller-manager.yaml | 13 +++++++++++-- 8 files changed, 80 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e2fe6c8..0b1fc449 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Change container image registry values name to use values from `config` repo. +- Add toleration for `node.cluster.x-k8s.io/uninitialized` taint. +- Remove toleration for old `node-role.kubernetes.io/master` taint. +- Add node affinity to prefer scheduling CAPI pods to control-plane nodes. ## [1.15.2] - 2024-01-22 diff --git a/Makefile.custom.mk b/Makefile.custom.mk index e307e828..27607099 100644 --- a/Makefile.custom.mk +++ b/Makefile.custom.mk @@ -5,9 +5,9 @@ generate: hack/fetch-manifest.sh # Kustomize templates. - rm helm/cluster-api/templates/*.yaml + rm -f helm/cluster-api/templates/*.yaml kubectl kustomize config/helm --output helm/cluster-api/templates - rm helm/cluster-api/templates/v1_configmap_watch-filter.yaml + rm -f helm/cluster-api/templates/v1_configmap_watch-filter.yaml # Move CRDs. hack/move-crds.sh diff --git a/config/helm/patches/deployments/capi-controller-manager.yaml b/config/helm/patches/deployments/capi-controller-manager.yaml index 0012dcd4..2989ca60 100644 --- a/config/helm/patches/deployments/capi-controller-manager.yaml +++ b/config/helm/patches/deployments/capi-controller-manager.yaml @@ -8,6 +8,14 @@ metadata: spec: template: spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + weight: 10 containers: - name: manager args: @@ -19,3 +27,9 @@ spec: - name: metrics protocol: TCP containerPort: 8080 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: "node.cluster.x-k8s.io/uninitialized" + operator: "Exists" diff --git a/config/helm/patches/deployments/capi-kubeadm-bootstrap-controller-manager.yaml b/config/helm/patches/deployments/capi-kubeadm-bootstrap-controller-manager.yaml index da2dcadc..54bac138 100644 --- a/config/helm/patches/deployments/capi-kubeadm-bootstrap-controller-manager.yaml +++ b/config/helm/patches/deployments/capi-kubeadm-bootstrap-controller-manager.yaml @@ -8,6 +8,14 @@ metadata: spec: template: spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + weight: 10 containers: - name: manager args: @@ -19,3 +27,9 @@ spec: - name: metrics protocol: TCP containerPort: 8080 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: "node.cluster.x-k8s.io/uninitialized" + operator: "Exists" diff --git a/config/helm/patches/deployments/capi-kubeadm-control-plane-controller-manager.yaml b/config/helm/patches/deployments/capi-kubeadm-control-plane-controller-manager.yaml index ce035b44..f13bd8fa 100644 --- a/config/helm/patches/deployments/capi-kubeadm-control-plane-controller-manager.yaml +++ b/config/helm/patches/deployments/capi-kubeadm-control-plane-controller-manager.yaml @@ -8,6 +8,14 @@ metadata: spec: template: spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + weight: 10 containers: - name: manager args: @@ -19,3 +27,9 @@ spec: - name: metrics protocol: TCP containerPort: 8080 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: "node.cluster.x-k8s.io/uninitialized" + operator: "Exists" diff --git a/helm/cluster-api/templates/apps_v1_deployment_capi-controller-manager.yaml b/helm/cluster-api/templates/apps_v1_deployment_capi-controller-manager.yaml index daeba14a..5c4643d8 100644 --- a/helm/cluster-api/templates/apps_v1_deployment_capi-controller-manager.yaml +++ b/helm/cluster-api/templates/apps_v1_deployment_capi-controller-manager.yaml @@ -33,6 +33,14 @@ spec: control-plane: controller-manager helm.sh/chart: cluster-api spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + weight: 10 containers: - args: - --metrics-bind-addr=0.0.0.0:8080 @@ -95,10 +103,11 @@ spec: serviceAccountName: capi-manager terminationGracePeriodSeconds: 10 tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - effect: NoSchedule key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node.cluster.x-k8s.io/uninitialized + operator: Exists volumes: - name: cert secret: diff --git a/helm/cluster-api/templates/apps_v1_deployment_capi-kubeadm-bootstrap-controller-manager.yaml b/helm/cluster-api/templates/apps_v1_deployment_capi-kubeadm-bootstrap-controller-manager.yaml index c116cd51..a9c1040f 100644 --- a/helm/cluster-api/templates/apps_v1_deployment_capi-kubeadm-bootstrap-controller-manager.yaml +++ b/helm/cluster-api/templates/apps_v1_deployment_capi-kubeadm-bootstrap-controller-manager.yaml @@ -33,6 +33,14 @@ spec: control-plane: controller-manager helm.sh/chart: cluster-api spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + weight: 10 containers: - args: - --metrics-bind-addr=0.0.0.0:8080 @@ -82,10 +90,11 @@ spec: serviceAccountName: capi-kubeadm-bootstrap-manager terminationGracePeriodSeconds: 10 tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - effect: NoSchedule key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node.cluster.x-k8s.io/uninitialized + operator: Exists volumes: - name: cert secret: diff --git a/helm/cluster-api/templates/apps_v1_deployment_capi-kubeadm-control-plane-controller-manager.yaml b/helm/cluster-api/templates/apps_v1_deployment_capi-kubeadm-control-plane-controller-manager.yaml index 4311300a..7884935b 100644 --- a/helm/cluster-api/templates/apps_v1_deployment_capi-kubeadm-control-plane-controller-manager.yaml +++ b/helm/cluster-api/templates/apps_v1_deployment_capi-kubeadm-control-plane-controller-manager.yaml @@ -33,6 +33,14 @@ spec: control-plane: controller-manager helm.sh/chart: cluster-api spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + weight: 10 containers: - args: - --metrics-bind-addr=0.0.0.0:8080 @@ -95,10 +103,11 @@ spec: serviceAccountName: capi-kubeadm-control-plane-manager terminationGracePeriodSeconds: 10 tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - effect: NoSchedule key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node.cluster.x-k8s.io/uninitialized + operator: Exists volumes: - name: cert secret: