From 9422f0f4116cf1c24e5528c82dd6fb5c769e878f Mon Sep 17 00:00:00 2001 From: zhouhaoA1 Date: Thu, 25 Apr 2024 13:33:03 +0800 Subject: [PATCH] deploy virtualcluster control plane on master nodes Signed-off-by: zhouhaoA1 --- .../apiserver/mainfests_deployment.go | 10 +++++++++ .../controlplane/etcd/mainfests_deployment.go | 21 ++++++++++++++++-- .../kube-controller/manifests_deployment.go | 10 +++++++++ .../scheduler/manifest_deployment.go | 22 ++++++++++++++++--- pkg/kubenest/tasks/manifests_components.go | 10 ++++----- 5 files changed, 63 insertions(+), 10 deletions(-) diff --git a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go index 0f2c59ed4..272c92679 100644 --- a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go +++ b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go @@ -22,6 +22,10 @@ spec: spec: automountServiceAccountToken: false hostNetwork: true + tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" containers: - name: kube-apiserver image: {{ .ImageRepository }}/kube-apiserver:{{ .Version }} @@ -81,6 +85,12 @@ spec: successThreshold: 1 timeoutSeconds: 15 affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: diff --git a/pkg/kubenest/manifest/controlplane/etcd/mainfests_deployment.go b/pkg/kubenest/manifest/controlplane/etcd/mainfests_deployment.go index 17b9b3378..3569b4af3 100644 --- a/pkg/kubenest/manifest/controlplane/etcd/mainfests_deployment.go +++ b/pkg/kubenest/manifest/controlplane/etcd/mainfests_deployment.go @@ -22,10 +22,27 @@ spec: metadata: labels: virtualCluster-app: etcd - tolerations: - - operator: Exists spec: automountServiceAccountToken: false + tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: virtualCluster-app + operator: In + values: ["etcd"] + topologyKey: kubernetes.io/hostname containers: - name: etcd image: {{ .ImageRepository }}/etcd:{{ .Version }} diff --git a/pkg/kubenest/manifest/controlplane/kube-controller/manifests_deployment.go b/pkg/kubenest/manifest/controlplane/kube-controller/manifests_deployment.go index e88d8eda9..2e8f11295 100644 --- a/pkg/kubenest/manifest/controlplane/kube-controller/manifests_deployment.go +++ b/pkg/kubenest/manifest/controlplane/kube-controller/manifests_deployment.go @@ -23,7 +23,17 @@ spec: spec: automountServiceAccountToken: false priorityClassName: system-node-critical + tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: diff --git a/pkg/kubenest/manifest/controlplane/scheduler/manifest_deployment.go b/pkg/kubenest/manifest/controlplane/scheduler/manifest_deployment.go index 0b9246f89..14a55ff61 100644 --- a/pkg/kubenest/manifest/controlplane/scheduler/manifest_deployment.go +++ b/pkg/kubenest/manifest/controlplane/scheduler/manifest_deployment.go @@ -21,9 +21,25 @@ spec: virtualCluster-app: scheduler spec: automountServiceAccountToken: false - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists + tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: virtualCluster-app + operator: In + values: ["scheduler"] + topologyKey: kubernetes.io/hostname containers: - name: scheduler image: {{ .ImageRepository }}/scheduler:{{ .Version }} diff --git a/pkg/kubenest/tasks/manifests_components.go b/pkg/kubenest/tasks/manifests_components.go index 73dd158cf..60471c94c 100644 --- a/pkg/kubenest/tasks/manifests_components.go +++ b/pkg/kubenest/tasks/manifests_components.go @@ -78,13 +78,13 @@ func applyComponentsManifests(r workflow.RunData) error { return err } + templatedMapping := make(map[string]interface{}, 2) + templatedMapping["KUBE_PROXY_KUBECONFIG"] = string(secret.Data[constants.KubeConfig]) + imageRepository, _ := util.GetImageMessage() + templatedMapping["ImageRepository"] = imageRepository + for _, component := range components { klog.V(2).Infof("Deploy component %s", component.Name) - - templatedMapping := make(map[string]interface{}, 2) - if component.Name == constants.VirtualClusterKubeProxyComponent { - templatedMapping["KUBE_PROXY_KUBECONFIG"] = string(secret.Data[constants.KubeConfig]) - } err = applyTemplatedManifests(dynamicClient, component.Path, templatedMapping) if err != nil { return err