deploy virtualcluster control plane on master nodes

Signed-off-by: zhouhaoA1 <[email protected]>
kosmos-io · Apr 25, 2024 · 9422f0f · 9422f0f
1 parent 55dd351
commit 9422f0f
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 10 deletions.
diff --git a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_deployment.go
@@ -22,6 +22,10 @@ spec:
     spec:
       automountServiceAccountToken: false
       hostNetwork: true
+      tolerations:
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: "Exists"
+        effect: "NoSchedule"
       containers:
       - name: kube-apiserver
         image:  {{ .ImageRepository }}/kube-apiserver:{{ .Version }}
@@ -81,6 +85,12 @@ spec:
           successThreshold: 1
           timeoutSeconds: 15
         affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+                - matchExpressions:
+                    - key: node-role.kubernetes.io/control-plane
+                      operator: Exists
           podAntiAffinity:
             requiredDuringSchedulingIgnoredDuringExecution:
             - labelSelector:

diff --git a/pkg/kubenest/manifest/controlplane/etcd/mainfests_deployment.go b/pkg/kubenest/manifest/controlplane/etcd/mainfests_deployment.go
@@ -22,10 +22,27 @@ spec:
     metadata:
       labels:
         virtualCluster-app: etcd
-    tolerations:
-    - operator: Exists
     spec:
       automountServiceAccountToken: false
+      tolerations:
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: "Exists"
+        effect: "NoSchedule"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+              matchExpressions:
+              - key: virtualCluster-app
+                operator: In
+                values: ["etcd"]
+              topologyKey: kubernetes.io/hostname
       containers:
       - name: etcd
         image:  {{ .ImageRepository }}/etcd:{{ .Version }}

diff --git a/pkg/kubenest/manifest/controlplane/kube-controller/manifests_deployment.go b/pkg/kubenest/manifest/controlplane/kube-controller/manifests_deployment.go
@@ -23,7 +23,17 @@ spec:
     spec:
       automountServiceAccountToken: false
       priorityClassName: system-node-critical
+      tolerations:
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: "Exists"
+        effect: "NoSchedule"
       affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             - labelSelector:

diff --git a/pkg/kubenest/manifest/controlplane/scheduler/manifest_deployment.go b/pkg/kubenest/manifest/controlplane/scheduler/manifest_deployment.go
@@ -21,9 +21,25 @@ spec:
         virtualCluster-app:  scheduler
     spec:
       automountServiceAccountToken: false
-      tolerations:    
-        - key: node-role.kubernetes.io/master
-          operator: Exists
+      tolerations:
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: "Exists"
+        effect: "NoSchedule"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+              matchExpressions:
+              - key: virtualCluster-app
+                operator: In
+                values: ["scheduler"]
+              topologyKey: kubernetes.io/hostname
       containers:     
       - name: scheduler
         image: {{ .ImageRepository }}/scheduler:{{ .Version }}

diff --git a/pkg/kubenest/tasks/manifests_components.go b/pkg/kubenest/tasks/manifests_components.go
@@ -78,13 +78,13 @@ func applyComponentsManifests(r workflow.RunData) error {
 		return err
 	}
 
+	templatedMapping := make(map[string]interface{}, 2)
+	templatedMapping["KUBE_PROXY_KUBECONFIG"] = string(secret.Data[constants.KubeConfig])
+	imageRepository, _ := util.GetImageMessage()
+	templatedMapping["ImageRepository"] = imageRepository
+
 	for _, component := range components {
 		klog.V(2).Infof("Deploy component %s", component.Name)
-
-		templatedMapping := make(map[string]interface{}, 2)
-		if component.Name == constants.VirtualClusterKubeProxyComponent {
-			templatedMapping["KUBE_PROXY_KUBECONFIG"] = string(secret.Data[constants.KubeConfig])
-		}
 		err = applyTemplatedManifests(dynamicClient, component.Path, templatedMapping)
 		if err != nil {
 			return err