Merge pull request #698 from OrangeBao/main

feat: support install corends to virtual cluster
kosmos-io · Sep 2, 2024 · 68d7df8 · 68d7df8
2 parents 9c9eaff + 14993e5
commit 68d7df8
Show file tree

Hide file tree

Showing 16 changed files with 605 additions and 59 deletions.
diff --git a/.gitignore b/.gitignore
@@ -36,4 +36,6 @@ cmd/kubenest/node-agent/cert.pem
 cmd/kubenest/node-agent/key.pem
 cmd/kubenest/node-agent/agent.env
 hack/k8s-in-k8s/nodes.txt
-develop
+develop
+
+cmd/kubenest/node-agent/app/client/app.log
diff --git a/deploy/crds/kosmos.io_kubenestconfigurations.yaml b/deploy/crds/kosmos.io_kubenestconfigurations.yaml
@@ -77,6 +77,9 @@ spec:
                       type: string
                     type: array
                 type: object
+              useTenantDns:
+                default: false
+                type: boolean
             type: object
           kubeNestType:
             type: string

diff --git a/deploy/crds/kosmos.io_virtualclusters.yaml b/deploy/crds/kosmos.io_virtualclusters.yaml
@@ -95,6 +95,9 @@ spec:
                           type: string
                         type: array
                     type: object
+                  useTenantDns:
+                    default: false
+                    type: boolean
                 type: object
               kubeconfig:
                 description: Kubeconfig is the kubeconfig of the virtual kubernetes's

diff --git a/deploy/virtual-cluster-components-manifest-cm.yaml b/deploy/virtual-cluster-components-manifest-cm.yaml
@@ -5,6 +5,7 @@ data:
       {"name": "kube-proxy", "path": "/kosmos/manifest/kube-proxy/*.yaml"},
       {"name": "calico", "path": "/kosmos/manifest/calico/*.yaml"},
       {"name": "keepalived", "path": "/kosmos/manifest/keepalived/*.yaml"},
+      {"name": "core-dns-tenant", "path": "/kosmos/manifest/core-dns/tenant/*.yaml"},
     ]
   host-core-dns-components: |
     [

diff --git a/hack/k8s-in-k8s/g.env.sh b/hack/k8s-in-k8s/g.env.sh
@@ -25,7 +25,7 @@ PATH_KUBELET_CONF=.
 KUBELET_CONFIG_NAME=
 HOST_CORE_DNS=10.96.0.10
 # kubeadm switch
-USE_KUBEADM=true
+USE_KUBEADM=false
 # Generate kubelet.conf TIMEOUT
 KUBELET_CONF_TIMEOUT=30
 

diff --git a/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go b/pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go
@@ -85,6 +85,10 @@ type KubeInKubeConfig struct {
 	// +kubebuilder:default=hostNetwork
 	// +optional
 	ApiServerServiceType ApiServerServiceType `yaml:"apiServerServiceType" json:"apiServerServiceType,omitempty"`
+
+	// +kubebuilder:default=false
+	// +optional
+	UseTenantDns bool `yaml:"useTenantDns" json:"useTenantDns,omitempty"`
 }
 
 // TenantEntrypoint contains the configuration for the tenant entrypoint.

diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go
diff --git a/pkg/kubenest/constants/constant.go b/pkg/kubenest/constants/constant.go
@@ -124,9 +124,10 @@ const (
 	// core-dns
 	KubeDNSSVCName = "kube-dns"
 	// nolint
-	HostCoreDnsComponents    = "host-core-dns-components"
-	VirtualCoreDnsComponents = "virtual-core-dns-components"
-	PrometheusRuleManifest   = "prometheus-rules"
+	HostCoreDnsComponents      = "host-core-dns-components"
+	VirtualCoreDnsComponents   = "virtual-core-dns-components"
+	PrometheusRuleManifest     = "prometheus-rules"
+	TenantCoreDnsComponentName = "core-dns-tenant"
 
 	StateLabelKey = "kosmos-io/state"
 

diff --git a/pkg/kubenest/controller/endpoints.sync.controller/coredns_sync_controller.go b/pkg/kubenest/controller/endpoints.sync.controller/coredns_sync_controller.go
@@ -151,6 +151,10 @@ func (e *CoreDNSController) Reconcile(ctx context.Context, request reconcile.Req
 		return reconcile.Result{RequeueAfter: utils.DefaultRequeueTime}, nil
 	}
 
+	if targetVirtualCluster.Spec.KubeInKubeConfig != nil && targetVirtualCluster.Spec.KubeInKubeConfig.UseTenantDns {
+		return reconcile.Result{}, nil
+	}
+
 	// Get the corresponding svc
 	var kubesvc v1.Service
 	if err := e.Get(ctx, request.NamespacedName, &kubesvc); err != nil {

diff --git a/pkg/kubenest/controller/virtualcluster_init_controller_test.go b/pkg/kubenest/controller/virtualcluster_init_controller_test.go
@@ -33,28 +33,29 @@ func TestCreateApiAnpServer(t *testing.T) {
 	if len(apiAnpAgentSvc.Spec.Ports) != 4 {
 		t.Fatalf("apiAnpAgentSvc.Spec.Ports len != 4")
 	}
-	if apiAnpAgentSvc.Spec.Ports[0].Name != "agentport" {
-		t.Fatalf("apiAnpAgentSvc.Spec.Ports[0].Name != agentport")
+	ports := make([]int32, 5)
+	for _, port := range apiAnpAgentSvc.Spec.Ports {
+		v, ok := nameMap[port.Name]
+		if ok {
+			ports[v] = port.Port
+		} else {
+			t.Fatalf("can not get node port for %s", port.Name)
+		}
 	}
-	if apiAnpAgentSvc.Spec.Ports[0].Port != 8081 {
+
+	if ports[1] != 8081 {
 		t.Fatalf("apiAnpAgentSvc.Spec.Ports[0].Port != 8081")
 	}
-	if apiAnpAgentSvc.Spec.Ports[1].Name != "serverport" {
-		t.Fatalf("apiAnpAgentSvc.Spec.Ports[1].Name != serverport")
-	}
-	if apiAnpAgentSvc.Spec.Ports[1].Port != 8082 {
+
+	if ports[2] != 8082 {
 		t.Fatalf("apiAnpAgentSvc.Spec.Ports[1].Port != 8082")
 	}
-	if apiAnpAgentSvc.Spec.Ports[2].Name != "healthport" {
-		t.Fatalf("apiAnpAgentSvc.Spec.Ports[2].Name != healthport")
-	}
-	if apiAnpAgentSvc.Spec.Ports[2].Port != 8083 {
+
+	if ports[3] != 8083 {
 		t.Fatalf("apiAnpAgentSvc.Spec.Ports[2].Port != 8083")
 	}
-	if apiAnpAgentSvc.Spec.Ports[3].Name != "adminport" {
-		t.Fatalf("apiAnpAgentSvc.Spec.Ports[3].Name != adminport")
-	}
-	if apiAnpAgentSvc.Spec.Ports[3].Port != 8084 {
-		t.Fatalf("apiAnpAgentSvc.Spec.Ports[3].Port != 8084")
+
+	if ports[4] != 8084 {
+		t.Fatalf("apiAnpAgentSvc.Spec.Ports[2].Port != 8084")
 	}
 }
diff --git a/pkg/kubenest/manifest/controlplane/apiserver/mainfests_service_test.go b/pkg/kubenest/manifest/controlplane/apiserver/mainfests_service_test.go
@@ -18,12 +18,14 @@ func ParseServerTemplate(apiServerServiceSubnet string) (*corev1.Service, error)
 		ServiceName, Namespace, ServiceType string
 		ServicePort                         int32
 		IPFamilies                          []corev1.IPFamily
+		UseApiServerNodePort                bool
 	}{
-		ServiceName: fmt.Sprintf("%s-%s", "test", "apiserver"),
-		Namespace:   "test-namespace",
-		ServiceType: constants.ApiServerServiceType,
-		ServicePort: 40010,
-		IPFamilies:  ipFamilies,
+		ServiceName:          fmt.Sprintf("%s-%s", "test", "apiserver"),
+		Namespace:            "test-namespace",
+		ServiceType:          constants.ApiServerServiceType,
+		ServicePort:          40010,
+		IPFamilies:           ipFamilies,
+		UseApiServerNodePort: false,
 	})
 
 	if err != nil {

diff --git a/pkg/kubenest/tasks/coredns.go b/pkg/kubenest/tasks/coredns.go
@@ -28,6 +28,7 @@ func NewCoreDNSTask() workflow.Task {
 	return workflow.Task{
 		Name:        "coreDns",
 		Run:         runCoreDns,
+		Skip:        skipCoreDns,
 		RunSubTasks: true,
 		Tasks: []workflow.Task{
 			{
@@ -46,6 +47,19 @@ func NewCoreDNSTask() workflow.Task {
 	}
 }
 
+func skipCoreDns(d workflow.RunData) (bool, error) {
+	data, ok := d.(InitData)
+	if !ok {
+		return false, errors.New("coreDns task invoked with an invalid data struct")
+	}
+
+	vc := data.VirtualCluster()
+	if vc.Spec.KubeInKubeConfig != nil && vc.Spec.KubeInKubeConfig.UseTenantDns {
+		return true, nil
+	}
+	return false, nil
+}
+
 func runCoreDns(r workflow.RunData) error {
 	data, ok := r.(InitData)
 	if !ok {

diff --git a/pkg/kubenest/tasks/manifests_components.go b/pkg/kubenest/tasks/manifests_components.go
@@ -29,6 +29,11 @@ type ComponentConfig struct {
 	Path string `json:"path" yaml:"path"`
 }
 
+type SkipComponentCondition struct {
+	Condition     bool
+	ComponentName string
+}
+
 func NewComponentsFromManifestsTask() workflow.Task {
 	return workflow.Task{
 		Name:        "manifests-components",
@@ -53,6 +58,14 @@ func runComponentsFromManifests(r workflow.RunData) error {
 	return nil
 }
 
+func getSkipComponentsForVirtualCluster(condition []*SkipComponentCondition) map[string]bool {
+	skipComponents := map[string]bool{}
+	for _, c := range condition {
+		skipComponents[c.ComponentName] = c.Condition
+	}
+	return skipComponents
+}
+
 func applyComponentsManifests(r workflow.RunData) error {
 	data, ok := r.(InitData)
 	if !ok {
@@ -96,10 +109,24 @@ func applyComponentsManifests(r workflow.RunData) error {
 		templatedMapping["KeepalivedReplicas"] = keepalivedReplicas
 	}
 
+	UseTenantDns := data.VirtualCluster().Spec.KubeInKubeConfig != nil && data.VirtualCluster().Spec.KubeInKubeConfig.UseTenantDns
+
+	skipComponents := getSkipComponentsForVirtualCluster([]*SkipComponentCondition{
+		{
+			// skip coredns component if tenant dns is enabled
+			Condition:     !UseTenantDns,
+			ComponentName: constants.TenantCoreDnsComponentName,
+		}, {
+			// skip keepalived component if vip is not enabled
+			Condition:     !keepalivedEnable,
+			ComponentName: constants.VipKeepalivedComponentName,
+		},
+	})
+
 	for _, component := range components {
 		klog.V(2).Infof("Deploy component %s", component.Name)
-		// skip keepalived component if vip is not enabled
-		if !keepalivedEnable && component.Name == constants.VipKeepalivedComponentName {
+		if v, ok := skipComponents[component.Name]; ok && v {
+			klog.V(2).Infof("Deploy component %s skipped", component.Name)
 			continue
 		}
 		err = applyTemplatedManifests(component.Name, dynamicClient, component.Path, templatedMapping)