Add NodeLocalDNS configuration to the kubenest workflow.

Signed-off-by: luoyuanze <[email protected]>

deploy/crds/kosmos.io_kubenestconfigurations.yaml

@@ -57,6 +57,9 @@ spec:
                 type: string
               etcdUnitSize:
                 type: string
+              externalPort:
+                format: int32
+                type: integer
               forceDestroy:
                 description: todo Group according to the parameters of apiserver,
                   etcd, coredns, etc. ForceDestroy indicates whether to force destroy
@@ -77,6 +80,9 @@ spec:
                       type: string
                     type: array
                 type: object
+              useNodeLocalDNS:
+                default: false
+                type: boolean
               useTenantDNS:
                 default: false
                 type: boolean

deploy/crds/kosmos.io_virtualclusters.yaml

@@ -75,8 +75,8 @@ spec:
                   etcdUnitSize:
                     type: string
                   externalPort:
-                    type: integer
                     format: int32
+                    type: integer
                   forceDestroy:
                     description: todo Group according to the parameters of apiserver,
                       etcd, coredns, etc. ForceDestroy indicates whether to force
@@ -98,6 +98,9 @@ spec:
                           type: string
                         type: array
                     type: object
+                  useNodeLocalDNS:
+                    default: false
+                    type: boolean
                   useTenantDNS:
                     default: false
                     type: boolean

hack/k8s-in-k8s/kubelet_node_helper.sh

@@ -8,6 +8,7 @@ LOG_NAME=${2:-kubelet}
 JOIN_HOST=$2
 JOIN_TOKEN=$3
 JOIN_CA_HASH=$4
+NODE_LOCAL_DNS_ADDRESS=$2
 
 function unjoin() {
     # before unjoin, you need delete node by kubectl
@@ -248,7 +249,14 @@ function join() {
         exit 1
     fi
     echo "exec(4/8): set core dns address...."
-    sed -e "s|__DNS_ADDRESS__|$DNS_ADDRESS|g" -e "w ${PATH_KUBELET_CONF}/${KUBELET_CONFIG_NAME}" "$PATH_FILE_TMP"/"$KUBELET_CONFIG_NAME"
+    if [ -z "$NODE_LOCAL_DNS_ADDRESS" ]; then
+        sed -e "/__DNS_ADDRESS__/i - ${NODE_LOCAL_DNS_ADDRESS}" \
+            -e "s|__DNS_ADDRESS__|${DNS_ADDRESS}|g" \
+            "$PATH_FILE_TMP/$KUBELET_CONFIG_NAME" \
+            > "${PATH_KUBELET_CONF}/${KUBELET_CONFIG_NAME}"
+    else
+        sed -e "s|__DNS_ADDRESS__|$DNS_ADDRESS|g" -e "w ${PATH_KUBELET_CONF}/${KUBELET_CONFIG_NAME}" "$PATH_FILE_TMP"/"$KUBELET_CONFIG_NAME"
+    fi
     if [ $? -ne 0 ]; then
         exit 1
     fi

pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go

@@ -91,6 +91,9 @@ type KubeInKubeConfig struct {
 	UseTenantDNS bool `yaml:"useTenantDNS" json:"useTenantDNS,omitempty"`
 	// +optional
 	ExternalPort int32 `json:"externalPort,omitempty"`
+	// +kubebuilder:default=false
+	// +optional
+	UseNodeLocalDNS bool `yaml:"useNodeLocalDNS" json:"useNodeLocalDNS,omitempty"`
 }
 
 // TenantEntrypoint contains the configuration for the tenant entrypoint.

pkg/kubenest/constants/constant.go

@@ -137,6 +137,11 @@ const (
 
 	//in virtual cluster
 	APIServerExternalService = "api-server-external-service"
+
+	//nodelocaldns
+	NodeLocalDNSComponentName = "virtual-node-local-dns"
+	NodeLocalDNSIp            = "169.254.20.1"
+	NodeLocalDNSClusterDomain = "cluster.local"
 )
 
 type Action string

pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go

@@ -254,9 +254,12 @@ func NewRemoteNodeJoinTask() Task {
 		Retry: true,
 		Run: func(ctx context.Context, to TaskOpt, _ interface{}) (interface{}, error) {
 			exectHelper := exector.NewExectorHelper(to.NodeInfo.Spec.NodeIP, "")
-
+			baseCmd := fmt.Sprintf("bash %s join %s", env.GetExectorShellName(), to.KubeDNSAddress)
+			if to.VirtualCluster.Spec.KubeInKubeConfig != nil && to.VirtualCluster.Spec.KubeInKubeConfig.UseNodeLocalDNS {
+				baseCmd = fmt.Sprintf("bash %s join %s %s", env.GetExectorShellName(), to.KubeDNSAddress, constants.NodeLocalDNSIp)
+			}
 			joinCmd := &exector.CMDExector{
-				Cmd: fmt.Sprintf("bash %s join %s", env.GetExectorShellName(), to.KubeDNSAddress),
+				Cmd: baseCmd,
 			}
 			to.Loger().Infof("join node %s with cmd: %s", to.NodeInfo.Name, joinCmd.Cmd)
 			ret := exectHelper.DoExector(ctx.Done(), joinCmd)

pkg/kubenest/tasks/manifests_components.go

@@ -2,10 +2,10 @@ package tasks
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 
-	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -15,13 +15,15 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 
 	"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
 	"github.com/kosmos.io/kosmos/pkg/kubenest/util"
 	apiclient "github.com/kosmos.io/kosmos/pkg/kubenest/util/api-client"
 	"github.com/kosmos.io/kosmos/pkg/kubenest/workflow"
+	"github.com/pkg/errors"
 )
 
 type ComponentConfig struct {
@@ -91,10 +93,21 @@ func applyComponentsManifests(r workflow.RunData) error {
 		return err
 	}
 
+	kubeDNSIP, err := getKubeDNSClusterIP(config)
+	if err != nil {
+		return errors.Wrap(err, "Failed to get kube-dns ClusterIP")
+	}
+	klog.Infof("kube-dns CLUSTER-IP: %s", kubeDNSIP)
+
 	templatedMapping := make(map[string]interface{}, 2)
 	templatedMapping["KUBE_PROXY_KUBECONFIG"] = string(secret.Data[constants.KubeConfig])
 	imageRepository, _ := util.GetImageMessage()
 	templatedMapping["ImageRepository"] = imageRepository
+
+	templatedMapping["PillarLocalDNS"] = constants.NodeLocalDNSIp
+	templatedMapping["PillarDNSDomain"] = constants.NodeLocalDNSClusterDomain
+	templatedMapping["PillarClusterDNS"] = kubeDNSIP
+
 	for k, v := range data.PluginOptions() {
 		templatedMapping[k] = v
 	}
@@ -111,6 +124,7 @@ func applyComponentsManifests(r workflow.RunData) error {
 	}
 
 	UseTenantDNS := data.VirtualCluster().Spec.KubeInKubeConfig != nil && data.VirtualCluster().Spec.KubeInKubeConfig.UseTenantDNS
+	UseNodeLocalDNS := data.VirtualCluster().Spec.KubeInKubeConfig != nil && data.VirtualCluster().Spec.KubeInKubeConfig.UseNodeLocalDNS
 
 	skipComponents := getSkipComponentsForVirtualCluster([]*SkipComponentCondition{
 		{
@@ -122,6 +136,11 @@ func applyComponentsManifests(r workflow.RunData) error {
 			Condition:     !keepalivedEnable,
 			ComponentName: constants.VipKeepalivedComponentName,
 		},
+		{
+			// skip nodelocaldns component if nodelocaldns is not enabled
+			Condition:     !UseNodeLocalDNS,
+			ComponentName: constants.NodeLocalDNSComponentName,
+		},
 	})
 
 	for _, component := range components {
@@ -215,3 +234,16 @@ func applyTemplatedManifests(component string, dynamicClient dynamic.Interface,
 	}
 	return nil
 }
+func getKubeDNSClusterIP(config *rest.Config) (string, error) {
+	client, err := clientset.NewForConfig(config)
+	if err != nil {
+		return "", fmt.Errorf("failed to create kubernetes client: %v", err)
+	}
+
+	svc, err := client.CoreV1().Services("kube-system").Get(context.TODO(), "kube-dns", metav1.GetOptions{})
+	if err != nil {
+		return "", fmt.Errorf("failed to get kube-dns service: %v", err)
+	}
+
+	return svc.Spec.ClusterIP, nil
+}