feat: add nginx for kubelet

Signed-off-by: baoyinghai_yewu <[email protected]>

hack/k8s-in-k8s/generate_env.sh

@@ -1,51 +1,6 @@
 #!/usr/bin/env bash
 
-# This script will generate an g.env.sh file, like the following:
-# #!/usr/bin/env bash
-
-# # #####
-# # Generate by script generate_env.sh
-# # #####
-
-# SCRIPT_VERSION=0.0.1
-# # tmp dir of kosmos
-# PATH_FILE_TMP=/apps/conf/kosmos/tmp
-# ##################################################
-# # path for kubeadm config
-# PATH_KUBEADM_CONFIG=/etc/kubeadm
-# ##################################################
-# # path for kubernetes, from kubelet args --config
-# PATH_KUBERNETES=/etc/kubernetes
-# PATH_KUBERNETES_PKI=/etc/kubernetes/pki
-# # name for kubelet kubeconfig file
-# KUBELET_KUBE_CONFIG_NAME=kubelet.conf
-# ##################################################
-# # path for kubelet
-# PATH_KUBELET_LIB=/var/lib/kubelet
-# # path for kubelet
-# PATH_KUBELET_CONF=/var/lib/kubelet
-# # name for config file of kubelet
-# KUBELET_CONFIG_NAME=config.yaml
-
-# function GenerateKubeadmConfig() {
-#     echo "---
-# apiVersion: kubeadm.k8s.io/v1beta2
-# discovery:
-#     bootstrapToken:
-#         apiServerEndpoint: apiserver.cluster.local:6443
-#         token: $1
-#         unsafeSkipCAVerification: true
-# kind: JoinConfiguration
-# nodeRegistration:
-#     criSocket: /run/containerd/containerd.sock
-#     kubeletExtraArgs:
-#     container-runtime: remote
-#     container-runtime-endpoint: unix:///run/containerd/containerd.sock
-#     taints: null" > $2/kubeadm.cfg.current
-# }
-
-
-
+# This script will generate an g.env.sh file.
 
 SCRIPT_VERSION=0.0.1
 # save tmp file
@@ -80,10 +35,18 @@ function GetFileName() {
 
 function GetDirectory() {
     local fullpath="$1"
+    if [ -z "$fullpath" ]; then
+        echo "Error: No directory found."
+        exit 1
+    fi
     local directory=$(dirname "$fullpath")
     echo "$directory"
 }
 
+function GetMasterNodeIPs() {
+  kubectl get nodes -l node-role.kubernetes.io/master="" -o jsonpath='{range .items[*]}{.status.addresses[?(@.type=="InternalIP")].address}{" "}{end}'
+}
+
 # kubelet config name
 KUBELET_CONFIG_NAME=$(GetFileName "$(GetKubeletConfigFilePath)")
 # path for kubelet 
@@ -97,6 +60,15 @@ PATH_KUBERNETES_PKI=$(GetDirectory "$(GetKubernetesCaPath)")
 PATH_KUBERNETES=$(GetDirectory $PATH_KUBERNETES_PKI)
 HOST_CORE_DNS=$(GetKubeDnsClusterIP)
 
+DOCKER_IMAGE_NGINX="registry.paas/cmss/nginx:1.21.4"
+SERVERS=$(GetMasterNodeIPs)
+if [ -z "$SERVERS" ]; then
+    echo "Error: No master nodes found or failed to retrieve node IPs."
+    exit 1
+fi
+LOCAL_PORT="6443"
+LOCAL_IP="127.0.0.1"  # [::1]
+
 echo "#!/usr/bin/env bash
 
 # #####
@@ -128,6 +100,13 @@ USE_KUBEADM=false
 # Generate kubelet.conf TIMEOUT
 KUBELET_CONF_TIMEOUT=30
 
+# load balance
+DOCKER_IMAGE_NGINX=$DOCKER_IMAGE_NGINX
+SERVERS=($SERVERS)
+LOCAL_PORT="6443"
+LOCAL_IP="127.0.0.1"  # [::1]
+USE_NGINX=true
+
 function GenerateKubeadmConfig() {
     echo \"---
 apiVersion: kubeadm.k8s.io/v1beta2
@@ -147,6 +126,10 @@ nodeRegistration:
 }
 
 function GenerateStaticNginxProxy() {
+    config_path=/apps/conf/nginx
+    if [ "\$1" == \"true\" ]; then
+      config_path=\$PATH_FILE_TMP
+    fi
     echo \"apiVersion: v1
 kind: Pod
 metadata:
@@ -155,7 +138,7 @@ metadata:
   namespace: kube-system
 spec:
   containers:
-  - image: registry.paas/cmss/nginx:1.21.4
+  - image: \$DOCKER_IMAGE_NGINX
     imagePullPolicy: IfNotPresent
     name: nginx-proxy
     resources:
@@ -175,7 +158,7 @@ spec:
   priorityClassName: system-node-critical
   volumes:
   - hostPath:
-      path: /apps/conf/nginx
+      path: \$config_path
       type: 
     name: etc-nginx
 status: {}\" > $PATH_KUBERNETES/manifests/nginx-proxy.yaml

hack/k8s-in-k8s/kubelet_node_helper.sh

@@ -330,11 +330,103 @@ function version() {
     echo "$SCRIPT_VERSION"
 }
 
+
+function is_ipv6() {
+    if [[ "$1" =~ : ]]; then
+        return 0 
+    else
+        return 1
+    fi
+}
+
+function install_lb() {
+    if [ -z "$USE_NGINX" ]; then
+      export USE_KUBEADM=false
+    fi
+
+    if [ "$USE_NGINX" = false ]; then
+        exit 0
+    fi
+
+    echo "exec(1/6): get port of apiserver...."
+
+    PORT=$(grep 'server:' "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" | awk -F '[:/]' '{print $NF}')
+
+    if [ -z "$PORT" ]; then
+        echo "can not get port"
+        exit 1
+    else
+        echo "port is $PORT"
+    fi
+
+    if [ "$LOCAL_PORT" -eq "$PORT" ]; then
+        echo "Error: LOCAL_PORT ($LOCAL_PORT) cannot be the same as the backend port ($PORT)."
+        exit 0
+    fi
+
+    # Start generating nginx.conf
+    echo "exec(1/6): generate nginx.conf...."
+    cat <<EOL > "$PATH_FILE_TMP/nginx.conf"
+error_log stderr notice;
+worker_processes 1;
+events {
+  multi_accept on;
+  use epoll;
+  worker_connections 1024;
+}
+
+stream {
+        upstream kube_apiserver {
+            least_conn;
+EOL
+
+    # Loop through the array and append each server to the nginx.conf file
+    for SERVER in "${SERVERS[@]}"; do
+        if is_ipv6 "$SERVER"; then
+            echo "            server [$SERVER]:$PORT;" >> "$PATH_FILE_TMP/nginx.conf"
+        else
+            echo "            server $SERVER:$PORT;" >> "$PATH_FILE_TMP/nginx.conf"
+        fi
+    done
+
+    # Continue writing the rest of the nginx.conf
+    cat <<EOL >> "$PATH_FILE_TMP/nginx.conf"
+        }
+        server {
+            listen        [::]:$LOCAL_PORT;
+            listen        6443;
+            proxy_pass    kube_apiserver;
+            proxy_timeout 10m;
+            proxy_connect_timeout 10s;
+        }
+}
+EOL
+
+    echo "exec(1/6): create static pod"
+    GenerateStaticNginxProxy true
+
+
+    echo "exec(1/6): restart static pod"
+    mv "${PATH_KUBERNETES}/manifests/nginx-proxy.yaml" "${PATH_KUBERNETES}/nginx-proxy.yaml"
+    sleep 2
+    mv "${PATH_KUBERNETES}/nginx-proxy.yaml" "${PATH_KUBERNETES}/manifests/nginx-proxy.yaml"
+
+    echo "exec(1/6): update kubelet.conf"
+    cp "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}.bak"
+    sed -i "s|server: .*|server: https://${LOCAL_IP}:${LOCAL_PORT}|" "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}"
+
+    echo "exec(1/6): restart kubelet"
+    systemctl restart kubelet
+}
+
 # See how we were called.
 case "$1" in
     unjoin)
     unjoin
     ;;
+    install_lb)
+    install_lb
+    ;;
     join)
     join
     ;;

pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go

@@ -336,6 +336,26 @@ func NewWaitNodeReadyTask(isHost bool) Task {
 	}
 }
 
+func NewInstallNginxTask() Task {
+	return Task{
+		Name:  "remote install nginx",
+		Retry: true,
+		Run: func(ctx context.Context, to TaskOpt, _ interface{}) (interface{}, error) {
+			exectHelper := exector.NewExectorHelper(to.NodeInfo.Spec.NodeIP, "")
+
+			joinCmd := &exector.CMDExector{
+				Cmd: fmt.Sprintf("bash %s install_lb", env.GetExectorShellName()),
+			}
+			to.Loger().Infof("install nginx %s with cmd: %s", to.NodeInfo.Name, joinCmd.Cmd)
+			ret := exectHelper.DoExector(ctx.Done(), joinCmd)
+			if ret.Status != exector.SUCCESS {
+				return nil, fmt.Errorf("nstall nginx %s failed: %s", to.NodeInfo.Name, ret.String())
+			}
+			return nil, nil
+		},
+	}
+}
+
 // nolint:dupl
 func NewUpdateVirtualNodeLabelsTask() Task {
 	return Task{

pkg/kubenest/controller/virtualcluster.node.controller/workflow/workflow.go

@@ -90,6 +90,7 @@ func NewJoinWorkFlow() WorkflowData {
 		task.NewRemoteUpdateConfigYamlTask(),
 		task.NewRemoteNodeJoinTask(),
 		task.NewWaitNodeReadyTask(false),
+		task.NewInstallNginxTask(),
 		task.NewUpdateVirtualNodeLabelsTask(),
 		task.NewUpdateNodePoolItemStatusTask(v1alpha1.NodeInUse, false),
 	}

pkg/kubenest/tasks/manifests_components.go

@@ -106,7 +106,6 @@ func applyComponentsManifests(r workflow.RunData) error {
 		if nodeCount < constants.VipKeepAlivedReplicas {
 			keepalivedReplicas = int(nodeCount)
 		}
-
 		templatedMapping["KeepalivedReplicas"] = keepalivedReplicas
 	}
 
@@ -124,6 +123,10 @@ func applyComponentsManifests(r workflow.RunData) error {
 		},
 	})
 
+	for k, v := range data.PluginOptions() {
+		templatedMapping[k] = v
+	}
+
 	for _, component := range components {
 		klog.V(2).Infof("Deploy component %s", component.Name)
 		if v, ok := skipComponents[component.Name]; ok && v {