From d6b7fa5e2eb691cde9dfca425e7742abe00629a6 Mon Sep 17 00:00:00 2001
From: baoyinghai_yewu <baoyinghai_yewu@cmss.chinamobile.com>
Date: Fri, 21 Jun 2024 15:53:02 +0800
Subject: [PATCH] fix: Fix exceptions that occur when adding nodes back to the
 host cluster

Signed-off-by: baoyinghai_yewu <baoyinghai_yewu@cmss.chinamobile.com>
---
 hack/k8s-in-k8s/generate_env.sh        | 42 ++++++++++++
 hack/k8s-in-k8s/kubelet_node_helper.sh | 93 +++++++++++++++++++++++---
 hack/k8s-in-k8s/port_check.sh          | 31 +++++++++
 3 files changed, 157 insertions(+), 9 deletions(-)
 create mode 100644 hack/k8s-in-k8s/port_check.sh

diff --git a/hack/k8s-in-k8s/generate_env.sh b/hack/k8s-in-k8s/generate_env.sh
index c6d4bafe5..11089b0cf 100644
--- a/hack/k8s-in-k8s/generate_env.sh
+++ b/hack/k8s-in-k8s/generate_env.sh
@@ -68,6 +68,10 @@ function GetKubernetesCaPath() {
     kubectl get cm kubelet-config -nkube-system -oyaml  | awk '/clientCAFile:/{print $2}'
 }
 
+function GetKubeDnsClusterIP() {
+    kubectl get svc -nkube-system kube-dns  -o jsonpath='{.spec.clusterIP}'
+}
+
 function GetFileName() {
     local fullpath="$1"
     local filename=$(basename "$fullpath")
@@ -91,6 +95,7 @@ KUBELET_KUBE_CONFIG_NAME=$(GetFileName "$(GetKubeletKubeConfigFilePath)")
 PATH_KUBERNETES_PKI=$(GetDirectory "$(GetKubernetesCaPath)")
 # length=${#PATH_KUBERNETES_PKI}
 PATH_KUBERNETES=$(GetDirectory $PATH_KUBERNETES_PKI)
+HOST_CORE_DNS=$(GetKubeDnsClusterIP)
 
 echo "#!/usr/bin/env bash
 
@@ -117,10 +122,12 @@ PATH_KUBELET_LIB=$PATH_KUBELET_LIB
 PATH_KUBELET_CONF=$PATH_KUBELET_CONF
 # name for config file of kubelet
 KUBELET_CONFIG_NAME=$KUBELET_CONFIG_NAME
+HOST_CORE_DNS=$HOST_CORE_DNS
 
 function GenerateKubeadmConfig() {
     echo \"---
 apiVersion: kubeadm.k8s.io/v1beta2
+caCertPath: $PATH_KUBERNETES_PKI/ca.crt
 discovery:
     bootstrapToken:
         apiServerEndpoint: apiserver.cluster.local:6443
@@ -135,6 +142,41 @@ nodeRegistration:
     taints: null\" > \$2/kubeadm.cfg.current
 }
 
+function GenerateStaticNginxProxy() {
+    echo \"apiVersion: v1
+kind: Pod
+metadata:
+  creationTimestamp: null
+  name: nginx-proxy
+  namespace: kube-system
+spec:
+  containers:
+  - image: registry.paas/cmss/nginx:1.21.4
+    imagePullPolicy: IfNotPresent
+    name: nginx-proxy
+    resources:
+      limits:
+        cpu: 300m
+        memory: 512M
+      requests:
+        cpu: 25m
+        memory: 32M
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/nginx
+      name: etc-nginx
+      readOnly: true
+  hostNetwork: true
+  priorityClassName: system-node-critical
+  volumes:
+  - hostPath:
+      path: /apps/conf/nginx
+      type: 
+    name: etc-nginx
+status: {}\" > $PATH_KUBERNETES/manifests/nginx-proxy.yaml
+}
+
 " > g.env.sh
 
 
diff --git a/hack/k8s-in-k8s/kubelet_node_helper.sh b/hack/k8s-in-k8s/kubelet_node_helper.sh
index e70b204cd..65180e719 100755
--- a/hack/k8s-in-k8s/kubelet_node_helper.sh
+++ b/hack/k8s-in-k8s/kubelet_node_helper.sh
@@ -11,54 +11,129 @@ JOIN_CA_HASH=$4
 
 function unjoin() {
     # before unjoin, you need delete node by kubectl
-    echo "exec(1/2): kubeadm reset...."
+    echo "exec(1/5): kubeadm reset...."
     echo "y" | kubeadm reset
     if [ $? -ne 0 ]; then
         exit 1
     fi
 
-    echo "exec(2/3): restart cotnainerd...."
+    echo "exec(2/5): restart cotnainerd...."
     systemctl restart containerd
     if [ $? -ne 0 ]; then
         exit 1
     fi
 
-    echo "exec(3/3): delete cni...."
+    echo "exec(3/5): delete cni...."
     if [ -d "/etc/cni/net.d" ]; then   
         mv /etc/cni/net.d '/etc/cni/net.d.kosmos.back'`date +%Y_%m_%d_%H_%M_%S`
         if [ $? -ne 0 ]; then
             exit 1
         fi
     fi
+
+    echo "exec(4/5): delete ca.crt"
+    if [ -f "$PATH_KUBERNETES_PKI/ca.crt" ]; then
+        echo "y" | rm "$PATH_KUBERNETES_PKI/ca.crt"
+        if [ $? -ne 0 ]; then
+            exit 1
+        fi
+    fi
+
+    echo "exec(5/5): delete kubelet.conf"
+    if [ -f "$PATH_KUBELET_CONF/${KUBELET_CONFIG_NAME}" ]; then
+        echo "y" | rm "$PATH_KUBELET_CONF/${KUBELET_CONFIG_NAME}"
+        if [ $? -ne 0 ]; then
+            exit 1
+        fi
+    fi
+}
+
+function beforeRevert() {
+    if [ -f "/apps/conf/nginx/nginx.conf" ]; then 
+        # modify  hosts
+        config_file="/apps/conf/nginx/nginx.conf"
+
+        server_address=$(grep -Po 'server\s+\K[^:]+(?=:6443)' "$config_file" | awk 'NR==1')
+        hostname=$(echo $JOIN_HOST | awk -F ":" '{print $1}')
+        host_record="$server_address $hostname"
+        if grep -qFx "$host_record" /etc/hosts; then
+            echo "Record $host_record already exists in /etc/hosts."
+        else
+            sed -i "1i $host_record" /etc/hosts
+            echo "Record $host_record inserted into /etc/hosts."
+        fi
+    fi
+}
+
+function afterRevert() {
+    if [ -f "/apps/conf/nginx/nginx.conf" ]; then 
+        # modify  hosts
+        config_file="/apps/conf/nginx/nginx.conf"
+
+        server_address=$(grep -Po 'server\s+\K[^:]+(?=:6443)' "$config_file" | awk 'NR==1')
+        hostname=$(echo $JOIN_HOST | awk -F ":" '{print $1}')
+        host_record="$server_address $hostname"
+        if grep -qFx "$host_record" /etc/hosts; then
+            sudo sed -i "/^$host_record/d" /etc/hosts
+        fi
+
+        local_record="127.0.0.1 $hostname"
+        if grep -qFx "$local_record" /etc/hosts; then
+            echo "Record $local_record already exists in /etc/hosts."
+        else
+            sed -i "1i $local_record" /etc/hosts
+            echo "Record $local_record inserted into /etc/hosts."
+        fi
+
+        GenerateStaticNginxProxy
+    fi
 }
 
 function revert() {
-    echo "exec(1/4): update kubeadm.cfg..."
+    echo "exec(1/5): update kubeadm.cfg..."
     if [ ! -f "$PATH_KUBEADM_CONFIG/kubeadm.cfg" ]; then
         GenerateKubeadmConfig  $JOIN_TOKEN $PATH_FILE_TMP
     else
       sed -e "s|token: .*$|token: $JOIN_TOKEN|g" -e "w $PATH_FILE_TMP/kubeadm.cfg.current" "$PATH_KUBEADM_CONFIG/kubeadm.cfg"
     fi
-
     
     # add taints
-    echo "exec(2/4): update kubeadm.cfg tanits..."
+    echo "exec(2/5): update kubeadm.cfg tanits..."
     sed -i "/kubeletExtraArgs/a \    register-with-taints: node.kosmos.io/unschedulable:NoSchedule"  "$PATH_FILE_TMP/kubeadm.cfg.current" 
     if [ $? -ne 0 ]; then
         exit 1
     fi
-    
-    echo "exec(3/4): execute join cmd...."
+
+    echo "exec(3/5): update kubelet-config..."
+    sed -e "s|__DNS_ADDRESS__|$HOST_CORE_DNS|g" -e "w ${PATH_KUBELET_CONF}/${KUBELET_CONFIG_NAME}" "$PATH_FILE_TMP"/"$KUBELET_CONFIG_NAME"
+    if [ $? -ne 0 ]; then
+        exit 1
+    fi
+
+    beforeRevert
+    if [ $? -ne 0 ]; then
+        exit 1
+    fi
+
+
+    echo "exec(4/5): execute join cmd...."
     kubeadm join --config "$PATH_FILE_TMP/kubeadm.cfg.current"
     if [ $? -ne 0 ]; then
         exit 1
     fi
 
-    echo "exec(4/4): restart cotnainerd...."
+    echo "exec(5/5): restart cotnainerd...."
     systemctl restart containerd
     if [ $? -ne 0 ]; then
         exit 1
     fi
+
+    afterRevert
+    if [ $? -ne 0 ]; then
+        exit 1
+    fi
+
+
 }
 
 # before join, you need upload ca.crt and kubeconfig to tmp dir!!!
diff --git a/hack/k8s-in-k8s/port_check.sh b/hack/k8s-in-k8s/port_check.sh
new file mode 100644
index 000000000..2966ee121
--- /dev/null
+++ b/hack/k8s-in-k8s/port_check.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+function check_port {
+    local ip=$1
+    local port=$2
+
+    if timeout 1 curl -s --connect-timeout 3 $ip:$port >/dev/null; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+nodes=$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name} {.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}')
+
+node_array=()
+
+while IFS= read -r line; do
+    node_array+=("$line")
+done <<< "$nodes"
+
+for node in "${node_array[@]}"; do
+    name=$(echo $node | awk '{print $1}')
+    ip=$(echo $node | awk '{print $2}')
+    
+    if check_port $ip 5678; then
+      echo ""
+    else
+        echo "节点: $name, IP: $ip 端口5678不可访问"
+    fi
+done