From afebfcf4dd43048a953fff6c6909099261c35129 Mon Sep 17 00:00:00 2001
From: qiuwei <qiuwei_yewu@cmss.chinamobile.com>
Date: Fri, 5 Jul 2024 16:30:15 +0800
Subject: [PATCH] feat: Add nodes back to the host cluster without kubeadm

Signed-off-by: qiuwei <qiuwei_yewu@cmss.chinamobile.com>
---
 hack/k8s-in-k8s/generate_env.sh        |  7 ++
 hack/k8s-in-k8s/kubelet_node_helper.sh | 92 +++++++++++++++++++++++++-
 2 files changed, 96 insertions(+), 3 deletions(-)

diff --git a/hack/k8s-in-k8s/generate_env.sh b/hack/k8s-in-k8s/generate_env.sh
index 11089b0cf..5a1045ada 100644
--- a/hack/k8s-in-k8s/generate_env.sh
+++ b/hack/k8s-in-k8s/generate_env.sh
@@ -107,6 +107,9 @@ SCRIPT_VERSION=$SCRIPT_VERSION
 # tmp dir of kosmos
 PATH_FILE_TMP=$PATH_FILE_TMP
 ##################################################
+# path for generate ca.crt
+PATH_FILE_CACRT=$PATH_FILE_CACRT
+##################################################
 # path for kubeadm config
 PATH_KUBEADM_CONFIG=$PATH_KUBEADM_CONFIG
 ##################################################
@@ -123,6 +126,10 @@ PATH_KUBELET_CONF=$PATH_KUBELET_CONF
 # name for config file of kubelet
 KUBELET_CONFIG_NAME=$KUBELET_CONFIG_NAME
 HOST_CORE_DNS=$HOST_CORE_DNS
+# kubeadm switch
+USE_KUBEADM=true
+# Generate kubelet.conf TIMEOUT
+KUBELET_CONF_TIMEOUT=30
 
 function GenerateKubeadmConfig() {
     echo \"---
diff --git a/hack/k8s-in-k8s/kubelet_node_helper.sh b/hack/k8s-in-k8s/kubelet_node_helper.sh
index 65180e719..1d015731a 100755
--- a/hack/k8s-in-k8s/kubelet_node_helper.sh
+++ b/hack/k8s-in-k8s/kubelet_node_helper.sh
@@ -89,6 +89,65 @@ function afterRevert() {
     fi
 }
 
+function get_ca_certificate() {
+     local output_file="$PATH_FILE_CACRT/ca.crt"
+     local kubeconfig_data=$(curl -sS --insecure "https://$JOIN_HOST/api/v1/namespaces/kube-public/configmaps/cluster-info" 2>/dev/null | \
+                             grep -oP 'certificate-authority-data:\s*\K.*(?=server:[^[:space:]]*?)' | \
+                             sed -e 's/^certificate-authority-data://' -e 's/[[:space:]]//g' -e 's/\\n$//g')
+
+     # verify the kubeconfig data is not empty
+     if [ -z "$kubeconfig_data" ]; then
+       echo "Failed to extract certificate-authority-data."
+       return 1
+     fi
+
+     # Base64 decoded and written to a file
+     echo "$kubeconfig_data" | base64 --decode > "$output_file"
+
+     # check that the file was created successfully
+     if [ -f "$output_file" ]; then
+         echo "certificate-authority-data saved to $output_file"
+     else
+         echo "Failed to save certificate-authority-data to $output_file"
+      return 1
+     fi
+}
+
+function create_kubelet_bootstrap_config() {
+   # Checks if the parameters are provided
+ if [ -z "$JOIN_HOST" ] || [ -z "$JOIN_TOKEN" ]; then
+     echo "Please provide server and token as parameters."
+     return 1
+ fi
+
+ # Define file contents
+ cat << EOF > bootstrap-kubelet.conf
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority: $PATH_FILE_CACRT/ca.crt
+    server: https://$JOIN_HOST
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: kubelet-bootstrap
+  name: kubelet-bootstrap-context
+current-context: kubelet-bootstrap-context
+preferences: {}
+users:
+- name: kubelet-bootstrap
+  user:
+    token: $JOIN_TOKEN
+EOF
+
+ # copy the file to the /etc/kubernetes directory
+ cp bootstrap-kubelet.conf $PATH_KUBERNETES
+
+ echo "the file bootstrap-kubelet.conf has stored in $PATH_KUBERNETES directory."
+}
+
 function revert() {
     echo "exec(1/5): update kubeadm.cfg..."
     if [ ! -f "$PATH_KUBEADM_CONFIG/kubeadm.cfg" ]; then
@@ -117,9 +176,17 @@ function revert() {
 
 
     echo "exec(4/5): execute join cmd...."
-    kubeadm join --config "$PATH_FILE_TMP/kubeadm.cfg.current"
-    if [ $? -ne 0 ]; then
-        exit 1
+    if [ "$USE_KUBEADM" = "true" ]; then
+       echo "use kubeadm to join node to host"
+       kubeadm join --config "$PATH_FILE_TMP/kubeadm.cfg.current"
+       if [ $? -ne 0 ]; then
+          exit 1
+       fi
+    else
+       echo "NONONO use kubeadm to join node to host"
+       get_ca_certificate $JOIN_HOST
+       create_kubelet_bootstrap_config $JOIN_HOST $JOIN_TOKEN
+       cp $PATH_FILE_TMP/kubeadm-flags.env $PATH_KUBELET_LIB
     fi
 
     echo "exec(5/5): restart cotnainerd...."
@@ -128,6 +195,24 @@ function revert() {
         exit 1
     fi
 
+    if [ "$USE_KUBEADM" = "false" ]; then
+           systemctl start kubelet
+           elapsed_time=0
+
+           while [ $elapsed_time -lt $KUBELET_CONF_TIMEOUT ]; do
+             if [ -f "/etc/kubernetes/kubelet.conf" ]; then
+                rm -f "/etc/kubernetes/bootstrap-kubelet.conf"
+                echo "Deleted bootstrap-kubelet.conf file as kubelet.conf exists."
+                break
+             fi
+             sleep 2
+             elapsed_time=$((elapsed_time + 2))
+           done
+
+           if [ $elapsed_time -ge $KUBELET_CONF_TIMEOUT ]; then
+              echo "Timeout: kubelet.conf was not generated within $KUBELET_CONF_TIMEOUT seconds. Continuing script execution."
+           fi
+        fi
     afterRevert
     if [ $? -ne 0 ]; then
         exit 1
@@ -219,6 +304,7 @@ function check() {
     fi
     
     echo "check(2/2): copy  kubeadm-flags.env  to create $PATH_FILE_TMP , remove args[cloud-provider] and taints"
+    cp "$PATH_KUBELET_LIB/kubeadm-flags.env" "${PATH_FILE_TMP}/kubeadm-flags.env.origin"
     sed -e "s| --cloud-provider=external | |g" -e "w ${PATH_FILE_TMP}/kubeadm-flags.env" "$PATH_KUBELET_LIB/kubeadm-flags.env"
     sed -i "s| --register-with-taints=node.kosmos.io/unschedulable:NoSchedule||g" "${PATH_FILE_TMP}/kubeadm-flags.env"
     if [ $? -ne 0 ]; then