diff --git a/hack/k8s-in-k8s/generate_env.sh b/hack/k8s-in-k8s/generate_env.sh index 11089b0cf..9b9cd0d1a 100644 --- a/hack/k8s-in-k8s/generate_env.sh +++ b/hack/k8s-in-k8s/generate_env.sh @@ -123,6 +123,10 @@ PATH_KUBELET_CONF=$PATH_KUBELET_CONF # name for config file of kubelet KUBELET_CONFIG_NAME=$KUBELET_CONFIG_NAME HOST_CORE_DNS=$HOST_CORE_DNS +# kubeadm switch +USE_KUBEADM=true +# Generate kubelet.conf TIMEOUT +KUBELET_CONF_TIMEOUT=30 function GenerateKubeadmConfig() { echo \"--- diff --git a/hack/k8s-in-k8s/kubelet_node_helper.sh b/hack/k8s-in-k8s/kubelet_node_helper.sh index 65180e719..3f409fd0b 100755 --- a/hack/k8s-in-k8s/kubelet_node_helper.sh +++ b/hack/k8s-in-k8s/kubelet_node_helper.sh @@ -89,6 +89,65 @@ function afterRevert() { fi } +function get_ca_certificate() { + local output_file="$PATH_KUBERNETES_PKI/ca.crt" + local kubeconfig_data=$(curl -sS --insecure "https://$JOIN_HOST/api/v1/namespaces/kube-public/configmaps/cluster-info" 2>/dev/null | \ + grep -oP 'certificate-authority-data:\s*\K.*(?=server:[^[:space:]]*?)' | \ + sed -e 's/^certificate-authority-data://' -e 's/[[:space:]]//g' -e 's/\\n$//g') + + # verify the kubeconfig data is not empty + if [ -z "$kubeconfig_data" ]; then + echo "Failed to extract certificate-authority-data." + return 1 + fi + + # Base64 decoded and written to a file + echo "$kubeconfig_data" | base64 --decode > "$output_file" + + # check that the file was created successfully + if [ -f "$output_file" ]; then + echo "certificate-authority-data saved to $output_file" + else + echo "Failed to save certificate-authority-data to $output_file" + return 1 + fi +} + +function create_kubelet_bootstrap_config() { + # Checks if the parameters are provided + if [ -z "$JOIN_HOST" ] || [ -z "$JOIN_TOKEN" ]; then + echo "Please provide server and token as parameters." + return 1 + fi + + # Define file contents + cat << EOF > bootstrap-kubelet.conf +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority: $PATH_KUBERNETES_PKI/ca.crt + server: https://$JOIN_HOST + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: kubelet-bootstrap + name: kubelet-bootstrap-context +current-context: kubelet-bootstrap-context +preferences: {} +users: +- name: kubelet-bootstrap + user: + token: $JOIN_TOKEN +EOF + + # copy the file to the /etc/kubernetes directory + cp bootstrap-kubelet.conf $PATH_KUBERNETES + + echo "the file bootstrap-kubelet.conf has stored in $PATH_KUBERNETES directory." +} + function revert() { echo "exec(1/5): update kubeadm.cfg..." if [ ! -f "$PATH_KUBEADM_CONFIG/kubeadm.cfg" ]; then @@ -117,9 +176,26 @@ function revert() { echo "exec(4/5): execute join cmd...." - kubeadm join --config "$PATH_FILE_TMP/kubeadm.cfg.current" - if [ $? -ne 0 ]; then - exit 1 + if [ -z "$USE_KUBEADM" ]; then + # if "USE_KUBEADM is not set, default set to true" + export USE_KUBEADM=true + fi + if [ "$USE_KUBEADM" = true ]; then + echo "use kubeadm to join node to host" + kubeadm join --config "$PATH_FILE_TMP/kubeadm.cfg.current" + if [ $? -ne 0 ]; then + exit 1 + fi + else + echo "NONONO use kubeadm to join node to host" + get_ca_certificate $JOIN_HOST + create_kubelet_bootstrap_config $JOIN_HOST $JOIN_TOKEN + if [ -f "${PATH_FILE_TMP}/kubeadm-flags.env.origin" ]; then + cp "${PATH_FILE_TMP}/kubeadm-flags.env.origin" "${PATH_KUBELET_LIB}" && \ + mv "${PATH_KUBELET_LIB}/kubeadm-flags.env.origin" "${PATH_KUBELET_LIB}/kubeadm-flags.env" + else + cp "${PATH_FILE_TMP}/kubeadm-flags.env" "${PATH_KUBELET_LIB}" + fi fi echo "exec(5/5): restart cotnainerd...." @@ -128,6 +204,24 @@ function revert() { exit 1 fi + if [ "$USE_KUBEADM" = false ]; then + systemctl start kubelet + elapsed_time=0 + + while [ $elapsed_time -lt $KUBELET_CONF_TIMEOUT ]; do + if [ -f "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" ]; then + rm -f "${PATH_KUBERNETES}/bootstrap-kubelet.conf" + echo "Deleted bootstrap-kubelet.conf file as kubelet.conf exists." + break + fi + sleep 2 + elapsed_time=$((elapsed_time + 2)) + done + + if [ $elapsed_time -ge $KUBELET_CONF_TIMEOUT ]; then + echo "Timeout: kubelet.conf was not generated within $KUBELET_CONF_TIMEOUT seconds. Continuing script execution." + fi + fi afterRevert if [ $? -ne 0 ]; then exit 1 @@ -219,6 +313,10 @@ function check() { fi echo "check(2/2): copy kubeadm-flags.env to create $PATH_FILE_TMP , remove args[cloud-provider] and taints" + # Since this function is used both to detach nodes, we need to make sure we haven't copied kubeadm-flags.env before + if [ ! -f "${PATH_FILE_TMP}/kubeadm-flags.env.origin" ]; then + cp "${PATH_KUBELET_LIB}/kubeadm-flags.env" "${PATH_FILE_TMP}/kubeadm-flags.env.origin" + fi sed -e "s| --cloud-provider=external | |g" -e "w ${PATH_FILE_TMP}/kubeadm-flags.env" "$PATH_KUBELET_LIB/kubeadm-flags.env" sed -i "s| --register-with-taints=node.kosmos.io/unschedulable:NoSchedule||g" "${PATH_FILE_TMP}/kubeadm-flags.env" if [ $? -ne 0 ]; then diff --git a/hack/k8s-in-k8s/port_check.sh b/hack/k8s-in-k8s/port_check.sh index 2966ee121..d27cb349d 100644 --- a/hack/k8s-in-k8s/port_check.sh +++ b/hack/k8s-in-k8s/port_check.sh @@ -4,7 +4,12 @@ function check_port { local ip=$1 local port=$2 - if timeout 1 curl -s --connect-timeout 3 $ip:$port >/dev/null; then + # Check if the IP address is IPv6, then enclose it in square brackets + if [[ $ip =~ .*:.* ]]; then + ip="[$ip]" + fi + + if timeout 1 curl -s --connect-timeout 3 "${ip}:${port}" >/dev/null; then return 0 else return 1 diff --git a/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller.go b/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller.go index 231cb0f29..b67075ae7 100644 --- a/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller.go +++ b/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller.go @@ -57,12 +57,12 @@ func (e *ApiServerExternalSyncController) SyncApiServerExternalEPS(ctx context.C kubeEndpoints, err := k8sClient.CoreV1().Endpoints(constants.DefaultNs).Get(ctx, "kubernetes", metav1.GetOptions{}) if err != nil { klog.Errorf("Error getting endpoints: %v", err) - return fmt.Errorf("failed to get endpoints for kubernetes service: %v", err) + return err } else { - klog.Infof("Endpoints for service 'kubernetes': %v", kubeEndpoints) + klog.V(4).Infof("Endpoints for service 'kubernetes': %v", kubeEndpoints) for _, subset := range kubeEndpoints.Subsets { for _, address := range subset.Addresses { - klog.Infof("IP: %s", address.IP) + klog.V(4).Infof("IP: %s", address.IP) } } } @@ -72,39 +72,42 @@ func (e *ApiServerExternalSyncController) SyncApiServerExternalEPS(ctx context.C } if kubeEndpoints.Subsets[0].Addresses == nil || len(kubeEndpoints.Subsets[0].Addresses) == 0 { - return fmt.Errorf("eps %s Addresses length is nil", "kubernetes") + klog.Errorf("eps %s Addresses length is nil", "kubernetes") + return err } apiServerExternalEndpoints, err := k8sClient.CoreV1().Endpoints(constants.DefaultNs).Get(ctx, constants.ApiServerExternalService, metav1.GetOptions{}) if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("failed to get endpoints for %s : %v", constants.ApiServerExternalService, err) + klog.Errorf("failed to get endpoints for %s : %v", constants.ApiServerExternalService, err) + return err } updateEPS := apiServerExternalEndpoints.DeepCopy() if apiServerExternalEndpoints != nil { - klog.Infof("apiServerExternalEndpoints: %v", apiServerExternalEndpoints) + klog.V(4).Infof("apiServerExternalEndpoints: %v", apiServerExternalEndpoints) } else { - klog.Info("apiServerExternalEndpoints is nil") + klog.V(4).Info("apiServerExternalEndpoints is nil") } if updateEPS != nil { - klog.Infof("updateEPS: %v", updateEPS) + klog.V(4).Infof("updateEPS: %v", updateEPS) } else { - klog.Info("updateEPS is nil") + klog.V(4).Info("updateEPS is nil") } if len(updateEPS.Subsets) == 1 && len(updateEPS.Subsets[0].Addresses) == 1 { ip := kubeEndpoints.Subsets[0].Addresses[0].IP - klog.Infof("IP address: %s", ip) + klog.V(4).Infof("IP address: %s", ip) updateEPS.Subsets[0].Addresses[0].IP = ip if _, err := k8sClient.CoreV1().Endpoints(constants.DefaultNs).Update(ctx, updateEPS, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("failed to update endpoints for api-server-external-service: %v", err) + klog.Errorf("failed to update endpoints for api-server-external-service: %v", err) + return err } } else { klog.ErrorS(err, "Unexpected format of endpoints for api-server-external-service", "endpoint_data", updateEPS) - return fmt.Errorf("unexpected format of endpoints for api-server-external-service") + return err } return nil @@ -137,7 +140,7 @@ func (e *ApiServerExternalSyncController) Reconcile(ctx context.Context, request return reconcile.Result{}, nil } - if targetVirtualCluster.Status.Phase != v1alpha1.AllNodeReady && targetVirtualCluster.Status.Phase != v1alpha1.Completed { + if targetVirtualCluster.Status.Phase != v1alpha1.Initialized { return reconcile.Result{RequeueAfter: utils.DefaultRequeueTime}, nil } diff --git a/pkg/kubenest/controlplane/endpoint.go b/pkg/kubenest/controlplane/endpoint.go index f0e05a76e..cf67d09ec 100644 --- a/pkg/kubenest/controlplane/endpoint.go +++ b/pkg/kubenest/controlplane/endpoint.go @@ -33,7 +33,7 @@ func EnsureApiServerExternalEndPoint(dynamicClient dynamic.Interface) error { } func installApiServerExternalEndpointInVirtualCluster(dynamicClient dynamic.Interface) error { - klog.Info("begin to get kubernetes endpoint") + klog.V(4).Info("begin to get kubernetes endpoint") kubeEndpointUnstructured, err := dynamicClient.Resource(schema.GroupVersionResource{ Group: "", Version: "v1", @@ -74,7 +74,7 @@ func installApiServerExternalEndpointInVirtualCluster(dynamicClient dynamic.Inte klog.Error("create api-server-external-service endpoint failed", err) return errors.Wrap(err, "failed to create api-server-external-service endpoint") } else { - klog.Info("success create api-server-external-service endpoint:", createResult) + klog.V(4).Info("success create api-server-external-service endpoint:", createResult) } } else { return errors.New("kubernetes endpoint does not exist") @@ -110,7 +110,7 @@ func installApiServerExternalServiceInVirtualCluster(dynamicClient dynamic.Inter } func getEndPointPort(dynamicClient dynamic.Interface) (int32, error) { - klog.Info("begin to get Endpoints ports...") + klog.V(4).Info("begin to get Endpoints ports...") endpointsRes := dynamicClient.Resource(schema.GroupVersionResource{ Group: "", Version: "v1", @@ -153,6 +153,6 @@ func getEndPointPort(dynamicClient dynamic.Interface) (int32, error) { return 0, fmt.Errorf("port field not found or error parsing it") } - klog.Infof("The port number was successfully obtained: %d", portNum) + klog.V(4).Infof("The port number was successfully obtained: %d", portNum) return int32(portNum), nil }