From de319c90fa2960e9b02f7ab177c0f8ae5529b55f Mon Sep 17 00:00:00 2001 From: baoyinghai_yewu Date: Mon, 21 Oct 2024 14:06:08 +0800 Subject: [PATCH] feat: add nginx for kubelet Signed-off-by: baoyinghai_yewu --- hack/k8s-in-k8s/g.env.sh | 85 ----------------- hack/k8s-in-k8s/generate_env.sh | 32 ++++++- hack/k8s-in-k8s/kubelet_node_helper.sh | 92 +++++++++++++++++++ .../workflow/task/task.go | 20 ++++ .../workflow/workflow.go | 1 + pkg/kubenest/tasks/manifests_components.go | 5 +- 6 files changed, 147 insertions(+), 88 deletions(-) delete mode 100644 hack/k8s-in-k8s/g.env.sh diff --git a/hack/k8s-in-k8s/g.env.sh b/hack/k8s-in-k8s/g.env.sh deleted file mode 100644 index fd1f0d64e..000000000 --- a/hack/k8s-in-k8s/g.env.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -# ##### -# Generate by script generate_env.sh -# ##### - -SCRIPT_VERSION=0.0.1 -# tmp dir of kosmos -PATH_FILE_TMP=/apps/conf/kosmos/tmp -################################################## -# path for kubeadm config -PATH_KUBEADM_CONFIG=/etc/kubeadm -################################################## -# path for kubernetes, from kubelet args --config -PATH_KUBERNETES=/etc/kubernetes -PATH_KUBERNETES_PKI=/etc/kubernetes/pki -# name for kubelet kubeconfig file -KUBELET_KUBE_CONFIG_NAME= -################################################## -# path for kubelet -PATH_KUBELET_LIB=/var/lib/kubelet -# path for kubelet -PATH_KUBELET_CONF=. -# name for config file of kubelet -KUBELET_CONFIG_NAME= -HOST_CORE_DNS=10.96.0.10 -# kubeadm switch -USE_KUBEADM=false -# Generate kubelet.conf TIMEOUT -KUBELET_CONF_TIMEOUT=30 - -function GenerateKubeadmConfig() { - echo "--- -apiVersion: kubeadm.k8s.io/v1beta2 -caCertPath: /etc/kubernetes/pki/ca.crt -discovery: - bootstrapToken: - apiServerEndpoint: apiserver.cluster.local:6443 - token: $1 - unsafeSkipCAVerification: true -kind: JoinConfiguration -nodeRegistration: - criSocket: /run/containerd/containerd.sock - kubeletExtraArgs: - container-runtime: remote - container-runtime-endpoint: unix:///run/containerd/containerd.sock - taints: null" > $2/kubeadm.cfg.current -} - -function GenerateStaticNginxProxy() { - echo "apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: nginx-proxy - namespace: kube-system -spec: - containers: - - image: registry.paas/cmss/nginx:1.21.4 - imagePullPolicy: IfNotPresent - name: nginx-proxy - resources: - limits: - cpu: 300m - memory: 512M - requests: - cpu: 25m - memory: 32M - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/nginx - name: etc-nginx - readOnly: true - hostNetwork: true - priorityClassName: system-node-critical - volumes: - - hostPath: - path: /apps/conf/nginx - type: - name: etc-nginx -status: {}" > /etc/kubernetes/manifests/nginx-proxy.yaml -} - - diff --git a/hack/k8s-in-k8s/generate_env.sh b/hack/k8s-in-k8s/generate_env.sh index 02eb5cc38..4fc5737cb 100644 --- a/hack/k8s-in-k8s/generate_env.sh +++ b/hack/k8s-in-k8s/generate_env.sh @@ -80,10 +80,18 @@ function GetFileName() { function GetDirectory() { local fullpath="$1" + if [ -z "$fullpath" ]; then + echo "Error: No directory found." + exit 1 + fi local directory=$(dirname "$fullpath") echo "$directory" } +function GetMasterNodeIPs() { + kubectl get nodes -l node-role.kubernetes.io/master="" -o jsonpath='{range .items[*]}{.status.addresses[?(@.type=="InternalIP")].address}{" "}{end}' +} + # kubelet config name KUBELET_CONFIG_NAME=$(GetFileName "$(GetKubeletConfigFilePath)") # path for kubelet @@ -97,6 +105,15 @@ PATH_KUBERNETES_PKI=$(GetDirectory "$(GetKubernetesCaPath)") PATH_KUBERNETES=$(GetDirectory $PATH_KUBERNETES_PKI) HOST_CORE_DNS=$(GetKubeDnsClusterIP) +DOCKER_IMAGE_NGINX="registry.paas/cmss/nginx:1.21.4" +SERVERS=$(GetMasterNodeIPs) +if [ -z "$SERVERS" ]; then + echo "Error: No master nodes found or failed to retrieve node IPs." + exit 1 +fi +LOCAL_PORT="6443" +LOCAL_IP="127.0.0.1" # [::1] + echo "#!/usr/bin/env bash # ##### @@ -128,6 +145,13 @@ USE_KUBEADM=false # Generate kubelet.conf TIMEOUT KUBELET_CONF_TIMEOUT=30 +# load balance +DOCKER_IMAGE_NGINX=$DOCKER_IMAGE_NGINX +SERVERS=($SERVERS) +LOCAL_PORT="6443" +LOCAL_IP="127.0.0.1" # [::1] +USE_NGINX=true + function GenerateKubeadmConfig() { echo \"--- apiVersion: kubeadm.k8s.io/v1beta2 @@ -147,6 +171,10 @@ nodeRegistration: } function GenerateStaticNginxProxy() { + config_path=/apps/conf/nginx + if [ "\$1" == \"true\" ]; then + config_path=\$PATH_FILE_TMP + fi echo \"apiVersion: v1 kind: Pod metadata: @@ -155,7 +183,7 @@ metadata: namespace: kube-system spec: containers: - - image: registry.paas/cmss/nginx:1.21.4 + - image: \$DOCKER_IMAGE_NGINX imagePullPolicy: IfNotPresent name: nginx-proxy resources: @@ -175,7 +203,7 @@ spec: priorityClassName: system-node-critical volumes: - hostPath: - path: /apps/conf/nginx + path: \$config_path type: name: etc-nginx status: {}\" > $PATH_KUBERNETES/manifests/nginx-proxy.yaml diff --git a/hack/k8s-in-k8s/kubelet_node_helper.sh b/hack/k8s-in-k8s/kubelet_node_helper.sh index 3f409fd0b..d02e75012 100755 --- a/hack/k8s-in-k8s/kubelet_node_helper.sh +++ b/hack/k8s-in-k8s/kubelet_node_helper.sh @@ -330,11 +330,103 @@ function version() { echo "$SCRIPT_VERSION" } + +function is_ipv6() { + if [[ "$1" =~ : ]]; then + return 0 + else + return 1 + fi +} + +function install_lb() { + if [ -z "$USE_NGINX" ]; then + export USE_KUBEADM=false + fi + + if [ "$USE_NGINX" = false ]; then + exit 0 + fi + + echo "exec(1/6): get port of apiserver...." + + PORT=$(grep 'server:' "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" | awk -F '[:/]' '{print $NF}') + + if [ -z "$PORT" ]; then + echo "can not get port" + exit 1 + else + echo "port is $PORT" + fi + + if [ "$LOCAL_PORT" -eq "$PORT" ]; then + echo "Error: LOCAL_PORT ($LOCAL_PORT) cannot be the same as the backend port ($PORT)." + exit 0 + fi + + # Start generating nginx.conf + echo "exec(1/6): generate nginx.conf...." + cat < "$PATH_FILE_TMP/nginx.conf" +error_log stderr notice; +worker_processes 1; +events { + multi_accept on; + use epoll; + worker_connections 1024; +} + +stream { + upstream kube_apiserver { + least_conn; +EOL + + # Loop through the array and append each server to the nginx.conf file + for SERVER in "${SERVERS[@]}"; do + if is_ipv6 "$SERVER"; then + echo " server [$SERVER]:$PORT;" >> "$PATH_FILE_TMP/nginx.conf" + else + echo " server $SERVER:$PORT;" >> "$PATH_FILE_TMP/nginx.conf" + fi + done + + # Continue writing the rest of the nginx.conf + cat <> "$PATH_FILE_TMP/nginx.conf" + } + server { + listen [::]:$LOCAL_PORT; + listen 6443; + proxy_pass kube_apiserver; + proxy_timeout 10m; + proxy_connect_timeout 10s; + } +} +EOL + + echo "exec(1/6): create static pod" + GenerateStaticNginxProxy true + + + echo "exec(1/6): restart static pod" + mv "${PATH_KUBERNETES}/manifests/nginx-proxy.yaml" "${PATH_KUBERNETES}/nginx-proxy.yaml" + sleep 2 + mv "${PATH_KUBERNETES}/nginx-proxy.yaml" "${PATH_KUBERNETES}/manifests/nginx-proxy.yaml" + + echo "exec(1/6): update kubelet.conf" + cp "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}.bak" + sed -i "s|server: .*|server: https://${LOCAL_IP}:${LOCAL_PORT}|" "${PATH_KUBERNETES}/${KUBELET_KUBE_CONFIG_NAME}" + + echo "exec(1/6): restart kubelet" + systemctl restart kubelet +} + # See how we were called. case "$1" in unjoin) unjoin ;; + install_lb) + install_lb + ;; join) join ;; diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go index d7a297367..087a759dc 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/task/task.go @@ -336,6 +336,26 @@ func NewWaitNodeReadyTask(isHost bool) Task { } } +func NewInstallNginxTask() Task { + return Task{ + Name: "remote install nginx", + Retry: true, + Run: func(ctx context.Context, to TaskOpt, _ interface{}) (interface{}, error) { + exectHelper := exector.NewExectorHelper(to.NodeInfo.Spec.NodeIP, "") + + joinCmd := &exector.CMDExector{ + Cmd: fmt.Sprintf("bash %s install_lb", env.GetExectorShellName()), + } + to.Loger().Infof("install nginx %s with cmd: %s", to.NodeInfo.Name, joinCmd.Cmd) + ret := exectHelper.DoExector(ctx.Done(), joinCmd) + if ret.Status != exector.SUCCESS { + return nil, fmt.Errorf("nstall nginx %s failed: %s", to.NodeInfo.Name, ret.String()) + } + return nil, nil + }, + } +} + // nolint:dupl func NewUpdateVirtualNodeLabelsTask() Task { return Task{ diff --git a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/workflow.go b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/workflow.go index a6e715f47..44d076bdc 100644 --- a/pkg/kubenest/controller/virtualcluster.node.controller/workflow/workflow.go +++ b/pkg/kubenest/controller/virtualcluster.node.controller/workflow/workflow.go @@ -90,6 +90,7 @@ func NewJoinWorkFlow() WorkflowData { task.NewRemoteUpdateConfigYamlTask(), task.NewRemoteNodeJoinTask(), task.NewWaitNodeReadyTask(false), + task.NewInstallNginxTask(), task.NewUpdateVirtualNodeLabelsTask(), task.NewUpdateNodePoolItemStatusTask(v1alpha1.NodeInUse, false), } diff --git a/pkg/kubenest/tasks/manifests_components.go b/pkg/kubenest/tasks/manifests_components.go index 6b59d9866..225d67ccf 100644 --- a/pkg/kubenest/tasks/manifests_components.go +++ b/pkg/kubenest/tasks/manifests_components.go @@ -106,7 +106,6 @@ func applyComponentsManifests(r workflow.RunData) error { if nodeCount < constants.VipKeepAlivedReplicas { keepalivedReplicas = int(nodeCount) } - templatedMapping["KeepalivedReplicas"] = keepalivedReplicas } @@ -124,6 +123,10 @@ func applyComponentsManifests(r workflow.RunData) error { }, }) + for k, v := range data.PluginOptions() { + templatedMapping[k] = v + } + for _, component := range components { klog.V(2).Infof("Deploy component %s", component.Name) if v, ok := skipComponents[component.Name]; ok && v {