From fa4f60aad90412c196b8dd0f977016b58c9010fa Mon Sep 17 00:00:00 2001 From: gaoyuan Date: Wed, 27 Nov 2024 15:16:36 +0800 Subject: [PATCH] update node-agent Heartbeat Signed-off-by: gaoyuan --- cmd/kubenest/node-agent/app/root.go | 12 ++-- cmd/kubenest/node-agent/app/serve/serve.go | 69 ++++++++++++++++++++++ deploy/virtual-cluster-operator.yml | 13 ++++ hack/node-agent/agent.env | 1 + hack/node-agent/init.sh | 4 +- 5 files changed, 92 insertions(+), 7 deletions(-) diff --git a/cmd/kubenest/node-agent/app/root.go b/cmd/kubenest/node-agent/app/root.go index aeb9d2d59..e5bdef67c 100644 --- a/cmd/kubenest/node-agent/app/root.go +++ b/cmd/kubenest/node-agent/app/root.go @@ -1,7 +1,6 @@ package app import ( - "os" "strings" "github.com/spf13/cobra" @@ -30,11 +29,12 @@ var RootCmd = &cobra.Command{ func initConfig() { // Tell Viper to automatically look for a .env file - viper.SetConfigFile("agent.env") - currentDir, _ := os.Getwd() - viper.AddConfigPath(currentDir) - viper.AddConfigPath("/srv/node-agent/agent.env") - viper.SetConfigType("toml") + //viper.SetConfigFile("agent.env") + viper.SetConfigFile("/srv/node-agent/agent.env") + //currentDir, _ := os.Getwd() + //viper.AddConfigPath(currentDir) + //viper.AddConfigPath("/srv/node-agent/agent.env") + //viper.SetConfigType("toml") // If a agent.env file is found, read it in. if err := viper.ReadInConfig(); err != nil { log.Warnf("Load config file error, %s", err) diff --git a/cmd/kubenest/node-agent/app/serve/serve.go b/cmd/kubenest/node-agent/app/serve/serve.go index 0ecedf783..d07debb6d 100644 --- a/cmd/kubenest/node-agent/app/serve/serve.go +++ b/cmd/kubenest/node-agent/app/serve/serve.go @@ -2,6 +2,7 @@ package serve import ( "bufio" + "context" "crypto/sha256" "crypto/tls" "encoding/base64" @@ -22,8 +23,12 @@ import ( "github.com/gorilla/websocket" "github.com/spf13/cobra" "github.com/spf13/viper" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" "github.com/kosmos.io/kosmos/cmd/kubenest/node-agent/app/logger" + "github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned" ) var ( @@ -36,6 +41,7 @@ var ( certFile string // SSL certificate file keyFile string // SSL key file addr string // server listen address + nodeName string // server nodename log = logger.GetLogger() ) @@ -47,14 +53,28 @@ var upgrader = websocket.Upgrader{ }, } // use default options +const ( + defaultKubeconfig = "/srv/node-agent/kubeconfigpath" +) // kubeconfig for heartbeatCheck + func init() { // setup flags ServeCmd.PersistentFlags().StringVarP(&addr, "addr", "a", ":5678", "websocket service address") ServeCmd.PersistentFlags().StringVarP(&certFile, "cert", "c", "cert.pem", "SSL certificate file") ServeCmd.PersistentFlags().StringVarP(&keyFile, "key", "k", "key.pem", "SSL key file") + ServeCmd.PersistentFlags().StringVarP(&nodeName, "nodename", "n", "", "set nodename") } func serveCmdRun(_ *cobra.Command, _ []string) error { + //start heartbeatCheck Goroutine + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if len(nodeName) == 0 { + nodeName = viper.GetString("NODE_NAME") + } + go heartbeatCheck(ctx, nodeName) + user := viper.GetString("WEB_USER") password := viper.GetString("WEB_PASS") port := viper.GetString("WEB_PORT") @@ -65,9 +85,57 @@ func serveCmdRun(_ *cobra.Command, _ []string) error { if port != "" { addr = ":" + port } + return Start(addr, certFile, keyFile, user, password) } +func heartbeatCheck(ctx context.Context, nodeName string) { + kubeconfigPath := defaultKubeconfig + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + log.Errorf("Failed to load kubeconfig from path %s:%v", kubeconfigPath, err) + return + } + kosmosClient, err := versioned.NewForConfig(config) + if err != nil { + log.Errorf("Failed to get config: %v", err) + return + } + + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Infof("Heartbeat for node %s stopped", nodeName) + return + case <-ticker.C: + node, err := kosmosClient.KosmosV1alpha1().GlobalNodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + log.Errorf("Failed to get node: %v", err) + } + heartbeatTime := metav1.Now() + + if len(node.Status.Conditions) == 0 { + log.Infof("GlobalNode %s has no conditions, initializing default condition", node.Name) + node.Status.Conditions = []corev1.NodeCondition{ + { + LastHeartbeatTime: heartbeatTime, + }, + } + } else { + node.Status.Conditions[0].LastHeartbeatTime = heartbeatTime + } + if _, err := kosmosClient.KosmosV1alpha1().GlobalNodes().UpdateStatus(ctx, node, metav1.UpdateOptions{}); err != nil { + log.Errorf("update node %s status for globalnode failed, %v", node.Name, err) + } else { + log.Infof("GlobalnodeHeartbeat: successfully updated global node %s, Status.Conditions: %+v", node.Name, node.Status.Conditions) + } + } + } +} + // start server func Start(addr, certFile, keyFile, user, password string) error { passwordHash := sha256.Sum256([]byte(password)) @@ -146,6 +214,7 @@ func Start(addr, certFile, keyFile, user, password string) error { TLSConfig: tlsConfig, ReadHeaderTimeout: 10 * time.Second, } + err := server.ListenAndServeTLS("", "") if err != nil { log.Errorf("failed to start server %v", err) diff --git a/deploy/virtual-cluster-operator.yml b/deploy/virtual-cluster-operator.yml index 77c6a542a..abc09291e 100644 --- a/deploy/virtual-cluster-operator.yml +++ b/deploy/virtual-cluster-operator.yml @@ -230,6 +230,11 @@ spec: secretKeyRef: name: node-agent-secret key: password + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName command: ["/bin/bash"] args: - "/app/init.sh" @@ -240,6 +245,10 @@ spec: - mountPath: /host-systemd name: systemd-path readOnly: false + - mountPath: /app/kubeconfigpath + name: kubeconfig + subPath: kubeconfig + readOnly: false containers: - name: install-agent image: cis-hub-huabei-3.cmecloud.cn/node-agent/node-agent:latest @@ -282,6 +291,10 @@ spec: hostPath: path: /etc/systemd/system type: DirectoryOrCreate + - name: kubeconfig + secret: + secretName: virtual-cluster-operator + --- apiVersion: v1 kind: Secret diff --git a/hack/node-agent/agent.env b/hack/node-agent/agent.env index 9aa151f03..ea0e69420 100644 --- a/hack/node-agent/agent.env +++ b/hack/node-agent/agent.env @@ -1,3 +1,4 @@ WEB_USER={{WEB_USER}} WEB_PASS={{WEB_PASS}} WEB_PORT={{WEB_PORT}} +NODE_NAME={{NODE_NAME}} \ No newline at end of file diff --git a/hack/node-agent/init.sh b/hack/node-agent/init.sh index 523dbf5f9..dc1918bbe 100644 --- a/hack/node-agent/init.sh +++ b/hack/node-agent/init.sh @@ -3,7 +3,9 @@ WEB_USER="$WEB_USER" sed -i 's/^WEB_USER=.*/WEB_USER="'"$WEB_USER"'"/' /app/agent.env WEB_PASS="$WEB_PASS" sed -i 's/^WEB_PASS=.*/WEB_PASS="'"$WEB_PASS"'"/' /app/agent.env WEB_PORT="$WEB_PORT" sed -i 's/^WEB_PORT=.*/WEB_PORT="'"$WEB_PORT"'"/' /app/agent.env +NODE_NAME="$NODE_NAME" sed -i 's/^NODE_NAME=.*/NODE_NAME="'"$NODE_NAME"'"/' /app/agent.env + sha256sum /app/node-agent > /app/node-agent.sum sha256sum /host-path/node-agent >> /app/node-agent.sum rsync -avz /app/ /host-path/ -cp /app/node-agent.service /host-systemd/node-agent.service +cp /app/node-agent.service /host-systemd/node-agent.service