From 27d8f535c0da8226324d50c947502c14cea49169 Mon Sep 17 00:00:00 2001 From: baoyinghai_yewu Date: Sun, 29 Sep 2024 14:28:31 +0800 Subject: [PATCH] feat: add probe Signed-off-by: baoyinghai_yewu --- cmd/kubenest/node-agent/app/serve/serve.go | 5 ++ cmd/kubenest/operator/app/operator.go | 15 ++++++ deploy/virtual-cluster-operator.yml | 53 +++++++++++++++++++--- hack/k8s-in-k8s/g.env.sh | 6 +-- 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/cmd/kubenest/node-agent/app/serve/serve.go b/cmd/kubenest/node-agent/app/serve/serve.go index b7b4045f0..aed88f45a 100644 --- a/cmd/kubenest/node-agent/app/serve/serve.go +++ b/cmd/kubenest/node-agent/app/serve/serve.go @@ -69,6 +69,11 @@ func Start(addr, certFile, keyFile, user, password string) error { passwordHash := sha256.Sum256([]byte(password)) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/healthz" || r.URL.Path == "/readyz" { + w.WriteHeader(http.StatusOK) + return + } + auth := r.Header.Get("Authorization") if auth == "" { http.Error(w, "Unauthorized", http.StatusUnauthorized) diff --git a/cmd/kubenest/operator/app/operator.go b/cmd/kubenest/operator/app/operator.go index 35d38067c..a403447c8 100644 --- a/cmd/kubenest/operator/app/operator.go +++ b/cmd/kubenest/operator/app/operator.go @@ -15,6 +15,7 @@ import ( cliflag "k8s.io/component-base/cli/flag" "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/kosmos.io/kosmos/cmd/kubenest/operator/app/config" @@ -223,11 +224,25 @@ func run(ctx context.Context, config *config.Config) error { LeaderElection: config.LeaderElection.LeaderElect, LeaderElectionID: config.LeaderElection.ResourceName, LeaderElectionNamespace: config.LeaderElection.ResourceNamespace, + LivenessEndpointName: "/healthz", + ReadinessEndpointName: "/readyz", + HealthProbeBindAddress: ":8081", }) if err != nil { return fmt.Errorf("failed to build controller manager: %v", err) } + // 向 manager 注册 liveness 和 readiness 的检测 + err = mgr.AddHealthzCheck("healthz", healthz.Ping) + if err != nil { + return fmt.Errorf("failed to build healthz: %v", err) + } + + err = mgr.AddReadyzCheck("readyz", healthz.Ping) + if err != nil { + return fmt.Errorf("failed to build readyz: %v", err) + } + hostKubeClient, err := kubernetes.NewForConfig(config.RestConfig) if err != nil { return fmt.Errorf("could not create clientset: %v", err) diff --git a/deploy/virtual-cluster-operator.yml b/deploy/virtual-cluster-operator.yml index cd8a6de39..df70bef2b 100644 --- a/deploy/virtual-cluster-operator.yml +++ b/deploy/virtual-cluster-operator.yml @@ -103,11 +103,37 @@ spec: values: - virtual-cluster-operator topologyKey: kubernetes.io/hostname + initContainers: + - name: init-container + image: 172.20.30.132:7080/cmss/kosmos/virtual-cluster-operator:v0.5.2 + command: + - sh + - "-c" + - | + mkdir -p /kosmos && \ + tar -xzf /config/manifest-use.tar -C /kosmos + volumeMounts: + - name: manifest-volume + mountPath: /kosmos + - name: manifest-tar + mountPath: /config containers: - name: virtual-cluster-operator # Change a valid image address image: ghcr.io/kosmos-io/virtual-cluster-operator:__VERSION__ imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 env: - name: IMAGE_REPOSITIRY value: ghcr.io/kosmos-io @@ -150,8 +176,8 @@ spec: - name: shellscript mountPath: /etc/vc-node-dir/config.yaml subPath: config.yaml - - mountPath: /kosmos/manifest - name: components-manifest + - mountPath: /kosmos + name: manifest-volume command: - virtual-cluster-operator - --kubeconfig=/etc/virtual-cluster-operator/kubeconfig @@ -163,10 +189,11 @@ spec: - name: shellscript configMap: name: virtual-cluster-operator - - hostPath: - path: /home/vc-operator/manifest - type: DirectoryOrCreate - name: components-manifest + - name: manifest-tar + configMap: + name: manifest-tar + - name: manifest-volume + emptyDir: {} --- apiVersion: apps/v1 kind: DaemonSet @@ -216,6 +243,20 @@ spec: containers: - name: install-agent image: cis-hub-huabei-3.cmecloud.cn/node-agent/node-agent:latest + livenessProbe: + httpGet: + path: /healthz + port: 5678 + scheme: HTTPS + initialDelaySeconds: 15 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /readyz + port: 5678 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 securityContext: privileged: true # container privileged command: diff --git a/hack/k8s-in-k8s/g.env.sh b/hack/k8s-in-k8s/g.env.sh index fd1f0d64e..3ddaa807a 100644 --- a/hack/k8s-in-k8s/g.env.sh +++ b/hack/k8s-in-k8s/g.env.sh @@ -15,14 +15,14 @@ PATH_KUBEADM_CONFIG=/etc/kubeadm PATH_KUBERNETES=/etc/kubernetes PATH_KUBERNETES_PKI=/etc/kubernetes/pki # name for kubelet kubeconfig file -KUBELET_KUBE_CONFIG_NAME= +KUBELET_KUBE_CONFIG_NAME=kubelet.conf ################################################## # path for kubelet PATH_KUBELET_LIB=/var/lib/kubelet # path for kubelet -PATH_KUBELET_CONF=. +PATH_KUBELET_CONF=/var/lib/kubelet # name for config file of kubelet -KUBELET_CONFIG_NAME= +KUBELET_CONFIG_NAME=config.yaml HOST_CORE_DNS=10.96.0.10 # kubeadm switch USE_KUBEADM=false