node pod probe daemon

Signed-off-by: liheng.zms <[email protected]>
openkruise · Sep 19, 2022 · 2ec676b · 2ec676b
1 parent 16277db
commit 2ec676b
Show file tree

Hide file tree

Showing 9 changed files with 1,489 additions and 0 deletions.
diff --git a/.github/workflows/e2e-1.16.yaml b/.github/workflows/e2e-1.16.yaml
@@ -85,6 +85,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   pullimages-containerrecreate:
@@ -154,6 +166,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   advanced-daemonset:
@@ -223,6 +247,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   sidecarset:
@@ -292,6 +328,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   podUnavailableBudget:
@@ -419,4 +467,16 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
diff --git a/.github/workflows/e2e-1.24.yaml b/.github/workflows/e2e-1.24.yaml
@@ -154,6 +154,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   advanced-daemonset:
@@ -223,6 +235,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   sidecarset:
@@ -292,6 +316,18 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
 
   ephemeraljob:
@@ -477,4 +513,16 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
           exit $retVal
diff --git a/config/rbac/daemon_role.yaml b/config/rbac/daemon_role.yaml
@@ -72,3 +72,19 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - apps.kruise.io
+  resources:
+  - nodepodprobes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps.kruise.io
+  resources:
+  - nodepodprobes/status
+  verbs:
+  - get
+  - patch
+  - update
diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go
@@ -23,6 +23,8 @@ import (
 	"net/http"
 	"sync"
 
+	"github.com/openkruise/kruise/pkg/daemon/podprobe"
+
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -147,9 +149,16 @@ func NewDaemon(cfg *rest.Config, bindAddress string) (Daemon, error) {
 		return nil, fmt.Errorf("failed to new crr daemon controller: %v", err)
 	}
 
+	// node pod probe
+	nppController, err := podprobe.NewController(opts)
+	if err != nil {
+		return nil, fmt.Errorf("failed to new nodePodProbe daemon controller: %v", err)
+	}
+
 	var runnables = []Runnable{
 		puller,
 		crrController,
+		nppController,
 	}
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.DaemonWatchingPod) {