fix(shutdown): stop querying pending/terminating neighbors

prevents false positive path_error when checks are made to pending or terminating pods Signed-off-by: Clément Nussbaumer <[email protected]>
postfinance · Dec 12, 2023 · 3d6050c · 3d6050c
1 parent 65ee3ec
commit 3d6050c
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 4 deletions.
diff --git a/internal/kubediscovery/kubediscovery.go b/internal/kubediscovery/kubediscovery.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"k8s.io/client-go/kubernetes"
@@ -35,7 +36,8 @@ type Neighbour struct {
 	HostIP          string
 	NodeName        string
 	NodeSchedulable NodeSchedulability
-	Phase           string // Pod Phase
+	Phase           v1.PodPhase
+	Terminating     bool
 }
 
 // New creates a new kubediscovery client. The context is used to stop the k8s watchers/informers.
@@ -92,8 +94,9 @@ func (c *Client) GetNeighbours(ctx context.Context, namespace, labelSelector str
 			PodName:         pod.Name,
 			PodIP:           pod.Status.PodIP,
 			HostIP:          pod.Status.HostIP,
-			Phase:           string(pod.Status.Phase),
+			Phase:           pod.Status.Phase,
 			NodeName:        pod.Spec.NodeName,
+			Terminating:     pod.DeletionTimestamp != nil,
 			NodeSchedulable: sched,
 		}
 		neighbours[idx] = n

diff --git a/internal/servicecheck/servicecheck.go b/internal/servicecheck/servicecheck.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/postfinance/kubenurse/internal/kubediscovery"
 	"github.com/prometheus/client_golang/prometheus"
+	v1 "k8s.io/api/core/v1"
 )
 
 const (
@@ -186,8 +187,10 @@ func (c *Checker) MeService() (string, error) {
 // which are not schedulable are excluded from this check to avoid possible false errors.
 func (c *Checker) checkNeighbours(nh []kubediscovery.Neighbour) {
 	for _, neighbour := range nh {
-		neighbour := neighbour // pin
-		if c.allowUnschedulable || neighbour.NodeSchedulable == kubediscovery.NodeSchedulable {
+		neighbour := neighbour                 // pin
+		if neighbour.Phase == v1.PodRunning && // only query running pods (excludes pending ones)
+			!neighbour.Terminating && // exclude terminating pods
+			(c.allowUnschedulable || neighbour.NodeSchedulable == kubediscovery.NodeSchedulable) {
 			check := func() (string, error) {
 				if c.UseTLS {
 					return c.doRequest("https://" + neighbour.PodIP + ":8443/alwayshappy")