fix: Add failure threshold to graceful shutdown

kubernetes · Nov 25, 2024 · 04b44eb · 04b44eb
1 parent 2ec8a55
commit 04b44eb
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go
@@ -400,12 +400,18 @@ func (n *NGINXController) stopWait() {
 	var scraper collectors.NginxStatusScraper
 	lastRequests := 0
 	noChangeTimes := 0
+	failures := 0
+	const failureThreshold = 5
 
 	for ; ; time.Sleep(checkFrequency) {
 		st, err := scraper.Scrape()
 		if err != nil {
 			klog.Warningf("failed to scrape nginx status: %v", err)
-			noChangeTimes = 0
+			failures++
+			if failures >= failureThreshold {
+				klog.Warningf("giving up graceful shutdown: too many nginx status scrape failures")
+				break
+			}
 			continue
 		}