add startupProbe to all provisioned containers (envoyproxy#3893)

* This ensures the readinessProbe kicks in only after the container has started * max startup time is 300s - 30 (failureThreshold) x 10 (periodSeconds). After this the container is killed and the `restartPolicy` kicks in https://kubernetes.io/docs/concepts/configuration/liveness-readiness-startup-probes/#startup-probe Fixes: envoyproxy#3511 Signed-off-by: Arko Dasgupta <[email protected]> (cherry picked from commit fa65316) Signed-off-by: Arko Dasgupta <[email protected]>
arkodg · Sep 7, 2024 · 9a3c765 · 9a3c765
1 parent 86a0e7d
commit 9a3c765
Show file tree

Hide file tree

Showing 32 changed files with 471 additions and 0 deletions.
diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go
@@ -198,6 +198,19 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
 			VolumeMounts:             expectedContainerVolumeMounts(deploymentConfig.Container),
 			TerminationMessagePolicy: corev1.TerminationMessageReadFile,
 			TerminationMessagePath:   "/dev/termination-log",
+			StartupProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					HTTPGet: &corev1.HTTPGetAction{
+						Path:   bootstrap.EnvoyReadinessPath,
+						Port:   intstr.IntOrString{Type: intstr.Int, IntVal: bootstrap.EnvoyReadinessPort},
+						Scheme: corev1.URISchemeHTTP,
+					},
+				},
+				TimeoutSeconds:   1,
+				PeriodSeconds:    10,
+				SuccessThreshold: 1,
+				FailureThreshold: 30,
+			},
 			ReadinessProbe: &corev1.Probe{
 				ProbeHandler: corev1.ProbeHandler{
 					HTTPGet: &corev1.HTTPGetAction{
@@ -231,6 +244,19 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
 			Resources:                *egv1a1.DefaultShutdownManagerContainerResourceRequirements(),
 			TerminationMessagePolicy: corev1.TerminationMessageReadFile,
 			TerminationMessagePath:   "/dev/termination-log",
+			StartupProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					HTTPGet: &corev1.HTTPGetAction{
+						Path:   envoy.ShutdownManagerHealthCheckPath,
+						Port:   intstr.IntOrString{Type: intstr.Int, IntVal: envoy.ShutdownManagerPort},
+						Scheme: corev1.URISchemeHTTP,
+					},
+				},
+				TimeoutSeconds:   1,
+				PeriodSeconds:    10,
+				SuccessThreshold: 1,
+				FailureThreshold: 30,
+			},
 			ReadinessProbe: &corev1.Probe{
 				ProbeHandler: corev1.ProbeHandler{
 					HTTPGet: &corev1.HTTPGetAction{

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml
@@ -89,6 +89,15 @@ spec:
           requests:
             cpu: 100m
             memory: 512Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -145,6 +154,15 @@ spec:
           requests:
             cpu: 10m
             memory: 32Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirst

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml
@@ -90,6 +90,15 @@ spec:
           requests:
             cpu: 100m
             memory: 512Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -146,6 +155,15 @@ spec:
           requests:
             cpu: 10m
             memory: 32Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirst

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml
@@ -220,6 +220,15 @@ spec:
             memory: 1Gi
         securityContext:
           privileged: true
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -276,6 +285,15 @@ spec:
           requests:
             cpu: 10m
             memory: 32Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirst

diff --git a/...rnal/infrastructure/kubernetes/proxy/testdata/deployments/custom_with_initcontainers.yaml b/...rnal/infrastructure/kubernetes/proxy/testdata/deployments/custom_with_initcontainers.yaml
@@ -220,6 +220,15 @@ spec:
             memory: 1Gi
         securityContext:
           privileged: true
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -278,6 +287,15 @@ spec:
           requests:
             cpu: 10m
             memory: 32Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirst

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml
@@ -218,6 +218,15 @@ spec:
             memory: 1Gi
         securityContext:
           privileged: true
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -274,6 +283,15 @@ spec:
           requests:
             cpu: 10m
             memory: 32Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirst

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml
@@ -212,6 +212,15 @@ spec:
           requests:
             cpu: 100m
             memory: 512Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -268,6 +277,15 @@ spec:
           requests:
             cpu: 10m
             memory: 32Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirst

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/disable-prometheus.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/disable-prometheus.yaml
@@ -183,6 +183,15 @@ spec:
           requests:
             cpu: 100m
             memory: 512Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -239,6 +248,15 @@ spec:
           requests:
             cpu: 10m
             memory: 32Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirst

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml
@@ -222,6 +222,15 @@ spec:
             memory: 1Gi
         securityContext:
           privileged: true
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -278,6 +287,15 @@ spec:
           requests:
             cpu: 10m
             memory: 32Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirst

diff --git a/...infrastructure/kubernetes/proxy/testdata/deployments/override-labels-and-annotations.yaml b/...infrastructure/kubernetes/proxy/testdata/deployments/override-labels-and-annotations.yaml
@@ -223,6 +223,15 @@ spec:
           requests:
             cpu: 100m
             memory: 512Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -279,6 +288,15 @@ spec:
           requests:
             cpu: 10m
             memory: 32Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirst

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/patch-deployment.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/patch-deployment.yaml
@@ -212,6 +212,15 @@ spec:
           requests:
             cpu: 100m
             memory: 512Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -268,6 +277,15 @@ spec:
           requests:
             cpu: 10m
             memory: 32Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirstWithHostNet

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml
@@ -213,6 +213,15 @@ spec:
           requests:
             cpu: 100m
             memory: 512Mi
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /ready
+            port: 19001
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
@@ -281,6 +290,15 @@ spec:
             memory: 64Mi
         securityContext:
           runAsUser: 1234
+        startupProbe:
+          failureThreshold: 30
+          httpGet:
+            path: /healthz
+            port: 19002
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
       dnsPolicy: ClusterFirst