fix(cspi): add timeout seconds in zfs cmd used for liveness probe (#1546

)

- add 120 seconds cmd timeout value for zfs command which will
  kill the cmd process if it exceeds more than 120 seconds and
  returns non-zero exit status.
- Change Probe timeout seconds to 150 seconds.
- Change PeriodSeconds interval to 60 seconds.
- add safe-to-evict to false to disable scaledown for CSP pods

Signed-off-by: prateekpandey14 <[email protected]>

cmd/cspc-operator/app/storagepool_create.go

@@ -345,13 +345,13 @@ func getPoolLivenessProbe() *corev1.Probe {
 	probe := &corev1.Probe{
 		Handler: corev1.Handler{
 			Exec: &corev1.ExecAction{
-				Command: []string{"/bin/sh", "-c", "zfs set io.openebs:livenesstimestamp=\"$(date +%s)\" cstor-$OPENEBS_IO_POOL_NAME"},
+				Command: []string{"/bin/sh", "-c", "timeout 120 zfs set io.openebs:livenesstimestamp=\"$(date +%s)\" cstor-$OPENEBS_IO_POOL_NAME"},
 			},
 		},
 		FailureThreshold:    3,
 		InitialDelaySeconds: 300,
-		PeriodSeconds:       10,
-		TimeoutSeconds:      300,
+		PeriodSeconds:       60,
+		TimeoutSeconds:      150,
 	}
 	return probe
 }

pkg/install/v1alpha1/cstor_pool.go

@@ -228,6 +228,7 @@ spec:
             prometheus.io/path: /metrics
             prometheus.io/port: "9500"
             prometheus.io/scrape: "true"
+            cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
         spec:
           serviceAccountName: {{ .Config.ServiceAccountName.value }}
           nodeSelector: