[bitnami/redis] Improve sentinel prestop hook to prevent service inte…

…rruption (#6080) * Wait until failover finishes during master pod shutdown This improves on #5528 by checking and waiting until the failover is finished on both the redis and the sentinel container. This completely eliminates momentary service interruption during rollouts. As we cannot guarantee the failover will be successful the wait time is capped by the termination grace period - 10s. * Separate terminationGracePeriod setings for each pod type * make the use of REDISCLI_AUTH clear * [bitnami/redis] Update components versions Signed-off-by: Bitnami Containers <[email protected]> Co-authored-by: Bitnami Containers <[email protected]>
bitnami · Apr 23, 2021 · 943c301 · 943c301
1 parent d1e367b
commit 943c301
Show file tree

Hide file tree

Showing 7 changed files with 83 additions and 17 deletions.
diff --git a/bitnami/redis/Chart.yaml b/bitnami/redis/Chart.yaml
@@ -25,4 +25,4 @@ name: redis
 sources:
   - https://github.com/bitnami/bitnami-docker-redis
   - http://redis.io/
-version: 14.0.2
+version: 14.1.0
diff --git a/bitnami/redis/README.md b/bitnami/redis/README.md
@@ -180,6 +180,7 @@ The command removes all the Kubernetes components associated with the chart and
 | `master.service.loadBalancerIP`             | Redis(TM) master service Load Balancer IP                                                        | `nil`           |
 | `master.service.loadBalancerSourceRanges`   | Redis(TM) master service Load Balancer sources                                                   | `[]`            |
 | `master.service.annotations`                | Additional custom annotations for Redis(TM) master service                                       | `{}`            |
+| `master.terminationGracePeriodSeconds`      | Integer setting the termination grace period for the redis-master pods                           | `30`            |
 
 
 ### Redis(TM) replicas configuration parameters
@@ -254,6 +255,7 @@ The command removes all the Kubernetes components associated with the chart and
 | `replica.service.loadBalancerIP`             | Redis(TM) replicas service Load Balancer IP                                                       | `nil`           |
 | `replica.service.loadBalancerSourceRanges`   | Redis(TM) replicas service Load Balancer sources                                                  | `[]`            |
 | `replica.service.annotations`                | Additional custom annotations for Redis(TM) replicas service                                      | `{}`            |
+| `replica.terminationGracePeriodSeconds`      | Integer setting the termination grace period for the redis-replicas pods                          | `30`            |
 
 
 ### Redis(TM) Sentinel configuration parameters
@@ -310,6 +312,7 @@ The command removes all the Kubernetes components associated with the chart and
 | `sentinel.service.loadBalancerIP`             | Redis(TM) Sentinel service Load Balancer IP                                                      | `nil`                    |
 | `sentinel.service.loadBalancerSourceRanges`   | Redis(TM) Sentinel service Load Balancer sources                                                 | `[]`                     |
 | `sentinel.service.annotations`                | Additional custom annotations for Redis(TM) Sentinel service                                     | `{}`                     |
+| `sentinel.terminationGracePeriodSeconds`      | Integer setting the termination grace period for the redis-node pods                             | `30`                     |
 
 
 ### Other Parameters

diff --git a/bitnami/redis/templates/master/statefulset.yaml b/bitnami/redis/templates/master/statefulset.yaml
@@ -78,6 +78,7 @@ spec:
       {{- if .Values.master.schedulerName }}
       schedulerName: {{ .Values.master.schedulerName | quote }}
       {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.master.terminationGracePeriodSeconds }}
       containers:
         - name: redis
           image: {{ template "redis.image" . }}

diff --git a/bitnami/redis/templates/replicas/statefulset.yaml b/bitnami/redis/templates/replicas/statefulset.yaml
@@ -79,6 +79,7 @@ spec:
       {{- if .Values.replica.schedulerName }}
       schedulerName: {{ .Values.replica.schedulerName | quote }}
       {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.replica.terminationGracePeriodSeconds }}
       containers:
         - name: redis
           image: {{ template "redis.image" . }}

diff --git a/bitnami/redis/templates/scripts-configmap.yaml b/bitnami/redis/templates/scripts-configmap.yaml
@@ -281,26 +281,70 @@ data:
     #!/bin/bash
 
     . /opt/bitnami/scripts/libvalidations.sh
+    . /opt/bitnami/scripts/libos.sh
 
-    REDIS_SERVICE="{{ include "common.names.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}"
-
-    [[ -f $REDIS_PASSWORD_FILE ]] && export REDIS_PASSWORD="$(< "${REDIS_PASSWORD_FILE}")"
+    run_sentinel_command() {
+        if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then
+            redis-cli -h "$REDIS_SERVICE" -p "{{ .Values.sentinel.service.sentinelPort }}" --tls --cert "$REDIS_SENTINEL_TLS_CERT_FILE" --key "$REDIS_SENTINEL_TLS_KEY_FILE" --cacert "$REDIS_SENTINEL_TLS_CA_FILE" sentinel "$@"
+        else
+            redis-cli -h "$REDIS_SERVICE" -p "{{ .Values.sentinel.service.sentinelPort }}" sentinel "$@"
+        fi
+    }
+    failover_finished() {
+      REDIS_SENTINEL_INFO=($(run_sentinel_command get-master-addr-by-name "{{ .Values.sentinel.masterSet }}"))
+      REDIS_MASTER_HOST="${REDIS_SENTINEL_INFO[0]}"
+      [[ "$REDIS_MASTER_HOST" != "$(hostname -i)" ]]
+    }
 
-    if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then
-        sentinel_info_command="redis-cli {{- if .Values.auth.enabled }} -a ${REDIS_PASSWORD} {{- end }} -h ${REDIS_SERVICE} -p {{ .Values.sentinel.service.sentinelPort }} --tls --cert ${REDIS_SENTINEL_TLS_CERT_FILE} --key ${REDIS_SENTINEL_TLS_KEY_FILE} --cacert ${REDIS_SENTINEL_TLS_CA_FILE} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}"
+    REDIS_SERVICE="{{ include "common.names.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}"
+   
+    # redis-cli automatically consumes credentials from the REDISCLI_AUTH variable
+    [[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD"
+    [[ -f "$REDIS_PASSWORD_FILE" ]] && export REDISCLI_AUTH="$(< "${REDIS_PASSWORD_FILE}")"
+
+    if ! failover_finished; then
+        echo "I am the master pod and you are stopping me. Starting sentinel failover"
+        # if I am the master, issue a command to failover once and then wait for the failover to finish
+        run_sentinel_command failover "{{ .Values.sentinel.masterSet }}"
+        if retry_while "failover_finished" "{{ sub .Values.sentinel.terminationGracePeriodSeconds 10 }}" 1; then
+            echo "Master has been successfuly failed over to a different pod."
+            exit 0
+        else
+            echo "Master failover failed"
+            exit 1
+        fi
     else
-        sentinel_info_command="redis-cli {{- if .Values.auth.enabled }} -a ${REDIS_PASSWORD} {{- end }} -h ${REDIS_SERVICE} -p {{ .Values.sentinel.service.sentinelPort }} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}"
+        exit 0
     fi
-    REDIS_SENTINEL_INFO=($($sentinel_info_command))
-    REDIS_MASTER_HOST="${REDIS_SENTINEL_INFO[0]}"
+  prestop-redis.sh: |
+    #!/bin/bash
 
-    if [[ "$REDIS_MASTER_HOST" = "$(hostname -i)" ]]; then
-        if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then
-            redis-cli {{- if .Values.auth.enabled }} -a "$REDIS_PASSWORD" {{- end }} -h "$REDIS_SERVICE" -p {{ .Values.sentinel.service.sentinelPort }} --tls --cert "$REDIS_SENTINEL_TLS_CERT_FILE" --key "$REDIS_SENTINEL_TLS_KEY_FILE" --cacert "$REDIS_SENTINEL_TLS_CA_FILE" sentinel failover {{ .Values.sentinel.masterSet }}
+    . /opt/bitnami/scripts/libvalidations.sh
+    . /opt/bitnami/scripts/libos.sh
+
+    run_redis_command() {
+        if is_boolean_yes "$REDIS_TLS_ENABLED"; then
+            redis-cli -h 127.0.0.1 -p "$REDIS_TLS_PORT" --tls --cert "$REDIS_TLS_CERT_FILE" --key "$REDIS_TLS_KEY_FILE" --cacert "$REDIS_TLS_CA_FILE" "$@"
         else
-            redis-cli {{- if .Values.auth.enabled }} -a "$REDIS_PASSWORD" {{- end }} -h "$REDIS_SERVICE" -p {{ .Values.sentinel.service.sentinelPort }} sentinel failover {{ .Values.sentinel.masterSet }}
+            redis-cli -h 127.0.0.1 -p ${REDIS_PORT} "$@"
         fi
+    }
+    failover_finished() {
+        REDIS_ROLE=$(run_redis_command role | head -1)
+        [[ "$REDIS_ROLE" != "master" ]]
+    }
+    
+    # redis-cli automatically consumes credentials from the REDISCLI_AUTH variable
+    [[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD"
+    [[ -f "$REDIS_PASSWORD_FILE" ]] && export REDISCLI_AUTH="$(< "${REDIS_PASSWORD_FILE}")"
+
+    if ! failover_finished; then
+        echo "Waiting for sentinel to run failover for up to {{ sub .Values.sentinel.terminationGracePeriodSeconds 10 }}s"
+        retry_while "failover_finished" "{{ sub .Values.sentinel.terminationGracePeriodSeconds 10 }}" 1
+    else
+        exit 0
     fi
+
 {{- else }}
   start-master.sh: |
     #!/bin/bash

diff --git a/bitnami/redis/templates/sentinel/statefulset.yaml b/bitnami/redis/templates/sentinel/statefulset.yaml
@@ -79,6 +79,7 @@ spec:
       {{- if .Values.replica.schedulerName }}
       schedulerName: {{ .Values.replica.schedulerName | quote }}
       {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.sentinel.terminationGracePeriodSeconds }}
       containers:
         - name: redis
           image: {{ template "redis.image" . }}
@@ -227,6 +228,13 @@ spec:
             {{- if .Values.replica.extraVolumeMounts }}
             {{- include "common.tplvalues.render" ( dict "value" .Values.replica.extraVolumeMounts "context" $ ) | nindent 12 }}
             {{- end }}
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /bin/bash
+                  - -c
+                  - /opt/bitnami/scripts/start-scripts/prestop-redis.sh
         - name: sentinel
           image: {{ template "redis.sentinel.image" . }}
           imagePullPolicy: {{ .Values.sentinel.image.pullPolicy | quote }}

diff --git a/bitnami/redis/values.yaml b/bitnami/redis/values.yaml
@@ -57,7 +57,7 @@ extraDeploy: []
 image:
   registry: docker.io
   repository: bitnami/redis
-  tag: 6.2.2-debian-10-r0
+  tag: 6.2.2-debian-10-r3
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
@@ -411,6 +411,9 @@ master:
     ## @param master.service.annotations Additional custom annotations for Redis(TM) master service
     ##
     annotations: {}
+  ## @param master.terminationGracePeriodSeconds Integer setting the termination grace period for the redis-master pods
+  ##
+  terminationGracePeriodSeconds: 30
 
 ## @section Redis(TM) replicas configuration parameters
 
@@ -704,6 +707,9 @@ replica:
     ## @param replica.service.annotations Additional custom annotations for Redis(TM) replicas service
     ##
     annotations: {}
+  ## @param replica.terminationGracePeriodSeconds Integer setting the termination grace period for the redis-replicas pods
+  ##
+  terminationGracePeriodSeconds: 30
 
 ## @section Redis(TM) Sentinel configuration parameters
 
@@ -725,7 +731,7 @@ sentinel:
   image:
     registry: docker.io
     repository: bitnami/redis-sentinel
-    tag: 6.2.1-debian-10-r46
+    tag: 6.2.2-debian-10-r2
     ## Specify a imagePullPolicy
     ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
     ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
@@ -882,6 +888,9 @@ sentinel:
     ## @param sentinel.service.annotations Additional custom annotations for Redis(TM) Sentinel service
     ##
     annotations: {}
+  ## @param sentinel.terminationGracePeriodSeconds Integer setting the termination grace period for the redis-node pods
+  ##
+  terminationGracePeriodSeconds: 30
 
 ## @section Other Parameters
 
@@ -1030,7 +1039,7 @@ metrics:
   image:
     registry: docker.io
     repository: bitnami/redis-exporter
-    tag: 1.20.0-debian-10-r27
+    tag: 1.22.0-debian-10-r0
     pullPolicy: IfNotPresent
     ## Optionally specify an array of imagePullSecrets.
     ## Secrets must be manually created in the namespace.
@@ -1120,7 +1129,7 @@ metrics:
     image:
       registry: docker.io
       repository: bitnami/redis-sentinel-exporter
-      tag: 1.7.1-debian-10-r119
+      tag: 1.7.1-debian-10-r122
       pullPolicy: IfNotPresent
       ## Optionally specify an array of imagePullSecrets.
       ## Secrets must be manually created in the namespace.