From 943c301359dfba1b028ebd7e37dee7428293e330 Mon Sep 17 00:00:00 2001 From: Petr Gregor <342731+Gregy@users.noreply.github.com> Date: Fri, 23 Apr 2021 11:11:31 +0200 Subject: [PATCH] [bitnami/redis] Improve sentinel prestop hook to prevent service interruption (#6080) * Wait until failover finishes during master pod shutdown This improves on #5528 by checking and waiting until the failover is finished on both the redis and the sentinel container. This completely eliminates momentary service interruption during rollouts. As we cannot guarantee the failover will be successful the wait time is capped by the termination grace period - 10s. * Separate terminationGracePeriod setings for each pod type * make the use of REDISCLI_AUTH clear * [bitnami/redis] Update components versions Signed-off-by: Bitnami Containers Co-authored-by: Bitnami Containers --- bitnami/redis/Chart.yaml | 2 +- bitnami/redis/README.md | 3 + .../redis/templates/master/statefulset.yaml | 1 + .../redis/templates/replicas/statefulset.yaml | 1 + .../redis/templates/scripts-configmap.yaml | 68 +++++++++++++++---- .../redis/templates/sentinel/statefulset.yaml | 8 +++ bitnami/redis/values.yaml | 17 +++-- 7 files changed, 83 insertions(+), 17 deletions(-) diff --git a/bitnami/redis/Chart.yaml b/bitnami/redis/Chart.yaml index a8f948e97fe95f..c18bf2da0a3c97 100644 --- a/bitnami/redis/Chart.yaml +++ b/bitnami/redis/Chart.yaml @@ -25,4 +25,4 @@ name: redis sources: - https://github.com/bitnami/bitnami-docker-redis - http://redis.io/ -version: 14.0.2 +version: 14.1.0 diff --git a/bitnami/redis/README.md b/bitnami/redis/README.md index dbbaa1480bdb14..7f6ca4978de575 100644 --- a/bitnami/redis/README.md +++ b/bitnami/redis/README.md @@ -180,6 +180,7 @@ The command removes all the Kubernetes components associated with the chart and | `master.service.loadBalancerIP` | Redis(TM) master service Load Balancer IP | `nil` | | `master.service.loadBalancerSourceRanges` | Redis(TM) master service Load Balancer sources | `[]` | | `master.service.annotations` | Additional custom annotations for Redis(TM) master service | `{}` | +| `master.terminationGracePeriodSeconds` | Integer setting the termination grace period for the redis-master pods | `30` | ### Redis(TM) replicas configuration parameters @@ -254,6 +255,7 @@ The command removes all the Kubernetes components associated with the chart and | `replica.service.loadBalancerIP` | Redis(TM) replicas service Load Balancer IP | `nil` | | `replica.service.loadBalancerSourceRanges` | Redis(TM) replicas service Load Balancer sources | `[]` | | `replica.service.annotations` | Additional custom annotations for Redis(TM) replicas service | `{}` | +| `replica.terminationGracePeriodSeconds` | Integer setting the termination grace period for the redis-replicas pods | `30` | ### Redis(TM) Sentinel configuration parameters @@ -310,6 +312,7 @@ The command removes all the Kubernetes components associated with the chart and | `sentinel.service.loadBalancerIP` | Redis(TM) Sentinel service Load Balancer IP | `nil` | | `sentinel.service.loadBalancerSourceRanges` | Redis(TM) Sentinel service Load Balancer sources | `[]` | | `sentinel.service.annotations` | Additional custom annotations for Redis(TM) Sentinel service | `{}` | +| `sentinel.terminationGracePeriodSeconds` | Integer setting the termination grace period for the redis-node pods | `30` | ### Other Parameters diff --git a/bitnami/redis/templates/master/statefulset.yaml b/bitnami/redis/templates/master/statefulset.yaml index 2834fd0644a4af..8ea35bd514bb3b 100644 --- a/bitnami/redis/templates/master/statefulset.yaml +++ b/bitnami/redis/templates/master/statefulset.yaml @@ -78,6 +78,7 @@ spec: {{- if .Values.master.schedulerName }} schedulerName: {{ .Values.master.schedulerName | quote }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.master.terminationGracePeriodSeconds }} containers: - name: redis image: {{ template "redis.image" . }} diff --git a/bitnami/redis/templates/replicas/statefulset.yaml b/bitnami/redis/templates/replicas/statefulset.yaml index 27a67830d41296..4fd3029fb13b48 100644 --- a/bitnami/redis/templates/replicas/statefulset.yaml +++ b/bitnami/redis/templates/replicas/statefulset.yaml @@ -79,6 +79,7 @@ spec: {{- if .Values.replica.schedulerName }} schedulerName: {{ .Values.replica.schedulerName | quote }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.replica.terminationGracePeriodSeconds }} containers: - name: redis image: {{ template "redis.image" . }} diff --git a/bitnami/redis/templates/scripts-configmap.yaml b/bitnami/redis/templates/scripts-configmap.yaml index 56f8661ad4c97e..7e98d68ed5f099 100644 --- a/bitnami/redis/templates/scripts-configmap.yaml +++ b/bitnami/redis/templates/scripts-configmap.yaml @@ -281,26 +281,70 @@ data: #!/bin/bash . /opt/bitnami/scripts/libvalidations.sh + . /opt/bitnami/scripts/libos.sh - REDIS_SERVICE="{{ include "common.names.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" - - [[ -f $REDIS_PASSWORD_FILE ]] && export REDIS_PASSWORD="$(< "${REDIS_PASSWORD_FILE}")" + run_sentinel_command() { + if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then + redis-cli -h "$REDIS_SERVICE" -p "{{ .Values.sentinel.service.sentinelPort }}" --tls --cert "$REDIS_SENTINEL_TLS_CERT_FILE" --key "$REDIS_SENTINEL_TLS_KEY_FILE" --cacert "$REDIS_SENTINEL_TLS_CA_FILE" sentinel "$@" + else + redis-cli -h "$REDIS_SERVICE" -p "{{ .Values.sentinel.service.sentinelPort }}" sentinel "$@" + fi + } + failover_finished() { + REDIS_SENTINEL_INFO=($(run_sentinel_command get-master-addr-by-name "{{ .Values.sentinel.masterSet }}")) + REDIS_MASTER_HOST="${REDIS_SENTINEL_INFO[0]}" + [[ "$REDIS_MASTER_HOST" != "$(hostname -i)" ]] + } - if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then - sentinel_info_command="redis-cli {{- if .Values.auth.enabled }} -a ${REDIS_PASSWORD} {{- end }} -h ${REDIS_SERVICE} -p {{ .Values.sentinel.service.sentinelPort }} --tls --cert ${REDIS_SENTINEL_TLS_CERT_FILE} --key ${REDIS_SENTINEL_TLS_KEY_FILE} --cacert ${REDIS_SENTINEL_TLS_CA_FILE} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + REDIS_SERVICE="{{ include "common.names.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" + + # redis-cli automatically consumes credentials from the REDISCLI_AUTH variable + [[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD" + [[ -f "$REDIS_PASSWORD_FILE" ]] && export REDISCLI_AUTH="$(< "${REDIS_PASSWORD_FILE}")" + + if ! failover_finished; then + echo "I am the master pod and you are stopping me. Starting sentinel failover" + # if I am the master, issue a command to failover once and then wait for the failover to finish + run_sentinel_command failover "{{ .Values.sentinel.masterSet }}" + if retry_while "failover_finished" "{{ sub .Values.sentinel.terminationGracePeriodSeconds 10 }}" 1; then + echo "Master has been successfuly failed over to a different pod." + exit 0 + else + echo "Master failover failed" + exit 1 + fi else - sentinel_info_command="redis-cli {{- if .Values.auth.enabled }} -a ${REDIS_PASSWORD} {{- end }} -h ${REDIS_SERVICE} -p {{ .Values.sentinel.service.sentinelPort }} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + exit 0 fi - REDIS_SENTINEL_INFO=($($sentinel_info_command)) - REDIS_MASTER_HOST="${REDIS_SENTINEL_INFO[0]}" + prestop-redis.sh: | + #!/bin/bash - if [[ "$REDIS_MASTER_HOST" = "$(hostname -i)" ]]; then - if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then - redis-cli {{- if .Values.auth.enabled }} -a "$REDIS_PASSWORD" {{- end }} -h "$REDIS_SERVICE" -p {{ .Values.sentinel.service.sentinelPort }} --tls --cert "$REDIS_SENTINEL_TLS_CERT_FILE" --key "$REDIS_SENTINEL_TLS_KEY_FILE" --cacert "$REDIS_SENTINEL_TLS_CA_FILE" sentinel failover {{ .Values.sentinel.masterSet }} + . /opt/bitnami/scripts/libvalidations.sh + . /opt/bitnami/scripts/libos.sh + + run_redis_command() { + if is_boolean_yes "$REDIS_TLS_ENABLED"; then + redis-cli -h 127.0.0.1 -p "$REDIS_TLS_PORT" --tls --cert "$REDIS_TLS_CERT_FILE" --key "$REDIS_TLS_KEY_FILE" --cacert "$REDIS_TLS_CA_FILE" "$@" else - redis-cli {{- if .Values.auth.enabled }} -a "$REDIS_PASSWORD" {{- end }} -h "$REDIS_SERVICE" -p {{ .Values.sentinel.service.sentinelPort }} sentinel failover {{ .Values.sentinel.masterSet }} + redis-cli -h 127.0.0.1 -p ${REDIS_PORT} "$@" fi + } + failover_finished() { + REDIS_ROLE=$(run_redis_command role | head -1) + [[ "$REDIS_ROLE" != "master" ]] + } + + # redis-cli automatically consumes credentials from the REDISCLI_AUTH variable + [[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD" + [[ -f "$REDIS_PASSWORD_FILE" ]] && export REDISCLI_AUTH="$(< "${REDIS_PASSWORD_FILE}")" + + if ! failover_finished; then + echo "Waiting for sentinel to run failover for up to {{ sub .Values.sentinel.terminationGracePeriodSeconds 10 }}s" + retry_while "failover_finished" "{{ sub .Values.sentinel.terminationGracePeriodSeconds 10 }}" 1 + else + exit 0 fi + {{- else }} start-master.sh: | #!/bin/bash diff --git a/bitnami/redis/templates/sentinel/statefulset.yaml b/bitnami/redis/templates/sentinel/statefulset.yaml index 36144a48f0d5b9..e299753f0416a9 100644 --- a/bitnami/redis/templates/sentinel/statefulset.yaml +++ b/bitnami/redis/templates/sentinel/statefulset.yaml @@ -79,6 +79,7 @@ spec: {{- if .Values.replica.schedulerName }} schedulerName: {{ .Values.replica.schedulerName | quote }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.sentinel.terminationGracePeriodSeconds }} containers: - name: redis image: {{ template "redis.image" . }} @@ -227,6 +228,13 @@ spec: {{- if .Values.replica.extraVolumeMounts }} {{- include "common.tplvalues.render" ( dict "value" .Values.replica.extraVolumeMounts "context" $ ) | nindent 12 }} {{- end }} + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - /opt/bitnami/scripts/start-scripts/prestop-redis.sh - name: sentinel image: {{ template "redis.sentinel.image" . }} imagePullPolicy: {{ .Values.sentinel.image.pullPolicy | quote }} diff --git a/bitnami/redis/values.yaml b/bitnami/redis/values.yaml index 7febb39bf96323..8e83091765cd59 100644 --- a/bitnami/redis/values.yaml +++ b/bitnami/redis/values.yaml @@ -57,7 +57,7 @@ extraDeploy: [] image: registry: docker.io repository: bitnami/redis - tag: 6.2.2-debian-10-r0 + tag: 6.2.2-debian-10-r3 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -411,6 +411,9 @@ master: ## @param master.service.annotations Additional custom annotations for Redis(TM) master service ## annotations: {} + ## @param master.terminationGracePeriodSeconds Integer setting the termination grace period for the redis-master pods + ## + terminationGracePeriodSeconds: 30 ## @section Redis(TM) replicas configuration parameters @@ -704,6 +707,9 @@ replica: ## @param replica.service.annotations Additional custom annotations for Redis(TM) replicas service ## annotations: {} + ## @param replica.terminationGracePeriodSeconds Integer setting the termination grace period for the redis-replicas pods + ## + terminationGracePeriodSeconds: 30 ## @section Redis(TM) Sentinel configuration parameters @@ -725,7 +731,7 @@ sentinel: image: registry: docker.io repository: bitnami/redis-sentinel - tag: 6.2.1-debian-10-r46 + tag: 6.2.2-debian-10-r2 ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images @@ -882,6 +888,9 @@ sentinel: ## @param sentinel.service.annotations Additional custom annotations for Redis(TM) Sentinel service ## annotations: {} + ## @param sentinel.terminationGracePeriodSeconds Integer setting the termination grace period for the redis-node pods + ## + terminationGracePeriodSeconds: 30 ## @section Other Parameters @@ -1030,7 +1039,7 @@ metrics: image: registry: docker.io repository: bitnami/redis-exporter - tag: 1.20.0-debian-10-r27 + tag: 1.22.0-debian-10-r0 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. @@ -1120,7 +1129,7 @@ metrics: image: registry: docker.io repository: bitnami/redis-sentinel-exporter - tag: 1.7.1-debian-10-r119 + tag: 1.7.1-debian-10-r122 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace.