From 8cf8d8c5c72c6ac00320708b363d66109f8c571d Mon Sep 17 00:00:00 2001 From: Chus Date: Mon, 2 Dec 2019 11:04:27 +0100 Subject: [PATCH 1/2] change readiness probe --- .../service/RedisFailoverClient.go | 14 +++ operator/redisfailover/ensurer.go | 3 + operator/redisfailover/ensurer_test.go | 1 + operator/redisfailover/service/client.go | 7 ++ operator/redisfailover/service/constants.go | 1 + operator/redisfailover/service/generator.go | 91 ++++++++++++++++--- .../redisfailover/service/generator_test.go | 76 ++++++++++++++++ operator/redisfailover/service/names.go | 5 + 8 files changed, 185 insertions(+), 13 deletions(-) diff --git a/mocks/operator/redisfailover/service/RedisFailoverClient.go b/mocks/operator/redisfailover/service/RedisFailoverClient.go index 0e583b45d..a37cda906 100644 --- a/mocks/operator/redisfailover/service/RedisFailoverClient.go +++ b/mocks/operator/redisfailover/service/RedisFailoverClient.go @@ -42,6 +42,20 @@ func (_m *RedisFailoverClient) EnsureRedisConfigMap(rFailover *v1.RedisFailover, return r0 } +// EnsureRedisReadinessConfigMap provides a mock function with given fields: rFailover, labels, ownerRefs +func (_m *RedisFailoverClient) EnsureRedisReadinessConfigMap(rFailover *v1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error { + ret := _m.Called(rFailover, labels, ownerRefs) + + var r0 error + if rf, ok := ret.Get(0).(func(*v1.RedisFailover, map[string]string, []metav1.OwnerReference) error); ok { + r0 = rf(rFailover, labels, ownerRefs) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // EnsureRedisService provides a mock function with given fields: rFailover, labels, ownerRefs func (_m *RedisFailoverClient) EnsureRedisService(rFailover *v1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error { ret := _m.Called(rFailover, labels, ownerRefs) diff --git a/operator/redisfailover/ensurer.go b/operator/redisfailover/ensurer.go index 11bc90bf0..12a00fdda 100644 --- a/operator/redisfailover/ensurer.go +++ b/operator/redisfailover/ensurer.go @@ -25,6 +25,9 @@ func (w *RedisFailoverHandler) Ensure(rf *redisfailoverv1.RedisFailover, labels if err := w.rfService.EnsureRedisShutdownConfigMap(rf, labels, or); err != nil { return err } + if err := w.rfService.EnsureRedisReadinessConfigMap(rf, labels, or); err != nil { + return err + } if err := w.rfService.EnsureRedisConfigMap(rf, labels, or); err != nil { return err } diff --git a/operator/redisfailover/ensurer_test.go b/operator/redisfailover/ensurer_test.go index 453c0dc4e..175f0d4a6 100644 --- a/operator/redisfailover/ensurer_test.go +++ b/operator/redisfailover/ensurer_test.go @@ -86,6 +86,7 @@ func TestEnsure(t *testing.T) { mrfs.On("EnsureSentinelConfigMap", rf, mock.Anything, mock.Anything).Once().Return(nil) mrfs.On("EnsureRedisConfigMap", rf, mock.Anything, mock.Anything).Once().Return(nil) mrfs.On("EnsureRedisShutdownConfigMap", rf, mock.Anything, mock.Anything).Once().Return(nil) + mrfs.On("EnsureRedisReadinessConfigMap", rf, mock.Anything, mock.Anything).Once().Return(nil) mrfs.On("EnsureRedisStatefulset", rf, mock.Anything, mock.Anything).Once().Return(nil) mrfs.On("EnsureSentinelDeployment", rf, mock.Anything, mock.Anything).Once().Return(nil) diff --git a/operator/redisfailover/service/client.go b/operator/redisfailover/service/client.go index 59581efc9..1e93587c0 100644 --- a/operator/redisfailover/service/client.go +++ b/operator/redisfailover/service/client.go @@ -19,6 +19,7 @@ type RedisFailoverClient interface { EnsureRedisStatefulset(rFailover *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error EnsureRedisService(rFailover *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error EnsureRedisShutdownConfigMap(rFailover *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error + EnsureRedisReadinessConfigMap(rFailover *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error EnsureRedisConfigMap(rFailover *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error EnsureNotPresentRedisService(rFailover *redisfailoverv1.RedisFailover) error } @@ -100,6 +101,12 @@ func (r *RedisFailoverKubeClient) EnsureRedisShutdownConfigMap(rf *redisfailover return nil } +// EnsureRedisReadinessConfigMap makes sure the redis configmap with shutdown script exists +func (r *RedisFailoverKubeClient) EnsureRedisReadinessConfigMap(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error { + cm := generateRedisReadinessConfigMap(rf, labels, ownerRefs) + return r.K8SService.CreateOrUpdateConfigMap(rf.Namespace, cm) +} + // EnsureRedisService makes sure the redis statefulset exists func (r *RedisFailoverKubeClient) EnsureRedisService(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error { svc := generateRedisService(rf, labels, ownerRefs) diff --git a/operator/redisfailover/service/constants.go b/operator/redisfailover/service/constants.go index 5cb650c9f..8ff81907d 100644 --- a/operator/redisfailover/service/constants.go +++ b/operator/redisfailover/service/constants.go @@ -26,6 +26,7 @@ const ( redisConfigFileName = "redis.conf" redisName = "r" redisShutdownName = "r-s" + redisReadinessName = "r-readiness" redisRoleName = "redis" redisGroupName = "mymaster" appLabel = "redis-failover" diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index 8b919cec1..9e0d499b5 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -17,6 +17,7 @@ import ( const ( redisConfigurationVolumeName = "redis-config" redisShutdownConfigurationVolumeName = "redis-shutdown-config" + redisReadinessVolumeName = "redis-readiness-config" redisStorageVolumeName = "redis-data" graceTime = 30 @@ -158,6 +159,58 @@ fi` }, } } +func generateRedisReadinessConfigMap(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) *corev1.ConfigMap { + name := GetRedisReadinessName(rf) + namespace := rf.Namespace + + labels = util.MergeLabels(labels, generateSelectorLabels(redisRoleName, rf.Name)) + readinessContent := `ROLE="role" + ROLE_MASTER="role:master" + ROLE_SLAVE="role:slave" + IN_SYNC="master_sync_in_progress:1" + NO_MASTER="master_host:127.0.0.1" + + check_master(){ + exit 0 + } + + check_slave(){ + in_sync=$(redis-cli info replication | grep $IN_SYNC | tr -d "\r" | tr -d "\n") + no_master=$(redis-cli info replication | grep $NO_MASTER | tr -d "\r" | tr -d "\n") + + if [ -z "$in_sync" ] && [ -z "$no_master" ]; then + exit 0 + fi + + exit 1 + } + + role=$(redis-cli info replication | grep $ROLE | tr -d "\r" | tr -d "\n") + + case $role in + $ROLE_MASTER) + check_master + ;; + $ROLE_SLAVE) + check_slave + ;; + *) + echo "unespected" + exit 1 + esac` + + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + OwnerReferences: ownerRefs, + }, + Data: map[string]string{ + "ready.sh": readinessContent, + }, + } +} func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) *appsv1.StatefulSet { name := GetRedisName(rf) @@ -191,10 +244,10 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri Annotations: rf.Spec.Redis.PodAnnotations, }, Spec: corev1.PodSpec{ - Affinity: getAffinity(rf.Spec.Redis.Affinity, labels), - Tolerations: rf.Spec.Redis.Tolerations, - NodeSelector: rf.Spec.Redis.NodeSelector, - SecurityContext: getSecurityContext(rf.Spec.Redis.SecurityContext), + Affinity: getAffinity(rf.Spec.Redis.Affinity, labels), + Tolerations: rf.Spec.Redis.Tolerations, + NodeSelector: rf.Spec.Redis.NodeSelector, + SecurityContext: getSecurityContext(rf.Spec.Redis.SecurityContext), ImagePullSecrets: rf.Spec.Redis.ImagePullSecrets, Containers: []corev1.Container{ { @@ -215,11 +268,7 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri TimeoutSeconds: 5, Handler: corev1.Handler{ Exec: &corev1.ExecAction{ - Command: []string{ - "sh", - "-c", - "redis-cli -h $(hostname) ping", - }, + Command: []string{"/bin/sh", "/redis-readiness/ready.sh"}, }, }, }, @@ -297,10 +346,10 @@ func generateSentinelDeployment(rf *redisfailoverv1.RedisFailover, labels map[st Annotations: rf.Spec.Sentinel.PodAnnotations, }, Spec: corev1.PodSpec{ - Affinity: getAffinity(rf.Spec.Sentinel.Affinity, labels), - Tolerations: rf.Spec.Sentinel.Tolerations, - NodeSelector: rf.Spec.Sentinel.NodeSelector, - SecurityContext: getSecurityContext(rf.Spec.Sentinel.SecurityContext), + Affinity: getAffinity(rf.Spec.Sentinel.Affinity, labels), + Tolerations: rf.Spec.Sentinel.Tolerations, + NodeSelector: rf.Spec.Sentinel.NodeSelector, + SecurityContext: getSecurityContext(rf.Spec.Sentinel.SecurityContext), ImagePullSecrets: rf.Spec.Sentinel.ImagePullSecrets, InitContainers: []corev1.Container{ { @@ -538,6 +587,10 @@ func getRedisVolumeMounts(rf *redisfailoverv1.RedisFailover) []corev1.VolumeMoun Name: redisShutdownConfigurationVolumeName, MountPath: "/redis-shutdown", }, + { + Name: redisReadinessVolumeName, + MountPath: "/redis-readiness", + }, { Name: getRedisDataVolumeName(rf), MountPath: "/data", @@ -550,6 +603,7 @@ func getRedisVolumeMounts(rf *redisfailoverv1.RedisFailover) []corev1.VolumeMoun func getRedisVolumes(rf *redisfailoverv1.RedisFailover) []corev1.Volume { configMapName := GetRedisName(rf) shutdownConfigMapName := GetRedisShutdownConfigMapName(rf) + readinessConfigMapName := GetRedisReadinessName(rf) executeMode := int32(0744) volumes := []corev1.Volume{ @@ -574,6 +628,17 @@ func getRedisVolumes(rf *redisfailoverv1.RedisFailover) []corev1.Volume { }, }, }, + { + Name: redisReadinessVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: readinessConfigMapName, + }, + DefaultMode: &executeMode, + }, + }, + }, } dataVolume := getRedisDataVolume(rf) diff --git a/operator/redisfailover/service/generator_test.go b/operator/redisfailover/service/generator_test.go index 045e03f79..2af4852f9 100644 --- a/operator/redisfailover/service/generator_test.go +++ b/operator/redisfailover/service/generator_test.go @@ -19,6 +19,7 @@ import ( func TestRedisStatefulSetStorageGeneration(t *testing.T) { configMapName := rfservice.GetRedisName(generateRF()) shutdownConfigMapName := rfservice.GetRedisShutdownConfigMapName(generateRF()) + readinesConfigMapName := rfservice.GetRedisReadinessName(generateRF()) executeMode := int32(0744) tests := []struct { name string @@ -43,6 +44,10 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) { Name: "redis-shutdown-config", MountPath: "/redis-shutdown", }, + { + Name: "redis-readiness-config", + MountPath: "/redis-readiness", + }, { Name: "redis-data", MountPath: "/data", @@ -72,6 +77,17 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) { }, }, }, + { + Name: "redis-readiness-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: readinesConfigMapName, + }, + DefaultMode: &executeMode, + }, + }, + }, { Name: "redis-data", VolumeSource: corev1.VolumeSource{ @@ -102,6 +118,10 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) { Name: "redis-shutdown-config", MountPath: "/redis-shutdown", }, + { + Name: "redis-readiness-config", + MountPath: "/redis-readiness", + }, { Name: "redis-data", MountPath: "/data", @@ -131,6 +151,17 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) { }, }, }, + { + Name: "redis-readiness-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: readinesConfigMapName, + }, + DefaultMode: &executeMode, + }, + }, + }, { Name: "redis-data", VolumeSource: corev1.VolumeSource{ @@ -167,6 +198,10 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) { Name: "redis-shutdown-config", MountPath: "/redis-shutdown", }, + { + Name: "redis-readiness-config", + MountPath: "/redis-readiness", + }, { Name: "pvc-data", MountPath: "/data", @@ -196,6 +231,17 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) { }, }, }, + { + Name: "redis-readiness-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: readinesConfigMapName, + }, + DefaultMode: &executeMode, + }, + }, + }, }, }, }, @@ -258,6 +304,10 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) { Name: "redis-shutdown-config", MountPath: "/redis-shutdown", }, + { + Name: "redis-readiness-config", + MountPath: "/redis-readiness", + }, { Name: "pvc-data", MountPath: "/data", @@ -287,6 +337,17 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) { }, }, }, + { + Name: "redis-readiness-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: readinesConfigMapName, + }, + DefaultMode: &executeMode, + }, + }, + }, }, }, }, @@ -354,6 +415,10 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) { Name: "redis-shutdown-config", MountPath: "/redis-shutdown", }, + { + Name: "redis-readiness-config", + MountPath: "/redis-readiness", + }, { Name: "pvc-data", MountPath: "/data", @@ -383,6 +448,17 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) { }, }, }, + { + Name: "redis-readiness-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: readinesConfigMapName, + }, + DefaultMode: &executeMode, + }, + }, + }, }, }, }, diff --git a/operator/redisfailover/service/names.go b/operator/redisfailover/service/names.go index eb9a2055f..98d761dc6 100644 --- a/operator/redisfailover/service/names.go +++ b/operator/redisfailover/service/names.go @@ -24,6 +24,11 @@ func GetRedisShutdownName(rf *redisfailoverv1.RedisFailover) string { return generateName(redisShutdownName, rf.Name) } +// GetRedisReadinessName returns the name for redis resources +func GetRedisReadinessName(rf *redisfailoverv1.RedisFailover) string { + return generateName(redisReadinessName, rf.Name) +} + // GetSentinelName returns the name for sentinel resources func GetSentinelName(rf *redisfailoverv1.RedisFailover) string { return generateName(sentinelName, rf.Name) From c8553c41affa10170c7468ef59f30d5b12973e3e Mon Sep 17 00:00:00 2001 From: Chus Date: Mon, 2 Dec 2019 17:21:19 +0100 Subject: [PATCH 2/2] avoid delay on launch --- operator/redisfailover/service/generator.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index 9e0d499b5..6f8da8e39 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -4,6 +4,7 @@ import ( "fmt" appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" "k8s.io/apimachinery/pkg/api/resource" @@ -233,8 +234,9 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri ServiceName: name, Replicas: &rf.Spec.Redis.Replicas, UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ - Type: "OnDelete", + Type: v1.OnDeleteStatefulSetStrategyType, }, + PodManagementPolicy: v1.ParallelPodManagement, Selector: &metav1.LabelSelector{ MatchLabels: selectorLabels, },