From 6816b1442ed6193f5baf2b3ea04a717cff5a3816 Mon Sep 17 00:00:00 2001 From: Samuel Vijaykumar M Date: Sun, 18 Dec 2022 15:05:09 +0530 Subject: [PATCH 1/4] WIP: Startup Probe changes --- api/redisfailover/v1/types.go | 2 + ...atabases.spotahome.com_redisfailovers.yaml | 4 ++ ...atabases.spotahome.com_redisfailovers.yaml | 4 ++ operator/redisfailover/service/generator.go | 42 ++++++++++++++++++- 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/api/redisfailover/v1/types.go b/api/redisfailover/v1/types.go index 38b81f656..77ef79443 100644 --- a/api/redisfailover/v1/types.go +++ b/api/redisfailover/v1/types.go @@ -46,6 +46,7 @@ type RedisSettings struct { CustomCommandRenames []RedisCommandRename `json:"customCommandRenames,omitempty"` Command []string `json:"command,omitempty"` ShutdownConfigMap string `json:"shutdownConfigMap,omitempty"` + StartupConfigMap string `json:"startupConfigMap,omitempty"` Storage RedisStorage `json:"storage,omitempty"` InitContainers []corev1.Container `json:"initContainers,omitempty"` Exporter Exporter `json:"exporter,omitempty"` @@ -76,6 +77,7 @@ type SentinelSettings struct { Resources corev1.ResourceRequirements `json:"resources,omitempty"` CustomConfig []string `json:"customConfig,omitempty"` Command []string `json:"command,omitempty"` + StartupConfigMap string `json:"startupConfigMap,omitempty"` Affinity *corev1.Affinity `json:"affinity,omitempty"` SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"` diff --git a/manifests/databases.spotahome.com_redisfailovers.yaml b/manifests/databases.spotahome.com_redisfailovers.yaml index 4739b1265..5ff22b87a 100644 --- a/manifests/databases.spotahome.com_redisfailovers.yaml +++ b/manifests/databases.spotahome.com_redisfailovers.yaml @@ -5785,6 +5785,8 @@ spec: type: object shutdownConfigMap: type: string + startupConfigMap: + type: string storage: description: RedisStorage defines the structure used to store the Redis Data @@ -12205,6 +12207,8 @@ spec: additionalProperties: type: string type: object + startupConfigMap: + type: string tolerations: items: description: The pod this Toleration is attached to tolerates diff --git a/manifests/kustomize/base/databases.spotahome.com_redisfailovers.yaml b/manifests/kustomize/base/databases.spotahome.com_redisfailovers.yaml index 4739b1265..5ff22b87a 100644 --- a/manifests/kustomize/base/databases.spotahome.com_redisfailovers.yaml +++ b/manifests/kustomize/base/databases.spotahome.com_redisfailovers.yaml @@ -5785,6 +5785,8 @@ spec: type: object shutdownConfigMap: type: string + startupConfigMap: + type: string storage: description: RedisStorage defines the structure used to store the Redis Data @@ -12205,6 +12207,8 @@ spec: additionalProperties: type: string type: object + startupConfigMap: + type: string tolerations: items: description: The pod this Toleration is attached to tolerates diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index 617af6b4c..8a740ab07 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -37,6 +37,7 @@ sentinel failover-timeout mymaster 3000 sentinel parallel-syncs mymaster 2` redisShutdownConfigurationVolumeName = "redis-shutdown-config" + redisStartupConfigurationVolumeName = "redis-startup-config" redisReadinessVolumeName = "redis-readiness-config" redisStorageVolumeName = "redis-data" @@ -347,7 +348,7 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri Command: []string{ "sh", "-c", - fmt.Sprintf("redis-cli -h $(hostname) -p %[1]v ping --user pinger --pass pingpass --no-auth-warning", rf.Spec.Redis.Port), + fmt.Sprintf("redis-cli -h 127.0.0.1 -p %[1]v ping --user pinger --pass pingpass --no-auth-warning", rf.Spec.Redis.Port), }, }, }, @@ -392,6 +393,20 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri } } + if rf.Spec.Redis.StartupConfigMap != "" { + ss.Spec.Template.Spec.Containers[0].StartupProbe = &corev1.Probe{ + InitialDelaySeconds: graceTime, + TimeoutSeconds: 5, + FailureThreshold: 6, + PeriodSeconds: 15, + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "/redis-startup/startup.sh"}, + }, + }, + } + } + if rf.Spec.Redis.Exporter.Enabled { exporter := createRedisExporterContainer(rf) ss.Spec.Template.Spec.Containers = append(ss.Spec.Template.Spec.Containers, exporter) @@ -751,6 +766,15 @@ func getRedisVolumeMounts(rf *redisfailoverv1.RedisFailover) []corev1.VolumeMoun }, } + if rf.Spec.Redis.StartupConfigMap != "" { + startupVolumeMount := corev1.VolumeMount{ + Name: redisStartupConfigurationVolumeName, + MountPath: "/redis-startup", + } + + volumeMounts = append(volumeMounts, startupVolumeMount) + } + if rf.Spec.Redis.ExtraVolumeMounts != nil { volumeMounts = append(volumeMounts, rf.Spec.Redis.ExtraVolumeMounts...) } @@ -814,6 +838,22 @@ func getRedisVolumes(rf *redisfailoverv1.RedisFailover) []corev1.Volume { }, } + if rf.Spec.Redis.StartupConfigMap != "" { + startupVolumeName := rf.Spec.Redis.StartupConfigMap + startupVolume := corev1.Volume{ + Name: redisStartupConfigurationVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: startupVolumeName, + }, + DefaultMode: &executeMode, + }, + }, + } + volumes = append(volumes, startupVolume) + } + if rf.Spec.Redis.ExtraVolumes != nil { volumes = append(volumes, rf.Spec.Redis.ExtraVolumes...) } From 26cce75370d6c1d897ec351302654d289b230966 Mon Sep 17 00:00:00 2001 From: Samuel Vijaykumar M Date: Sun, 18 Dec 2022 15:59:13 +0530 Subject: [PATCH 2/4] revert hostname change --- operator/redisfailover/service/generator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index 8a740ab07..6127d7299 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -348,7 +348,7 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri Command: []string{ "sh", "-c", - fmt.Sprintf("redis-cli -h 127.0.0.1 -p %[1]v ping --user pinger --pass pingpass --no-auth-warning", rf.Spec.Redis.Port), + fmt.Sprintf("redis-cli -h ${hostname} -p %[1]v ping --user pinger --pass pingpass --no-auth-warning", rf.Spec.Redis.Port), }, }, }, From 5d5ea750c4398f74c553f687539a7697842c630f Mon Sep 17 00:00:00 2001 From: Samuel Vijaykumar M Date: Sun, 18 Dec 2022 16:03:05 +0530 Subject: [PATCH 3/4] Revert back changes --- operator/redisfailover/service/generator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index 6127d7299..2d15264db 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -348,7 +348,7 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri Command: []string{ "sh", "-c", - fmt.Sprintf("redis-cli -h ${hostname} -p %[1]v ping --user pinger --pass pingpass --no-auth-warning", rf.Spec.Redis.Port), + fmt.Sprintf("redis-cli -h $(hostname) -p %[1]v ping --user pinger --pass pingpass --no-auth-warning", rf.Spec.Redis.Port), }, }, }, From 2275c0311b6e1c4b3b30089934f7888d65c06d46 Mon Sep 17 00:00:00 2001 From: Samuel Vijaykumar M Date: Mon, 19 Dec 2022 18:27:40 +0530 Subject: [PATCH 4/4] Startup config for sentinels --- example/operator/custom-startup-config.yaml | 32 ++++++ operator/redisfailover/service/generator.go | 49 +++++++- .../redisfailover/service/generator_test.go | 106 ++++++++++++++++++ 3 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 example/operator/custom-startup-config.yaml diff --git a/example/operator/custom-startup-config.yaml b/example/operator/custom-startup-config.yaml new file mode 100644 index 000000000..2b1841441 --- /dev/null +++ b/example/operator/custom-startup-config.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: startup-config +data: + startup.sh: | + #!/bin/bash + redis-cli -h 127.0.0.1 -p ${REDIS_PORT} ping --user pinger --pass pingpass --no-auth-warning +--- +apiVersion: databases.spotahome.com/v1 +kind: RedisFailover +metadata: + name: redisfailover +spec: + sentinel: + replicas: 3 + resources: + requests: + cpu: 100m + limits: + memory: 100Mi + redis: + replicas: 3 + startupConfigMap: startup-config + resources: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 400m + memory: 500Mi diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index 2d15264db..3a98d8fa6 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -36,10 +36,11 @@ sentinel down-after-milliseconds mymaster 1000 sentinel failover-timeout mymaster 3000 sentinel parallel-syncs mymaster 2` - redisShutdownConfigurationVolumeName = "redis-shutdown-config" - redisStartupConfigurationVolumeName = "redis-startup-config" - redisReadinessVolumeName = "redis-readiness-config" - redisStorageVolumeName = "redis-data" + redisShutdownConfigurationVolumeName = "redis-shutdown-config" + redisStartupConfigurationVolumeName = "redis-startup-config" + redisReadinessVolumeName = "redis-readiness-config" + redisStorageVolumeName = "redis-data" + sentinelStartupConfigurationVolumeName = "sentinel-startup-config" graceTime = 30 ) @@ -550,6 +551,21 @@ func generateSentinelDeployment(rf *redisfailoverv1.RedisFailover, labels map[st }, }, } + + if rf.Spec.Sentinel.StartupConfigMap != "" { + sd.Spec.Template.Spec.Containers[0].StartupProbe = &corev1.Probe{ + InitialDelaySeconds: graceTime, + TimeoutSeconds: 5, + FailureThreshold: 6, + PeriodSeconds: 15, + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "/sentinel-startup/startup.sh"}, + }, + }, + } + } + if rf.Spec.Sentinel.Exporter.Enabled { exporter := createSentinelExporterContainer(rf) sd.Spec.Template.Spec.Containers = append(sd.Spec.Template.Spec.Containers, exporter) @@ -790,6 +806,13 @@ func getSentinelVolumeMounts(rf *redisfailoverv1.RedisFailover) []corev1.VolumeM }, } + if rf.Spec.Sentinel.StartupConfigMap != "" { + startupVolumeMount := corev1.VolumeMount{ + Name: "sentinel-startup-config", + MountPath: "/sentinel-startup", + } + volumeMounts = append(volumeMounts, startupVolumeMount) + } if rf.Spec.Sentinel.ExtraVolumeMounts != nil { volumeMounts = append(volumeMounts, rf.Spec.Sentinel.ExtraVolumeMounts...) } @@ -867,6 +890,8 @@ func getRedisVolumes(rf *redisfailoverv1.RedisFailover) []corev1.Volume { } func getSentinelVolumes(rf *redisfailoverv1.RedisFailover, configMapName string) []corev1.Volume { + executeMode := int32(0744) + volumes := []corev1.Volume{ { Name: "sentinel-config", @@ -886,6 +911,22 @@ func getSentinelVolumes(rf *redisfailoverv1.RedisFailover, configMapName string) }, } + if rf.Spec.Sentinel.StartupConfigMap != "" { + startupVolumeName := rf.Spec.Sentinel.StartupConfigMap + startupVolume := corev1.Volume{ + Name: sentinelStartupConfigurationVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: startupVolumeName, + }, + DefaultMode: &executeMode, + }, + }, + } + volumes = append(volumes, startupVolume) + } + if rf.Spec.Sentinel.ExtraVolumes != nil { volumes = append(volumes, rf.Spec.Sentinel.ExtraVolumes...) } diff --git a/operator/redisfailover/service/generator_test.go b/operator/redisfailover/service/generator_test.go index 94779bcf0..289dd999b 100644 --- a/operator/redisfailover/service/generator_test.go +++ b/operator/redisfailover/service/generator_test.go @@ -1773,3 +1773,109 @@ func TestRedisEnv(t *testing.T) { assert.Equal(test.expectedRedisEnv, env) } } + +func TestRedisStartupProbe(t *testing.T) { + mode := int32(0744) + tests := []struct { + name string + expectedVolume corev1.Volume + expectedVolumeMount corev1.VolumeMount + }{ + { + name: "startup_config", + expectedVolume: corev1.Volume{ + Name: "redis-startup-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "startup_config", + }, + DefaultMode: &mode, + }, + }, + }, + expectedVolumeMount: corev1.VolumeMount{ + Name: "redis-startup-config", + MountPath: "/redis-startup", + }, + }, + } + + for _, test := range tests { + assert := assert.New(t) + + var startupVolumes []corev1.Volume + var startupVolumeMounts []corev1.VolumeMount + + rf := generateRF() + rf.Spec.Redis.StartupConfigMap = test.name + + ms := &mK8SService.Services{} + ms.On("CreateOrUpdatePodDisruptionBudget", namespace, mock.Anything).Once().Return(nil, nil) + ms.On("CreateOrUpdateStatefulSet", namespace, mock.Anything).Once().Run(func(args mock.Arguments) { + s := args.Get(1).(*appsv1.StatefulSet) + startupVolumes = s.Spec.Template.Spec.Volumes + startupVolumeMounts = s.Spec.Template.Spec.Containers[0].VolumeMounts + }).Return(nil) + + client := rfservice.NewRedisFailoverKubeClient(ms, log.Dummy, metrics.Dummy) + err := client.EnsureRedisStatefulset(rf, nil, []metav1.OwnerReference{}) + + assert.NoError(err) + assert.Contains(startupVolumes, test.expectedVolume) + assert.Contains(startupVolumeMounts, test.expectedVolumeMount) + } +} + +func TestSentinelStartupProbe(t *testing.T) { + mode := int32(0744) + tests := []struct { + name string + expectedVolume corev1.Volume + expectedVolumeMount corev1.VolumeMount + }{ + { + name: "startup_config", + expectedVolume: corev1.Volume{ + Name: "sentinel-startup-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "startup_config", + }, + DefaultMode: &mode, + }, + }, + }, + expectedVolumeMount: corev1.VolumeMount{ + Name: "sentinel-startup-config", + MountPath: "/sentinel-startup", + }, + }, + } + + for _, test := range tests { + assert := assert.New(t) + + var startupVolumes []corev1.Volume + var startupVolumeMounts []corev1.VolumeMount + + rf := generateRF() + rf.Spec.Sentinel.StartupConfigMap = test.name + + ms := &mK8SService.Services{} + ms.On("CreateOrUpdatePodDisruptionBudget", namespace, mock.Anything).Once().Return(nil, nil) + ms.On("CreateOrUpdateDeployment", namespace, mock.Anything).Once().Run(func(args mock.Arguments) { + d := args.Get(1).(*appsv1.Deployment) + startupVolumes = d.Spec.Template.Spec.Volumes + startupVolumeMounts = d.Spec.Template.Spec.Containers[0].VolumeMounts + }).Return(nil) + + client := rfservice.NewRedisFailoverKubeClient(ms, log.Dummy, metrics.Dummy) + err := client.EnsureSentinelDeployment(rf, nil, []metav1.OwnerReference{}) + + assert.NoError(err) + assert.Contains(startupVolumes, test.expectedVolume) + assert.Contains(startupVolumeMounts, test.expectedVolumeMount) + } +}