Skip to content

Commit

Permalink
Merge pull request #206 from spotahome/improve-readiness-probe
Browse files Browse the repository at this point in the history 
change readiness probe
  • Loading branch information
chusAlvarez authored Dec 3, 2019
2 parents 2659077 + c8553c4 commit de5a07e
Show file tree
Hide file tree
Showing 8 changed files with 188 additions and 14 deletions.
14 changes: 14 additions & 0 deletions mocks/operator/redisfailover/service/RedisFailoverClient.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions operator/redisfailover/ensurer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ func (w *RedisFailoverHandler) Ensure(rf *redisfailoverv1.RedisFailover, labels
if err := w.rfService.EnsureRedisShutdownConfigMap(rf, labels, or); err != nil {
return err
}
if err := w.rfService.EnsureRedisReadinessConfigMap(rf, labels, or); err != nil {
return err
}
if err := w.rfService.EnsureRedisConfigMap(rf, labels, or); err != nil {
return err
}
Expand Down
1 change: 1 addition & 0 deletions operator/redisfailover/ensurer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ func TestEnsure(t *testing.T) {
mrfs.On("EnsureSentinelConfigMap", rf, mock.Anything, mock.Anything).Once().Return(nil)
mrfs.On("EnsureRedisConfigMap", rf, mock.Anything, mock.Anything).Once().Return(nil)
mrfs.On("EnsureRedisShutdownConfigMap", rf, mock.Anything, mock.Anything).Once().Return(nil)
mrfs.On("EnsureRedisReadinessConfigMap", rf, mock.Anything, mock.Anything).Once().Return(nil)
mrfs.On("EnsureRedisStatefulset", rf, mock.Anything, mock.Anything).Once().Return(nil)
mrfs.On("EnsureSentinelDeployment", rf, mock.Anything, mock.Anything).Once().Return(nil)

Expand Down
7 changes: 7 additions & 0 deletions operator/redisfailover/service/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ type RedisFailoverClient interface {
EnsureRedisStatefulset(rFailover *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error
EnsureRedisService(rFailover *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error
EnsureRedisShutdownConfigMap(rFailover *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error
EnsureRedisReadinessConfigMap(rFailover *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error
EnsureRedisConfigMap(rFailover *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error
EnsureNotPresentRedisService(rFailover *redisfailoverv1.RedisFailover) error
}
Expand Down Expand Up @@ -100,6 +101,12 @@ func (r *RedisFailoverKubeClient) EnsureRedisShutdownConfigMap(rf *redisfailover
return nil
}

// EnsureRedisReadinessConfigMap makes sure the redis configmap with shutdown script exists
func (r *RedisFailoverKubeClient) EnsureRedisReadinessConfigMap(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error {
cm := generateRedisReadinessConfigMap(rf, labels, ownerRefs)
return r.K8SService.CreateOrUpdateConfigMap(rf.Namespace, cm)
}

// EnsureRedisService makes sure the redis statefulset exists
func (r *RedisFailoverKubeClient) EnsureRedisService(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error {
svc := generateRedisService(rf, labels, ownerRefs)
Expand Down
1 change: 1 addition & 0 deletions operator/redisfailover/service/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const (
redisConfigFileName = "redis.conf"
redisName = "r"
redisShutdownName = "r-s"
redisReadinessName = "r-readiness"
redisRoleName = "redis"
redisGroupName = "mymaster"
appLabel = "redis-failover"
Expand Down
95 changes: 81 additions & 14 deletions operator/redisfailover/service/generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"

appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
policyv1beta1 "k8s.io/api/policy/v1beta1"
"k8s.io/apimachinery/pkg/api/resource"
Expand All @@ -17,6 +18,7 @@ import (
const (
redisConfigurationVolumeName = "redis-config"
redisShutdownConfigurationVolumeName = "redis-shutdown-config"
redisReadinessVolumeName = "redis-readiness-config"
redisStorageVolumeName = "redis-data"

graceTime = 30
Expand Down Expand Up @@ -158,6 +160,58 @@ fi`
},
}
}
func generateRedisReadinessConfigMap(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) *corev1.ConfigMap {
name := GetRedisReadinessName(rf)
namespace := rf.Namespace

labels = util.MergeLabels(labels, generateSelectorLabels(redisRoleName, rf.Name))
readinessContent := `ROLE="role"
ROLE_MASTER="role:master"
ROLE_SLAVE="role:slave"
IN_SYNC="master_sync_in_progress:1"
NO_MASTER="master_host:127.0.0.1"
check_master(){
exit 0
}
check_slave(){
in_sync=$(redis-cli info replication | grep $IN_SYNC | tr -d "\r" | tr -d "\n")
no_master=$(redis-cli info replication | grep $NO_MASTER | tr -d "\r" | tr -d "\n")
if [ -z "$in_sync" ] && [ -z "$no_master" ]; then
exit 0
fi
exit 1
}
role=$(redis-cli info replication | grep $ROLE | tr -d "\r" | tr -d "\n")
case $role in
$ROLE_MASTER)
check_master
;;
$ROLE_SLAVE)
check_slave
;;
*)
echo "unespected"
exit 1
esac`

return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: labels,
OwnerReferences: ownerRefs,
},
Data: map[string]string{
"ready.sh": readinessContent,
},
}
}

func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) *appsv1.StatefulSet {
name := GetRedisName(rf)
Expand All @@ -180,8 +234,9 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri
ServiceName: name,
Replicas: &rf.Spec.Redis.Replicas,
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: "OnDelete",
Type: v1.OnDeleteStatefulSetStrategyType,
},
PodManagementPolicy: v1.ParallelPodManagement,
Selector: &metav1.LabelSelector{
MatchLabels: selectorLabels,
},
Expand All @@ -191,10 +246,10 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri
Annotations: rf.Spec.Redis.PodAnnotations,
},
Spec: corev1.PodSpec{
Affinity: getAffinity(rf.Spec.Redis.Affinity, labels),
Tolerations: rf.Spec.Redis.Tolerations,
NodeSelector: rf.Spec.Redis.NodeSelector,
SecurityContext: getSecurityContext(rf.Spec.Redis.SecurityContext),
Affinity: getAffinity(rf.Spec.Redis.Affinity, labels),
Tolerations: rf.Spec.Redis.Tolerations,
NodeSelector: rf.Spec.Redis.NodeSelector,
SecurityContext: getSecurityContext(rf.Spec.Redis.SecurityContext),
ImagePullSecrets: rf.Spec.Redis.ImagePullSecrets,
Containers: []corev1.Container{
{
Expand All @@ -215,11 +270,7 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri
TimeoutSeconds: 5,
Handler: corev1.Handler{
Exec: &corev1.ExecAction{
Command: []string{
"sh",
"-c",
"redis-cli -h $(hostname) ping",
},
Command: []string{"/bin/sh", "/redis-readiness/ready.sh"},
},
},
},
Expand Down Expand Up @@ -297,10 +348,10 @@ func generateSentinelDeployment(rf *redisfailoverv1.RedisFailover, labels map[st
Annotations: rf.Spec.Sentinel.PodAnnotations,
},
Spec: corev1.PodSpec{
Affinity: getAffinity(rf.Spec.Sentinel.Affinity, labels),
Tolerations: rf.Spec.Sentinel.Tolerations,
NodeSelector: rf.Spec.Sentinel.NodeSelector,
SecurityContext: getSecurityContext(rf.Spec.Sentinel.SecurityContext),
Affinity: getAffinity(rf.Spec.Sentinel.Affinity, labels),
Tolerations: rf.Spec.Sentinel.Tolerations,
NodeSelector: rf.Spec.Sentinel.NodeSelector,
SecurityContext: getSecurityContext(rf.Spec.Sentinel.SecurityContext),
ImagePullSecrets: rf.Spec.Sentinel.ImagePullSecrets,
InitContainers: []corev1.Container{
{
Expand Down Expand Up @@ -538,6 +589,10 @@ func getRedisVolumeMounts(rf *redisfailoverv1.RedisFailover) []corev1.VolumeMoun
Name: redisShutdownConfigurationVolumeName,
MountPath: "/redis-shutdown",
},
{
Name: redisReadinessVolumeName,
MountPath: "/redis-readiness",
},
{
Name: getRedisDataVolumeName(rf),
MountPath: "/data",
Expand All @@ -550,6 +605,7 @@ func getRedisVolumeMounts(rf *redisfailoverv1.RedisFailover) []corev1.VolumeMoun
func getRedisVolumes(rf *redisfailoverv1.RedisFailover) []corev1.Volume {
configMapName := GetRedisName(rf)
shutdownConfigMapName := GetRedisShutdownConfigMapName(rf)
readinessConfigMapName := GetRedisReadinessName(rf)

executeMode := int32(0744)
volumes := []corev1.Volume{
Expand All @@ -574,6 +630,17 @@ func getRedisVolumes(rf *redisfailoverv1.RedisFailover) []corev1.Volume {
},
},
},
{
Name: redisReadinessVolumeName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: readinessConfigMapName,
},
DefaultMode: &executeMode,
},
},
},
}

dataVolume := getRedisDataVolume(rf)
Expand Down
76 changes: 76 additions & 0 deletions operator/redisfailover/service/generator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
func TestRedisStatefulSetStorageGeneration(t *testing.T) {
configMapName := rfservice.GetRedisName(generateRF())
shutdownConfigMapName := rfservice.GetRedisShutdownConfigMapName(generateRF())
readinesConfigMapName := rfservice.GetRedisReadinessName(generateRF())
executeMode := int32(0744)
tests := []struct {
name string
Expand All @@ -43,6 +44,10 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) {
Name: "redis-shutdown-config",
MountPath: "/redis-shutdown",
},
{
Name: "redis-readiness-config",
MountPath: "/redis-readiness",
},
{
Name: "redis-data",
MountPath: "/data",
Expand Down Expand Up @@ -72,6 +77,17 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) {
},
},
},
{
Name: "redis-readiness-config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: readinesConfigMapName,
},
DefaultMode: &executeMode,
},
},
},
{
Name: "redis-data",
VolumeSource: corev1.VolumeSource{
Expand Down Expand Up @@ -102,6 +118,10 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) {
Name: "redis-shutdown-config",
MountPath: "/redis-shutdown",
},
{
Name: "redis-readiness-config",
MountPath: "/redis-readiness",
},
{
Name: "redis-data",
MountPath: "/data",
Expand Down Expand Up @@ -131,6 +151,17 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) {
},
},
},
{
Name: "redis-readiness-config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: readinesConfigMapName,
},
DefaultMode: &executeMode,
},
},
},
{
Name: "redis-data",
VolumeSource: corev1.VolumeSource{
Expand Down Expand Up @@ -167,6 +198,10 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) {
Name: "redis-shutdown-config",
MountPath: "/redis-shutdown",
},
{
Name: "redis-readiness-config",
MountPath: "/redis-readiness",
},
{
Name: "pvc-data",
MountPath: "/data",
Expand Down Expand Up @@ -196,6 +231,17 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) {
},
},
},
{
Name: "redis-readiness-config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: readinesConfigMapName,
},
DefaultMode: &executeMode,
},
},
},
},
},
},
Expand Down Expand Up @@ -258,6 +304,10 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) {
Name: "redis-shutdown-config",
MountPath: "/redis-shutdown",
},
{
Name: "redis-readiness-config",
MountPath: "/redis-readiness",
},
{
Name: "pvc-data",
MountPath: "/data",
Expand Down Expand Up @@ -287,6 +337,17 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) {
},
},
},
{
Name: "redis-readiness-config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: readinesConfigMapName,
},
DefaultMode: &executeMode,
},
},
},
},
},
},
Expand Down Expand Up @@ -354,6 +415,10 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) {
Name: "redis-shutdown-config",
MountPath: "/redis-shutdown",
},
{
Name: "redis-readiness-config",
MountPath: "/redis-readiness",
},
{
Name: "pvc-data",
MountPath: "/data",
Expand Down Expand Up @@ -383,6 +448,17 @@ func TestRedisStatefulSetStorageGeneration(t *testing.T) {
},
},
},
{
Name: "redis-readiness-config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: readinesConfigMapName,
},
DefaultMode: &executeMode,
},
},
},
},
},
},
Expand Down
5 changes: 5 additions & 0 deletions operator/redisfailover/service/names.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@ func GetRedisShutdownName(rf *redisfailoverv1.RedisFailover) string {
return generateName(redisShutdownName, rf.Name)
}

// GetRedisReadinessName returns the name for redis resources
func GetRedisReadinessName(rf *redisfailoverv1.RedisFailover) string {
return generateName(redisReadinessName, rf.Name)
}

// GetSentinelName returns the name for sentinel resources
func GetSentinelName(rf *redisfailoverv1.RedisFailover) string {
return generateName(sentinelName, rf.Name)
Expand Down

0 comments on commit de5a07e

Please sign in to comment.