From 255abfb665b96c14213c54effeaff3e9f6924f2a Mon Sep 17 00:00:00 2001 From: Chus Date: Mon, 18 Nov 2019 13:14:09 +0100 Subject: [PATCH 1/8] change update strategy --- charts/redisoperator/Chart.yaml | 2 +- .../service/RedisFailoverCheck.go | 107 +++++++ .../service/RedisFailoverHeal.go | 14 + mocks/service/redis/Client.go | 21 ++ operator/redisfailover/checker.go | 60 ++++ operator/redisfailover/checker_test.go | 293 +++++++++++++++++- operator/redisfailover/service/check.go | 104 +++++++ operator/redisfailover/service/check_test.go | 156 ++++++++++ operator/redisfailover/service/generator.go | 2 +- operator/redisfailover/service/heal.go | 7 + service/redis/client.go | 17 + 11 files changed, 780 insertions(+), 3 deletions(-) diff --git a/charts/redisoperator/Chart.yaml b/charts/redisoperator/Chart.yaml index 00bcb2fd4..b1347be7e 100644 --- a/charts/redisoperator/Chart.yaml +++ b/charts/redisoperator/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: A Helm chart for the Spotahome Redis Operator name: redisoperator -version: 3.0.0 +version: 3.1.0 diff --git a/mocks/operator/redisfailover/service/RedisFailoverCheck.go b/mocks/operator/redisfailover/service/RedisFailoverCheck.go index f118a3369..9c9be5d4a 100644 --- a/mocks/operator/redisfailover/service/RedisFailoverCheck.go +++ b/mocks/operator/redisfailover/service/RedisFailoverCheck.go @@ -43,6 +43,27 @@ func (_m *RedisFailoverCheck) CheckRedisNumber(rFailover *v1.RedisFailover) erro return r0 } +// CheckRedisSyncing provides a mock function with given fields: slaveIP, rFailover +func (_m *RedisFailoverCheck) CheckRedisSyncing(slaveIP string, rFailover *v1.RedisFailover) (bool, error) { + ret := _m.Called(slaveIP, rFailover) + + var r0 bool + if rf, ok := ret.Get(0).(func(string, *v1.RedisFailover) bool); ok { + r0 = rf(slaveIP, rFailover) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, *v1.RedisFailover) error); ok { + r1 = rf(slaveIP, rFailover) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // CheckSentinelMonitor provides a mock function with given fields: sentinel, monitor func (_m *RedisFailoverCheck) CheckSentinelMonitor(sentinel string, monitor string) error { ret := _m.Called(sentinel, monitor) @@ -162,6 +183,27 @@ func (_m *RedisFailoverCheck) GetNumberMasters(rFailover *v1.RedisFailover) (int return r0, r1 } +// GetRedisRevisionHash provides a mock function with given fields: podName, rFailover +func (_m *RedisFailoverCheck) GetRedisRevisionHash(podName string, rFailover *v1.RedisFailover) (string, error) { + ret := _m.Called(podName, rFailover) + + var r0 string + if rf, ok := ret.Get(0).(func(string, *v1.RedisFailover) string); ok { + r0 = rf(podName, rFailover) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, *v1.RedisFailover) error); ok { + r1 = rf(podName, rFailover) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetRedisesIPs provides a mock function with given fields: rFailover func (_m *RedisFailoverCheck) GetRedisesIPs(rFailover *v1.RedisFailover) ([]string, error) { ret := _m.Called(rFailover) @@ -185,6 +227,50 @@ func (_m *RedisFailoverCheck) GetRedisesIPs(rFailover *v1.RedisFailover) ([]stri return r0, r1 } +// GetRedisesMasterPod provides a mock function with given fields: rFailover +func (_m *RedisFailoverCheck) GetRedisesMasterPod(rFailover *v1.RedisFailover) (string, error) { + ret := _m.Called(rFailover) + + var r0 string + if rf, ok := ret.Get(0).(func(*v1.RedisFailover) string); ok { + r0 = rf(rFailover) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(*v1.RedisFailover) error); ok { + r1 = rf(rFailover) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetRedisesSlavesPods provides a mock function with given fields: rFailover +func (_m *RedisFailoverCheck) GetRedisesSlavesPods(rFailover *v1.RedisFailover) ([]string, error) { + ret := _m.Called(rFailover) + + var r0 []string + if rf, ok := ret.Get(0).(func(*v1.RedisFailover) []string); ok { + r0 = rf(rFailover) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*v1.RedisFailover) error); ok { + r1 = rf(rFailover) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetSentinelsIPs provides a mock function with given fields: rFailover func (_m *RedisFailoverCheck) GetSentinelsIPs(rFailover *v1.RedisFailover) ([]string, error) { ret := _m.Called(rFailover) @@ -207,3 +293,24 @@ func (_m *RedisFailoverCheck) GetSentinelsIPs(rFailover *v1.RedisFailover) ([]st return r0, r1 } + +// GetStatefulSetUpdateRevision provides a mock function with given fields: rFailover +func (_m *RedisFailoverCheck) GetStatefulSetUpdateRevision(rFailover *v1.RedisFailover) (string, error) { + ret := _m.Called(rFailover) + + var r0 string + if rf, ok := ret.Get(0).(func(*v1.RedisFailover) string); ok { + r0 = rf(rFailover) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(*v1.RedisFailover) error); ok { + r1 = rf(rFailover) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/mocks/operator/redisfailover/service/RedisFailoverHeal.go b/mocks/operator/redisfailover/service/RedisFailoverHeal.go index d719edbef..8156f5829 100644 --- a/mocks/operator/redisfailover/service/RedisFailoverHeal.go +++ b/mocks/operator/redisfailover/service/RedisFailoverHeal.go @@ -13,6 +13,20 @@ type RedisFailoverHeal struct { mock.Mock } +// DeletePod provides a mock function with given fields: podName, rFailover +func (_m *RedisFailoverHeal) DeletePod(podName string, rFailover *v1.RedisFailover) error { + ret := _m.Called(podName, rFailover) + + var r0 error + if rf, ok := ret.Get(0).(func(string, *v1.RedisFailover) error); ok { + r0 = rf(podName, rFailover) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // MakeMaster provides a mock function with given fields: ip, rFailover func (_m *RedisFailoverHeal) MakeMaster(ip string, rFailover *v1.RedisFailover) error { ret := _m.Called(ip, rFailover) diff --git a/mocks/service/redis/Client.go b/mocks/service/redis/Client.go index 290a4b269..f44cf04d2 100644 --- a/mocks/service/redis/Client.go +++ b/mocks/service/redis/Client.go @@ -114,6 +114,27 @@ func (_m *Client) IsMaster(ip string, password string) (bool, error) { return r0, r1 } +// IsSyncing provides a mock function with given fields: ip, password +func (_m *Client) IsSyncing(ip string, password string) (bool, error) { + ret := _m.Called(ip, password) + + var r0 bool + if rf, ok := ret.Get(0).(func(string, string) bool); ok { + r0 = rf(ip, password) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(ip, password) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // MakeMaster provides a mock function with given fields: ip, password func (_m *Client) MakeMaster(ip string, password string) error { ret := _m.Called(ip, password) diff --git a/operator/redisfailover/checker.go b/operator/redisfailover/checker.go index 3db05c0e4..f9b81e72a 100644 --- a/operator/redisfailover/checker.go +++ b/operator/redisfailover/checker.go @@ -11,6 +11,61 @@ const ( timeToPrepare = 2 * time.Minute ) +//Checks if the running version of pods are equal to the statefulset one +func (r *RedisFailoverHandler) UpdateRedisesPods(rf *redisfailoverv1.RedisFailover) error { + redises, err := r.rfChecker.GetRedisesIPs(rf) + if err != nil { + return err + } + + //If we have syincing nodes we finish checks + for _, rp := range redises { + sync, err := r.rfChecker.CheckRedisSyncing(rp, rf) + if err != nil { + return err + } + if sync { + //currently syncing, we wait next round + return nil + } + } + + ssUR, err := r.rfChecker.GetStatefulSetUpdateRevision(rf) + if err != nil { + return err + } + + redisesPods, err := r.rfChecker.GetRedisesSlavesPods(rf) + if err != nil { + return err + } + + //If some slaves are not in the correct version, delete them + for _, pod := range redisesPods { + revision, err := r.rfChecker.GetRedisRevisionHash(pod, rf) + if err != nil { + return err + } + if revision != ssUR { + //Delete pod and wait next round to check if the new one is synced + r.rfHealer.DeletePod(pod, rf) + return nil + } + } + + //If all slaves are up and synced, we check master + master, err := r.rfChecker.GetRedisesMasterPod(rf) + + masterRevision, err := r.rfChecker.GetRedisRevisionHash(master, rf) + if masterRevision != ssUR { + //Delete pod and wait next round to check if the new one is synced + r.rfHealer.DeletePod(master, rf) + return nil + } + + return nil +} + func (r *RedisFailoverHandler) CheckAndHeal(rf *redisfailoverv1.RedisFailover) error { // Number of redis is equal as the set on the RF spec // Number of sentinel is equal as the set on the RF spec @@ -87,6 +142,11 @@ func (r *RedisFailoverHandler) CheckAndHeal(rf *redisfailoverv1.RedisFailover) e } } + err = r.UpdateRedisesPods(rf) + if err != nil { + return err + } + sentinels, err := r.rfChecker.GetSentinelsIPs(rf) if err != nil { return err diff --git a/operator/redisfailover/checker_test.go b/operator/redisfailover/checker_test.go index fef6321ee..8c4873b89 100644 --- a/operator/redisfailover/checker_test.go +++ b/operator/redisfailover/checker_test.go @@ -8,6 +8,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/spotahome/redis-operator/log" "github.com/spotahome/redis-operator/metrics" mRFService "github.com/spotahome/redis-operator/mocks/operator/redisfailover/service" @@ -165,7 +169,12 @@ func TestCheckAndHeal(t *testing.T) { mrfc.On("CheckAllSlavesFromMaster", master, rf).Once().Return(errors.New("")) mrfh.On("SetMasterOnAll", master, rf).Once().Return(nil) } - mrfc.On("GetRedisesIPs", rf).Once().Return([]string{master}, nil) + mrfc.On("GetRedisesIPs", rf).Twice().Return([]string{master}, nil) + mrfc.On("CheckRedisSyncing", master, rf).Once().Return(false, nil) + mrfc.On("GetStatefulSetUpdateRevision", rf).Once().Return("1", nil) + mrfc.On("GetRedisesSlavesPods", rf).Once().Return([]string{}, nil) + mrfc.On("GetRedisesMasterPod", rf).Once().Return(master, nil) + mrfc.On("GetRedisRevisionHash", master, rf).Once().Return("1", nil) mrfh.On("SetRedisCustomConfig", master, rf).Once().Return(nil) mrfc.On("GetSentinelsIPs", rf).Once().Return([]string{sentinel}, nil) if test.sentinelMonitorOK { @@ -202,3 +211,285 @@ func TestCheckAndHeal(t *testing.T) { }) } } + +func TestUpdate(t *testing.T) { + type podStatus struct { + pod corev1.Pod + syncing bool + master bool + } + tests := []struct { + name string + pods []podStatus + ssVersion string + errExpected error + syncing bool + }{ + { + name: "all ok, no change needed", + pods: []podStatus{ + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slave1", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "0.0.0.0", + }, + }, + master: false, + syncing: false, + }, + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slave2", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "0.0.0.1", + }, + }, + master: false, + syncing: false, + }, + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "1.1.1.1", + }, + }, + master: true, + syncing: false, + }, + }, + ssVersion: "10", + errExpected: nil, + }, + { + name: "syncing", + pods: []podStatus{ + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slave1", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "0.0.0.0", + }, + }, + master: false, + syncing: false, + }, + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slave2", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "0.0.0.1", + }, + }, + master: false, + syncing: true, + }, + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "1.1.1.1", + }, + }, + master: true, + syncing: false, + }, + }, + ssVersion: "10", + errExpected: nil, + }, + { + name: "pod version incorrect", + pods: []podStatus{ + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slave1", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "0.0.0.0", + }, + }, + master: false, + syncing: false, + }, + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slave2", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "0.0.0.1", + }, + }, + master: false, + syncing: true, + }, + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "1.1.1.1", + }, + }, + master: true, + syncing: false, + }, + }, + ssVersion: "1", + errExpected: nil, + }, + { + name: "master version incorrect", + pods: []podStatus{ + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slave1", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "0.0.0.0", + }, + }, + master: false, + syncing: false, + }, + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slave2", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + Status: corev1.PodStatus{ + PodIP: "0.0.0.1", + }, + }, + master: false, + syncing: true, + }, + { + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "master", + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "1", + }, + }, + Status: corev1.PodStatus{ + PodIP: "1.1.1.1", + }, + }, + master: true, + syncing: false, + }, + }, + ssVersion: "10", + errExpected: nil, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert := assert.New(t) + + rf := generateRF(false) + + config := generateConfig() + mrfs := &mRFService.RedisFailoverClient{} + + mrfc := &mRFService.RedisFailoverCheck{} + mrfc.On("GetRedisesIPs", rf).Once().Return([]string{"0.0.0.0", "0.0.0.1", "1.1.1.1"}, nil) + + next := true + + for _, pod := range test.pods { + mrfc.On("CheckRedisSyncing", pod.pod.Status.PodIP, rf).Once().Return(pod.syncing, nil) + if pod.syncing { + next = false + break + } + } + mrfh := &mRFService.RedisFailoverHeal{} + + if next { + mrfc.On("GetStatefulSetUpdateRevision", rf).Once().Return(test.ssVersion, nil) + mrfc.On("GetRedisesSlavesPods", rf).Once().Return([]string{"slave1", "slave2"}, nil) + + for _, pod := range test.pods { + mrfc.On("GetRedisRevisionHash", pod.pod.ObjectMeta.Name, rf).Once().Return(pod.pod.ObjectMeta.Labels[appsv1.ControllerRevisionHashLabelKey], nil) + if pod.pod.ObjectMeta.Labels[appsv1.ControllerRevisionHashLabelKey] != test.ssVersion { + mrfh.On("DeletePod", mock.Anything, mock.Anything) + if pod.master == false { + next = false + break + } + } + } + if next { + mrfc.On("GetRedisesMasterPod", rf).Once().Return("master", nil) + } + } + + mk := &mK8SService.Services{} + + handler := rfOperator.NewRedisFailoverHandler(config, mrfs, mrfc, mrfh, mk, metrics.Dummy, log.Dummy) + err := handler.UpdateRedisesPods(rf) + + if test.errExpected != nil { + assert.Error(err) + } else { + assert.NoError(err) + } + + mrfc.AssertExpectations(t) + mrfh.AssertExpectations(t) + + }) + } +} diff --git a/operator/redisfailover/service/check.go b/operator/redisfailover/service/check.go index ab28d0f0c..1bc562ab9 100644 --- a/operator/redisfailover/service/check.go +++ b/operator/redisfailover/service/check.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" redisfailoverv1 "github.com/spotahome/redis-operator/api/redisfailover/v1" @@ -26,6 +27,11 @@ type RedisFailoverCheck interface { GetRedisesIPs(rFailover *redisfailoverv1.RedisFailover) ([]string, error) GetSentinelsIPs(rFailover *redisfailoverv1.RedisFailover) ([]string, error) GetMinimumRedisPodTime(rFailover *redisfailoverv1.RedisFailover) (time.Duration, error) + GetRedisesSlavesPods(rFailover *redisfailoverv1.RedisFailover) ([]string, error) + GetRedisesMasterPod(rFailover *redisfailoverv1.RedisFailover) (string, error) + GetStatefulSetUpdateRevision(rFailover *redisfailoverv1.RedisFailover) (string, error) + GetRedisRevisionHash(podName string, rFailover *redisfailoverv1.RedisFailover) (string, error) + CheckRedisSyncing(slaveIP string, rFailover *redisfailoverv1.RedisFailover) (bool, error) } // RedisFailoverChecker is our implementation of RedisFailoverCheck interface @@ -230,3 +236,101 @@ func (r *RedisFailoverChecker) GetMinimumRedisPodTime(rf *redisfailoverv1.RedisF } return minTime, nil } + +// GetRedisesSlavesPods returns pods names of the Redis slave nodes +func (r *RedisFailoverChecker) GetRedisesSlavesPods(rf *redisfailoverv1.RedisFailover) ([]string, error) { + redises := []string{} + rps, err := r.k8sService.GetStatefulSetPods(rf.Namespace, GetRedisName(rf)) + if err != nil { + return nil, err + } + + password, err := k8s.GetRedisPassword(r.k8sService, rf) + if err != nil { + return redises, err + } + + for _, rp := range rps.Items { + if rp.Status.Phase == corev1.PodRunning { // Only work with running + master, err := r.redisClient.IsMaster(rp.Status.PodIP, password) + if err != nil { + return []string{}, err + } + if !master { + redises = append(redises, rp.ObjectMeta.Name) + } + } + } + return redises, nil +} + +// GetRedisesMasterPod returns pods names of the Redis slave nodes +func (r *RedisFailoverChecker) GetRedisesMasterPod(rFailover *redisfailoverv1.RedisFailover) (string, error) { + rps, err := r.k8sService.GetStatefulSetPods(rFailover.Namespace, GetRedisName(rFailover)) + if err != nil { + return "", err + } + + password, err := k8s.GetRedisPassword(r.k8sService, rFailover) + if err != nil { + return "", err + } + + for _, rp := range rps.Items { + if rp.Status.Phase == corev1.PodRunning { // Only work with running + master, err := r.redisClient.IsMaster(rp.Status.PodIP, password) + if err != nil { + return "", err + } + if master { + return rp.ObjectMeta.Name, nil + } + } + } + return "", errors.New("redis nodes known as master not found") +} + +// GetStatefulSetUpdateRevision returns current version for the statefulSet +// If the label don't exists, we return an empty value and no error, so previous versions don't break +func (r *RedisFailoverChecker) GetStatefulSetUpdateRevision(rFailover *redisfailoverv1.RedisFailover) (string, error) { + ss, err := r.k8sService.GetStatefulSet(rFailover.Namespace, GetRedisName(rFailover)) + if err != nil { + return "", err + } + + if ss == nil { + return "", errors.New("statefulSet not found") + } + + return ss.Status.UpdateRevision, nil +} + +// GetRedisRevisionHash returns the statefulset uid for the pod +func (r *RedisFailoverChecker) GetRedisRevisionHash(podName string, rFailover *redisfailoverv1.RedisFailover) (string, error) { + pod, err := r.k8sService.GetPod(rFailover.Namespace, podName) + if err != nil { + return "", err + } + + if pod == nil { + return "", errors.New("pod not found") + } + + if pod.ObjectMeta.Labels == nil { + return "", errors.New("labels not found") + } + + val, _ := pod.ObjectMeta.Labels[appsv1.ControllerRevisionHashLabelKey] + + return val, nil +} + +// CheckRedisSyncing returns true if the slave is still syncing +func (r *RedisFailoverChecker) CheckRedisSyncing(ip string, rFailover *redisfailoverv1.RedisFailover) (bool, error) { + password, err := k8s.GetRedisPassword(r.k8sService, rFailover) + if err != nil { + return false, err + } + + return r.redisClient.IsSyncing(ip, password) +} diff --git a/operator/redisfailover/service/check_test.go b/operator/redisfailover/service/check_test.go index 83f03298d..741f8e34e 100644 --- a/operator/redisfailover/service/check_test.go +++ b/operator/redisfailover/service/check_test.go @@ -667,3 +667,159 @@ func TestGetMinimumRedisPodTime(t *testing.T) { expected := now.Sub(oneMinute).Round(time.Second) assert.Equal(expected, minTime.Round(time.Second), "the closest time should be given") } + +func TestGetRedisPodsNames(t *testing.T) { + assert := assert.New(t) + rf := generateRF() + + pods := &corev1.PodList{ + Items: []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "slave1", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + PodIP: "0.0.0.0", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + PodIP: "1.1.1.1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "slave2", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + PodIP: "0.0.0.0", + }, + }, + }, + } + + ms := &mK8SService.Services{} + ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) + mr := &mRedisService.Client{} + mr.On("IsMaster", "0.0.0.0", "").Twice().Return(false, nil) + mr.On("IsMaster", "1.1.1.1", "").Once().Return(true, nil) + + checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) + master, err := checker.GetRedisesMasterPod(rf) + + assert.NoError(err) + + assert.Equal(master, "master") + + ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) + mr.On("IsMaster", "0.0.0.0", "").Twice().Return(false, nil) + mr.On("IsMaster", "1.1.1.1", "").Once().Return(true, nil) + + namePods, err := checker.GetRedisesSlavesPods(rf) + + assert.NoError(err) + + assert.Equal(namePods, []string{"slave1", "slave2"}) +} + +func TestGetStatefulSetUpdateRevision(t *testing.T) { + tests := []struct { + name string + ss *appsv1.StatefulSet + expectedUVersion string + expectedError error + }{ + { + name: "revision ok", + ss: &appsv1.StatefulSet{ + Status: appsv1.StatefulSetStatus{ + UpdateRevision: "10", + }, + }, + expectedUVersion: "10", + expectedError: nil, + }, + { + name: "no stateful set", + ss: nil, + expectedUVersion: "", + expectedError: errors.New("not found"), + }, + } + + for _, test := range tests { + assert := assert.New(t) + + rf := generateRF() + ms := &mK8SService.Services{} + ms.On("GetStatefulSet", namespace, rfservice.GetRedisName(rf)).Once().Return(test.ss, nil) + mr := &mRedisService.Client{} + + checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) + version, err := checker.GetStatefulSetUpdateRevision(rf) + + if test.expectedError == nil { + assert.NoError(err) + } else { + assert.Error(err) + } + + assert.Equal(version, test.expectedUVersion) + } + +} + +func TestGetRedisRevisionHash(t *testing.T) { + tests := []struct { + name string + pod *corev1.Pod + expectedHash string + expectedError error + }{ + { + name: "has ok", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1.ControllerRevisionHashLabelKey: "10", + }, + }, + }, + expectedHash: "10", + expectedError: nil, + }, + { + name: "no pod", + pod: nil, + expectedHash: "", + expectedError: errors.New("not found"), + }, + } + + for _, test := range tests { + assert := assert.New(t) + + rf := generateRF() + ms := &mK8SService.Services{} + ms.On("GetPod", namespace, "namepod").Once().Return(test.pod, nil) + mr := &mRedisService.Client{} + + checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) + hash, err := checker.GetRedisRevisionHash("namepod", rf) + + if test.expectedError == nil { + assert.NoError(err) + } else { + assert.Error(err) + } + + assert.Equal(hash, test.expectedHash) + } + +} diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index 8defb50d8..8b919cec1 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -180,7 +180,7 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri ServiceName: name, Replicas: &rf.Spec.Redis.Replicas, UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ - Type: "RollingUpdate", + Type: "OnDelete", }, Selector: &metav1.LabelSelector{ MatchLabels: selectorLabels, diff --git a/operator/redisfailover/service/heal.go b/operator/redisfailover/service/heal.go index 88f5c13d0..0c1232349 100644 --- a/operator/redisfailover/service/heal.go +++ b/operator/redisfailover/service/heal.go @@ -20,6 +20,7 @@ type RedisFailoverHeal interface { RestoreSentinel(ip string) error SetSentinelCustomConfig(ip string, rFailover *redisfailoverv1.RedisFailover) error SetRedisCustomConfig(ip string, rFailover *redisfailoverv1.RedisFailover) error + DeletePod(podName string, rFailover *redisfailoverv1.RedisFailover) error } // RedisFailoverHealer is our implementation of RedisFailoverCheck interface @@ -149,3 +150,9 @@ func (r *RedisFailoverHealer) SetRedisCustomConfig(ip string, rf *redisfailoverv return r.redisClient.SetCustomRedisConfig(ip, rf.Spec.Redis.CustomConfig, password) } + +//DeletePod delete a failing pod so kubernetes relaunch it again +func (r *RedisFailoverHealer) DeletePod(podName string, rFailover *redisfailoverv1.RedisFailover) error { + r.logger.Debugf("Deleting pods %s...", podName) + return r.k8sService.DeletePod(rFailover.Namespace, podName) +} diff --git a/service/redis/client.go b/service/redis/client.go index 0f84fd237..44ed7f532 100644 --- a/service/redis/client.go +++ b/service/redis/client.go @@ -23,6 +23,7 @@ type Client interface { GetSentinelMonitor(ip string) (string, error) SetCustomSentinelConfig(ip string, configs []string) error SetCustomRedisConfig(ip string, configs []string, password string) error + IsSyncing(ip, password string) (bool, error) } type client struct { @@ -39,6 +40,7 @@ const ( sentinelStatusREString = "status=([a-z]+)" redisMasterHostREString = "master_host:([0-9.]+)" redisRoleMaster = "role:master" + redisSyncing = "master_sync_in_progress:1" redisPort = "6379" sentinelPort = "26379" masterName = "mymaster" @@ -301,3 +303,18 @@ func (c *client) getConfigParameters(config string) (parameter string, value str } return s[0], strings.Join(s[1:], " "), nil } + +func (c *client) IsSyncing(ip, password string) (bool, error) { + options := &rediscli.Options{ + Addr: fmt.Sprintf("%s:%s", ip, redisPort), + Password: password, + DB: 0, + } + rClient := rediscli.NewClient(options) + defer rClient.Close() + info, err := rClient.Info("replication").Result() + if err != nil { + return false, err + } + return strings.Contains(info, redisSyncing), nil +} From 8e1bb826ca0dc9f2c7bf7e11d360fc0c06fd5b6d Mon Sep 17 00:00:00 2001 From: chusAlvarez Date: Mon, 25 Nov 2019 18:49:27 +0100 Subject: [PATCH 2/8] Update operator/redisfailover/checker.go Co-Authored-By: Sergio Ballesteros --- operator/redisfailover/checker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/redisfailover/checker.go b/operator/redisfailover/checker.go index f9b81e72a..0b9ba61d0 100644 --- a/operator/redisfailover/checker.go +++ b/operator/redisfailover/checker.go @@ -18,7 +18,7 @@ func (r *RedisFailoverHandler) UpdateRedisesPods(rf *redisfailoverv1.RedisFailov return err } - //If we have syincing nodes we finish checks + // No perform updates when nodes are syncing. for _, rp := range redises { sync, err := r.rfChecker.CheckRedisSyncing(rp, rf) if err != nil { From 1ed524cd94fba051273a27a6731ec487db86792b Mon Sep 17 00:00:00 2001 From: chusAlvarez Date: Mon, 25 Nov 2019 18:49:40 +0100 Subject: [PATCH 3/8] Update operator/redisfailover/checker.go Co-Authored-By: Sergio Ballesteros --- operator/redisfailover/checker.go | 1 - 1 file changed, 1 deletion(-) diff --git a/operator/redisfailover/checker.go b/operator/redisfailover/checker.go index 0b9ba61d0..5738eb3d1 100644 --- a/operator/redisfailover/checker.go +++ b/operator/redisfailover/checker.go @@ -25,7 +25,6 @@ func (r *RedisFailoverHandler) UpdateRedisesPods(rf *redisfailoverv1.RedisFailov return err } if sync { - //currently syncing, we wait next round return nil } } From 38f049dcf8160dc20e28ff4bd13852f71cab4f3d Mon Sep 17 00:00:00 2001 From: chusAlvarez Date: Mon, 25 Nov 2019 18:49:56 +0100 Subject: [PATCH 4/8] Update operator/redisfailover/checker.go Co-Authored-By: Sergio Ballesteros --- operator/redisfailover/checker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/redisfailover/checker.go b/operator/redisfailover/checker.go index 5738eb3d1..11476c721 100644 --- a/operator/redisfailover/checker.go +++ b/operator/redisfailover/checker.go @@ -39,7 +39,7 @@ func (r *RedisFailoverHandler) UpdateRedisesPods(rf *redisfailoverv1.RedisFailov return err } - //If some slaves are not in the correct version, delete them + // Update stale pods with slave role for _, pod := range redisesPods { revision, err := r.rfChecker.GetRedisRevisionHash(pod, rf) if err != nil { From 40749f106e23bf7410bb65af2d8f03c69ee1446a Mon Sep 17 00:00:00 2001 From: chusAlvarez Date: Mon, 25 Nov 2019 18:50:03 +0100 Subject: [PATCH 5/8] Update operator/redisfailover/checker.go Co-Authored-By: Sergio Ballesteros --- operator/redisfailover/checker.go | 1 - 1 file changed, 1 deletion(-) diff --git a/operator/redisfailover/checker.go b/operator/redisfailover/checker.go index 11476c721..5f1ede78a 100644 --- a/operator/redisfailover/checker.go +++ b/operator/redisfailover/checker.go @@ -57,7 +57,6 @@ func (r *RedisFailoverHandler) UpdateRedisesPods(rf *redisfailoverv1.RedisFailov masterRevision, err := r.rfChecker.GetRedisRevisionHash(master, rf) if masterRevision != ssUR { - //Delete pod and wait next round to check if the new one is synced r.rfHealer.DeletePod(master, rf) return nil } From 6aada9cfdc81cfc1d7ccd4203a2ae3f5d5ede029 Mon Sep 17 00:00:00 2001 From: chusAlvarez Date: Mon, 25 Nov 2019 18:50:14 +0100 Subject: [PATCH 6/8] Update operator/redisfailover/checker.go Co-Authored-By: Sergio Ballesteros --- operator/redisfailover/checker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/redisfailover/checker.go b/operator/redisfailover/checker.go index 5f1ede78a..4645ab87b 100644 --- a/operator/redisfailover/checker.go +++ b/operator/redisfailover/checker.go @@ -52,7 +52,7 @@ func (r *RedisFailoverHandler) UpdateRedisesPods(rf *redisfailoverv1.RedisFailov } } - //If all slaves are up and synced, we check master + // Update stale pod with role master master, err := r.rfChecker.GetRedisesMasterPod(rf) masterRevision, err := r.rfChecker.GetRedisRevisionHash(master, rf) From 0a22fede27c8adfbb871f892920cdc4eb186837b Mon Sep 17 00:00:00 2001 From: chusAlvarez Date: Mon, 25 Nov 2019 19:13:35 +0100 Subject: [PATCH 7/8] Update operator/redisfailover/checker.go Co-Authored-By: fixmie[bot] <44270338+fixmie[bot]@users.noreply.github.com> --- operator/redisfailover/checker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/redisfailover/checker.go b/operator/redisfailover/checker.go index 4645ab87b..a83c4c5ce 100644 --- a/operator/redisfailover/checker.go +++ b/operator/redisfailover/checker.go @@ -11,7 +11,7 @@ const ( timeToPrepare = 2 * time.Minute ) -//Checks if the running version of pods are equal to the statefulset one +//UpdateRedisesPods if the running version of pods are equal to the statefulset one func (r *RedisFailoverHandler) UpdateRedisesPods(rf *redisfailoverv1.RedisFailover) error { redises, err := r.rfChecker.GetRedisesIPs(rf) if err != nil { From 694251f25907dfc6b570288c7af8b4980af39f70 Mon Sep 17 00:00:00 2001 From: Chus Date: Tue, 26 Nov 2019 17:54:42 +0100 Subject: [PATCH 8/8] more deep slaves checking when updated --- Gopkg.lock | 2 +- charts/redisoperator/Chart.yaml | 2 +- .../service/RedisFailoverCheck.go | 4 +- mocks/service/redis/Client.go | 4 +- operator/redisfailover/checker.go | 17 +++-- operator/redisfailover/checker_test.go | 70 ++++++++++--------- operator/redisfailover/service/check.go | 8 +-- service/redis/client.go | 13 +++- 8 files changed, 66 insertions(+), 54 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a6e0cb52a..6e15c959c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -668,7 +668,7 @@ "github.com/stretchr/testify/assert", "github.com/stretchr/testify/mock", "github.com/stretchr/testify/require", - "k8s.io/api/apps/v1beta2", + "k8s.io/api/apps/v1", "k8s.io/api/core/v1", "k8s.io/api/policy/v1beta1", "k8s.io/api/rbac/v1", diff --git a/charts/redisoperator/Chart.yaml b/charts/redisoperator/Chart.yaml index b1347be7e..00bcb2fd4 100644 --- a/charts/redisoperator/Chart.yaml +++ b/charts/redisoperator/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: A Helm chart for the Spotahome Redis Operator name: redisoperator -version: 3.1.0 +version: 3.0.0 diff --git a/mocks/operator/redisfailover/service/RedisFailoverCheck.go b/mocks/operator/redisfailover/service/RedisFailoverCheck.go index 9c9be5d4a..5f12097dd 100644 --- a/mocks/operator/redisfailover/service/RedisFailoverCheck.go +++ b/mocks/operator/redisfailover/service/RedisFailoverCheck.go @@ -43,8 +43,8 @@ func (_m *RedisFailoverCheck) CheckRedisNumber(rFailover *v1.RedisFailover) erro return r0 } -// CheckRedisSyncing provides a mock function with given fields: slaveIP, rFailover -func (_m *RedisFailoverCheck) CheckRedisSyncing(slaveIP string, rFailover *v1.RedisFailover) (bool, error) { +// CheckRedisSlavesReady provides a mock function with given fields: slaveIP, rFailover +func (_m *RedisFailoverCheck) CheckRedisSlavesReady(slaveIP string, rFailover *v1.RedisFailover) (bool, error) { ret := _m.Called(slaveIP, rFailover) var r0 bool diff --git a/mocks/service/redis/Client.go b/mocks/service/redis/Client.go index f44cf04d2..582761e7e 100644 --- a/mocks/service/redis/Client.go +++ b/mocks/service/redis/Client.go @@ -114,8 +114,8 @@ func (_m *Client) IsMaster(ip string, password string) (bool, error) { return r0, r1 } -// IsSyncing provides a mock function with given fields: ip, password -func (_m *Client) IsSyncing(ip string, password string) (bool, error) { +// SlaveIsReady provides a mock function with given fields: ip, password +func (_m *Client) SlaveIsReady(ip string, password string) (bool, error) { ret := _m.Called(ip, password) var r0 bool diff --git a/operator/redisfailover/checker.go b/operator/redisfailover/checker.go index a83c4c5ce..0ec963537 100644 --- a/operator/redisfailover/checker.go +++ b/operator/redisfailover/checker.go @@ -18,14 +18,17 @@ func (r *RedisFailoverHandler) UpdateRedisesPods(rf *redisfailoverv1.RedisFailov return err } - // No perform updates when nodes are syncing. + masterIP, err := r.rfChecker.GetMasterIP(rf) + // No perform updates when nodes are syncing, still not connected, etc. for _, rp := range redises { - sync, err := r.rfChecker.CheckRedisSyncing(rp, rf) - if err != nil { - return err - } - if sync { - return nil + if rp != masterIP { + ready, err := r.rfChecker.CheckRedisSlavesReady(rp, rf) + if err != nil { + return err + } + if !ready { + return nil + } } } diff --git a/operator/redisfailover/checker_test.go b/operator/redisfailover/checker_test.go index 8c4873b89..6a045200a 100644 --- a/operator/redisfailover/checker_test.go +++ b/operator/redisfailover/checker_test.go @@ -162,7 +162,7 @@ func TestCheckAndHeal(t *testing.T) { expErr = true } if !expErr && continueTests { - mrfc.On("GetMasterIP", rf).Once().Return(master, nil) + mrfc.On("GetMasterIP", rf).Twice().Return(master, nil) if test.slavesOK { mrfc.On("CheckAllSlavesFromMaster", master, rf).Once().Return(nil) } else { @@ -170,7 +170,6 @@ func TestCheckAndHeal(t *testing.T) { mrfh.On("SetMasterOnAll", master, rf).Once().Return(nil) } mrfc.On("GetRedisesIPs", rf).Twice().Return([]string{master}, nil) - mrfc.On("CheckRedisSyncing", master, rf).Once().Return(false, nil) mrfc.On("GetStatefulSetUpdateRevision", rf).Once().Return("1", nil) mrfc.On("GetRedisesSlavesPods", rf).Once().Return([]string{}, nil) mrfc.On("GetRedisesMasterPod", rf).Once().Return(master, nil) @@ -214,16 +213,15 @@ func TestCheckAndHeal(t *testing.T) { func TestUpdate(t *testing.T) { type podStatus struct { - pod corev1.Pod - syncing bool - master bool + pod corev1.Pod + ready bool + master bool } tests := []struct { name string pods []podStatus ssVersion string errExpected error - syncing bool }{ { name: "all ok, no change needed", @@ -240,8 +238,8 @@ func TestUpdate(t *testing.T) { PodIP: "0.0.0.0", }, }, - master: false, - syncing: false, + master: false, + ready: true, }, { pod: corev1.Pod{ @@ -255,8 +253,8 @@ func TestUpdate(t *testing.T) { PodIP: "0.0.0.1", }, }, - master: false, - syncing: false, + master: false, + ready: true, }, { pod: corev1.Pod{ @@ -270,8 +268,8 @@ func TestUpdate(t *testing.T) { PodIP: "1.1.1.1", }, }, - master: true, - syncing: false, + master: true, + ready: true, }, }, ssVersion: "10", @@ -292,8 +290,8 @@ func TestUpdate(t *testing.T) { PodIP: "0.0.0.0", }, }, - master: false, - syncing: false, + master: false, + ready: true, }, { pod: corev1.Pod{ @@ -307,8 +305,8 @@ func TestUpdate(t *testing.T) { PodIP: "0.0.0.1", }, }, - master: false, - syncing: true, + master: false, + ready: false, }, { pod: corev1.Pod{ @@ -322,8 +320,8 @@ func TestUpdate(t *testing.T) { PodIP: "1.1.1.1", }, }, - master: true, - syncing: false, + master: true, + ready: true, }, }, ssVersion: "10", @@ -344,8 +342,8 @@ func TestUpdate(t *testing.T) { PodIP: "0.0.0.0", }, }, - master: false, - syncing: false, + master: false, + ready: true, }, { pod: corev1.Pod{ @@ -359,8 +357,8 @@ func TestUpdate(t *testing.T) { PodIP: "0.0.0.1", }, }, - master: false, - syncing: true, + master: false, + ready: true, }, { pod: corev1.Pod{ @@ -374,8 +372,8 @@ func TestUpdate(t *testing.T) { PodIP: "1.1.1.1", }, }, - master: true, - syncing: false, + master: true, + ready: true, }, }, ssVersion: "1", @@ -396,8 +394,8 @@ func TestUpdate(t *testing.T) { PodIP: "0.0.0.0", }, }, - master: false, - syncing: false, + master: false, + ready: true, }, { pod: corev1.Pod{ @@ -411,8 +409,8 @@ func TestUpdate(t *testing.T) { PodIP: "0.0.0.1", }, }, - master: false, - syncing: true, + master: false, + ready: true, }, { pod: corev1.Pod{ @@ -426,8 +424,8 @@ func TestUpdate(t *testing.T) { PodIP: "1.1.1.1", }, }, - master: true, - syncing: false, + master: true, + ready: true, }, }, ssVersion: "10", @@ -445,12 +443,15 @@ func TestUpdate(t *testing.T) { mrfc := &mRFService.RedisFailoverCheck{} mrfc.On("GetRedisesIPs", rf).Once().Return([]string{"0.0.0.0", "0.0.0.1", "1.1.1.1"}, nil) + mrfc.On("GetMasterIP", rf).Once().Return("1.1.1.1", nil) next := true - for _, pod := range test.pods { - mrfc.On("CheckRedisSyncing", pod.pod.Status.PodIP, rf).Once().Return(pod.syncing, nil) - if pod.syncing { + + if !pod.master { + mrfc.On("CheckRedisSlavesReady", pod.pod.Status.PodIP, rf).Once().Return(pod.ready, nil) + } + if !pod.ready { next = false break } @@ -464,7 +465,7 @@ func TestUpdate(t *testing.T) { for _, pod := range test.pods { mrfc.On("GetRedisRevisionHash", pod.pod.ObjectMeta.Name, rf).Once().Return(pod.pod.ObjectMeta.Labels[appsv1.ControllerRevisionHashLabelKey], nil) if pod.pod.ObjectMeta.Labels[appsv1.ControllerRevisionHashLabelKey] != test.ssVersion { - mrfh.On("DeletePod", mock.Anything, mock.Anything) + mrfh.On("DeletePod", pod.pod.ObjectMeta.Name, rf).Once().Return(nil) if pod.master == false { next = false break @@ -473,6 +474,7 @@ func TestUpdate(t *testing.T) { } if next { mrfc.On("GetRedisesMasterPod", rf).Once().Return("master", nil) + } } diff --git a/operator/redisfailover/service/check.go b/operator/redisfailover/service/check.go index 1bc562ab9..b29bf7e45 100644 --- a/operator/redisfailover/service/check.go +++ b/operator/redisfailover/service/check.go @@ -31,7 +31,7 @@ type RedisFailoverCheck interface { GetRedisesMasterPod(rFailover *redisfailoverv1.RedisFailover) (string, error) GetStatefulSetUpdateRevision(rFailover *redisfailoverv1.RedisFailover) (string, error) GetRedisRevisionHash(podName string, rFailover *redisfailoverv1.RedisFailover) (string, error) - CheckRedisSyncing(slaveIP string, rFailover *redisfailoverv1.RedisFailover) (bool, error) + CheckRedisSlavesReady(slaveIP string, rFailover *redisfailoverv1.RedisFailover) (bool, error) } // RedisFailoverChecker is our implementation of RedisFailoverCheck interface @@ -325,12 +325,12 @@ func (r *RedisFailoverChecker) GetRedisRevisionHash(podName string, rFailover *r return val, nil } -// CheckRedisSyncing returns true if the slave is still syncing -func (r *RedisFailoverChecker) CheckRedisSyncing(ip string, rFailover *redisfailoverv1.RedisFailover) (bool, error) { +// CheckRedisSlavesReady returns true if the slave is ready (sync, connected, etc) +func (r *RedisFailoverChecker) CheckRedisSlavesReady(ip string, rFailover *redisfailoverv1.RedisFailover) (bool, error) { password, err := k8s.GetRedisPassword(r.k8sService, rFailover) if err != nil { return false, err } - return r.redisClient.IsSyncing(ip, password) + return r.redisClient.SlaveIsReady(ip, password) } diff --git a/service/redis/client.go b/service/redis/client.go index 44ed7f532..8391e6d2c 100644 --- a/service/redis/client.go +++ b/service/redis/client.go @@ -23,7 +23,7 @@ type Client interface { GetSentinelMonitor(ip string) (string, error) SetCustomSentinelConfig(ip string, configs []string) error SetCustomRedisConfig(ip string, configs []string, password string) error - IsSyncing(ip, password string) (bool, error) + SlaveIsReady(ip, password string) (bool, error) } type client struct { @@ -41,6 +41,8 @@ const ( redisMasterHostREString = "master_host:([0-9.]+)" redisRoleMaster = "role:master" redisSyncing = "master_sync_in_progress:1" + redisMasterSillPending = "master_host:127.0.0.1" + redisLinkUp = "master_link_status:up" redisPort = "6379" sentinelPort = "26379" masterName = "mymaster" @@ -304,7 +306,7 @@ func (c *client) getConfigParameters(config string) (parameter string, value str return s[0], strings.Join(s[1:], " "), nil } -func (c *client) IsSyncing(ip, password string) (bool, error) { +func (c *client) SlaveIsReady(ip, password string) (bool, error) { options := &rediscli.Options{ Addr: fmt.Sprintf("%s:%s", ip, redisPort), Password: password, @@ -316,5 +318,10 @@ func (c *client) IsSyncing(ip, password string) (bool, error) { if err != nil { return false, err } - return strings.Contains(info, redisSyncing), nil + + ok := !strings.Contains(info, redisSyncing) && + !strings.Contains(info, redisMasterSillPending) && + strings.Contains(info, redisLinkUp) + + return ok, nil }