diff --git a/test/integration/redisfailover/creation_test.go b/test/integration/redisfailover/creation_test.go index f68a07ead..8340c799b 100644 --- a/test/integration/redisfailover/creation_test.go +++ b/test/integration/redisfailover/creation_test.go @@ -37,7 +37,7 @@ import ( const ( name = "testing" - namespace = "rf-integration-tests" + namespace = "testns" redisSize = int32(3) sentinelSize = int32(3) authSecretPath = "redis-auth" @@ -52,23 +52,25 @@ type clients struct { redisClient redis.Client } -func (c *clients) prepareNS() error { +func (c *clients) prepareNS(currentNamespace string) error { ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: namespace, + Name: currentNamespace, }, } _, err := c.k8sClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) return err } -func (c *clients) cleanup(stopC chan struct{}) { - c.k8sClient.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{}) +func (c *clients) cleanup(stopC chan struct{}, currentNamespace string) { + c.k8sClient.CoreV1().Namespaces().Delete(context.Background(), currentNamespace, metav1.DeleteOptions{}) close(stopC) } func TestRedisFailover(t *testing.T) { require := require.New(t) + disableMyMaster := true + currentNamespace := namespace // Create signal channels. stopC := make(chan struct{}) @@ -97,7 +99,7 @@ func TestRedisFailover(t *testing.T) { k8sservice := k8s.New(k8sClient, customClient, aeClientset, log.Dummy, metrics.Dummy) // Prepare namespace - prepErr := clients.prepareNS() + prepErr := clients.prepareNS(currentNamespace) require.NoError(prepErr) // Give time to the namespace to be ready @@ -112,7 +114,7 @@ func TestRedisFailover(t *testing.T) { }() // Prepare cleanup for when the test ends - defer clients.cleanup(stopC) + defer clients.cleanup(stopC, currentNamespace) // Give time to the operator to start time.Sleep(15 * time.Second) @@ -131,42 +133,170 @@ func TestRedisFailover(t *testing.T) { require.NoError(err) // Check that if we create a RedisFailover, it is certainly created and we can get it - ok := t.Run("Check Custom Resource Creation", clients.testCRCreation) + ok := t.Run("Check Custom Resource Creation", func(t *testing.T) { + clients.testCRCreation(t, currentNamespace, disableMyMaster) + }) require.True(ok, "the custom resource has to be created to continue") // Giving time to the operator to create the resources time.Sleep(3 * time.Minute) // Verify that auth is set and actually working - t.Run("Check that auth is set in sentinel and redis configs", clients.testAuth) + t.Run("Check that auth is set in sentinel and redis configs", func(t *testing.T) { + clients.testAuth(t, currentNamespace) + }) // Check custom config is set - t.Run("Check that custom config is behave expected", clients.testCustomConfig) + t.Run("Check that custom config is behave expected", func(t *testing.T) { + clients.testCustomConfig(t, currentNamespace) + }) // Check that a Redis Statefulset is created and the size of it is the one defined by the // Redis Failover definition created before. - t.Run("Check Redis Statefulset existing and size", clients.testRedisStatefulSet) + t.Run("Check Redis Statefulset existing and size", func(t *testing.T) { + clients.testRedisStatefulSet(t, currentNamespace) + }) // Check that a Sentinel Deployment is created and the size of it is the one defined by the // Redis Failover definition created before. - t.Run("Check Sentinel Deployment existing and size", clients.testSentinelDeployment) + t.Run("Check Sentinel Deployment existing and size", func(t *testing.T) { + clients.testSentinelDeployment(t, currentNamespace) + }) // Connect to all the Redis pods and, asking to the Redis running inside them, check // that only one of them is the master of the failover. - t.Run("Check Only One Redis Master", clients.testRedisMaster) + t.Run("Check Only One Redis Master", func(t *testing.T) { + clients.testRedisMaster(t, currentNamespace) + }) // Connect to all the Sentinel pods and, asking to the Sentinel running inside them, // check that all of them are connected to the same Redis node, and also that that node // is the master. - t.Run("Check Sentinels Checking the Redis Master", clients.testSentinelMonitoring) + t.Run("Check Sentinels Checking the Redis Master", func(t *testing.T) { + clients.testSentinelMonitoring(t, currentNamespace, disableMyMaster) + }) } -func (c *clients) testCRCreation(t *testing.T) { +func TestRedisFailoverMyMaster(t *testing.T) { + require := require.New(t) + disableMyMaster := false + currentNamespace := "mymaster-" + namespace + + // Create signal channels. + stopC := make(chan struct{}) + errC := make(chan error) + + flags := &utils.CMDFlags{ + KubeConfig: filepath.Join(homedir.HomeDir(), ".kube", "config"), + Development: true, + } + + // Kubernetes clients. + k8sClient, customClient, aeClientset, err := utils.CreateKubernetesClients(flags) + require.NoError(err) + + // Create the redis clients + redisClient := redis.New(metrics.Dummy) + + clients := clients{ + k8sClient: k8sClient, + rfClient: customClient, + aeClient: aeClientset, + redisClient: redisClient, + } + + // Create kubernetes service. + k8sservice := k8s.New(k8sClient, customClient, aeClientset, log.Dummy, metrics.Dummy) + + // Prepare namespace + prepErr := clients.prepareNS(currentNamespace) + require.NoError(prepErr) + + // Give time to the namespace to be ready + time.Sleep(15 * time.Second) + + // Create operator and run. + redisfailoverOperator, err := redisfailover.New(redisfailover.Config{}, k8sservice, k8sClient, currentNamespace, redisClient, metrics.Dummy, log.Dummy) + require.NoError(err) + + go func() { + errC <- redisfailoverOperator.Run(context.Background()) + }() + + // Prepare cleanup for when the test ends + defer clients.cleanup(stopC, currentNamespace) + + // Give time to the operator to start + time.Sleep(15 * time.Second) + + // Create secret + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: authSecretPath, + Namespace: currentNamespace, + }, + Data: map[string][]byte{ + "password": []byte(testPass), + }, + } + _, err = k8sClient.CoreV1().Secrets(currentNamespace).Create(context.Background(), secret, metav1.CreateOptions{}) + require.NoError(err) + + // Check that if we create a RedisFailover, it is certainly created and we can get it + ok := t.Run("Check Custom Resource Creation", func(t *testing.T) { + clients.testCRCreation(t, currentNamespace, disableMyMaster) + }) + require.True(ok, "the custom resource has to be created to continue") + + // Giving time to the operator to create the resources + time.Sleep(3 * time.Minute) + + // Verify that auth is set and actually working + t.Run("Check that auth is set in sentinel and redis configs", func(t *testing.T) { + clients.testAuth(t, currentNamespace) + }) + + // Check custom config is set + t.Run("Check that custom config is behave expected", func(t *testing.T) { + clients.testCustomConfig(t, currentNamespace) + }) + + // Check that a Redis Statefulset is created and the size of it is the one defined by the + // Redis Failover definition created before. + t.Run("Check Redis Statefulset existing and size", func(t *testing.T) { + clients.testRedisStatefulSet(t, currentNamespace) + }) + + // Check that a Sentinel Deployment is created and the size of it is the one defined by the + // Redis Failover definition created before. + t.Run("Check Sentinel Deployment existing and size", func(t *testing.T) { + clients.testSentinelDeployment(t, currentNamespace) + }) + + // Connect to all the Redis pods and, asking to the Redis running inside them, check + // that only one of them is the master of the failover. + t.Run("Check Only One Redis Master", func(t *testing.T) { + clients.testRedisMaster(t, currentNamespace) + }) + + // Connect to all the Sentinel pods and, asking to the Sentinel running inside them, + // check that all of them are connected to the same Redis node, and also that that node + // is the master. + t.Run("Check Sentinels Checking the Redis Master", func(t *testing.T) { + clients.testSentinelMonitoring(t, currentNamespace, disableMyMaster) + }) +} + +func (c *clients) testCRCreation(t *testing.T, currentNamespace string, args ...bool) { + disableMyMaster := false + if len(args) > 0 && args[0] { + disableMyMaster = args[0] + } assert := assert.New(t) toCreate := &redisfailoverv1.RedisFailover{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: namespace, + Namespace: currentNamespace, }, Spec: redisfailoverv1.RedisFailoverSpec{ Redis: redisfailoverv1.RedisSettings{ @@ -177,7 +307,8 @@ func (c *clients) testCRCreation(t *testing.T) { CustomConfig: []string{`save ""`}, }, Sentinel: redisfailoverv1.SentinelSettings{ - Replicas: sentinelSize, + Replicas: sentinelSize, + DisableMyMaster: disableMyMaster, }, Auth: redisfailoverv1.AuthSettings{ SecretPath: authSecretPath, @@ -185,39 +316,39 @@ func (c *clients) testCRCreation(t *testing.T) { }, } - c.rfClient.DatabasesV1().RedisFailovers(namespace).Create(context.Background(), toCreate, metav1.CreateOptions{}) - gotRF, err := c.rfClient.DatabasesV1().RedisFailovers(namespace).Get(context.Background(), name, metav1.GetOptions{}) + c.rfClient.DatabasesV1().RedisFailovers(currentNamespace).Create(context.Background(), toCreate, metav1.CreateOptions{}) + gotRF, err := c.rfClient.DatabasesV1().RedisFailovers(currentNamespace).Get(context.Background(), name, metav1.GetOptions{}) assert.NoError(err) assert.Equal(toCreate.Spec, gotRF.Spec) } -func (c *clients) testRedisStatefulSet(t *testing.T) { +func (c *clients) testRedisStatefulSet(t *testing.T, currentNamespace string) { assert := assert.New(t) - redisSS, err := c.k8sClient.AppsV1().StatefulSets(namespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{}) + redisSS, err := c.k8sClient.AppsV1().StatefulSets(currentNamespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{}) assert.NoError(err) assert.Equal(redisSize, int32(redisSS.Status.Replicas)) } -func (c *clients) testSentinelDeployment(t *testing.T) { +func (c *clients) testSentinelDeployment(t *testing.T, currentNamespace string) { assert := assert.New(t) - sentinelD, err := c.k8sClient.AppsV1().Deployments(namespace).Get(context.Background(), fmt.Sprintf("rfs-%s", name), metav1.GetOptions{}) + sentinelD, err := c.k8sClient.AppsV1().Deployments(currentNamespace).Get(context.Background(), fmt.Sprintf("rfs-%s", name), metav1.GetOptions{}) assert.NoError(err) assert.Equal(3, int(sentinelD.Status.Replicas)) } -func (c *clients) testRedisMaster(t *testing.T) { +func (c *clients) testRedisMaster(t *testing.T, currentNamespace string) { assert := assert.New(t) masters := []string{} - redisSS, err := c.k8sClient.AppsV1().StatefulSets(namespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{}) + redisSS, err := c.k8sClient.AppsV1().StatefulSets(currentNamespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{}) assert.NoError(err) listOptions := metav1.ListOptions{ LabelSelector: labels.FormatLabels(redisSS.Spec.Selector.MatchLabels), } - redisPodList, err := c.k8sClient.CoreV1().Pods(namespace).List(context.Background(), listOptions) + redisPodList, err := c.k8sClient.CoreV1().Pods(currentNamespace).List(context.Background(), listOptions) assert.NoError(err) @@ -231,22 +362,32 @@ func (c *clients) testRedisMaster(t *testing.T) { assert.Equal(1, len(masters), "only one master expected") } -func (c *clients) testSentinelMonitoring(t *testing.T) { +func (c *clients) testSentinelMonitoring(t *testing.T, currentNamespace string, args ...bool) { + disableMyMaster := false + if len(args) > 0 { + disableMyMaster = args[0] + } + + masterName := "mymaster" + if disableMyMaster { + masterName = name + } + assert := assert.New(t) masters := []string{} - sentinelD, err := c.k8sClient.AppsV1().Deployments(namespace).Get(context.Background(), fmt.Sprintf("rfs-%s", name), metav1.GetOptions{}) + sentinelD, err := c.k8sClient.AppsV1().Deployments(currentNamespace).Get(context.Background(), fmt.Sprintf("rfs-%s", name), metav1.GetOptions{}) assert.NoError(err) listOptions := metav1.ListOptions{ LabelSelector: labels.FormatLabels(sentinelD.Spec.Selector.MatchLabels), } - sentinelPodList, err := c.k8sClient.CoreV1().Pods(namespace).List(context.Background(), listOptions) + sentinelPodList, err := c.k8sClient.CoreV1().Pods(currentNamespace).List(context.Background(), listOptions) assert.NoError(err) for _, pod := range sentinelPodList.Items { ip := pod.Status.PodIP - master, _, _ := c.redisClient.GetSentinelMonitor(ip, "mymaster") + master, _, _ := c.redisClient.GetSentinelMonitor(ip, masterName) masters = append(masters, master) } @@ -259,15 +400,15 @@ func (c *clients) testSentinelMonitoring(t *testing.T) { assert.True(isMaster, "Sentinel should monitor the Redis master") } -func (c *clients) testAuth(t *testing.T) { +func (c *clients) testAuth(t *testing.T, currentNamespace string) { assert := assert.New(t) - redisCfg, err := c.k8sClient.CoreV1().ConfigMaps(namespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{}) + redisCfg, err := c.k8sClient.CoreV1().ConfigMaps(currentNamespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{}) assert.NoError(err) assert.Contains(redisCfg.Data["redis.conf"], "requirepass "+testPass) assert.Contains(redisCfg.Data["redis.conf"], "masterauth "+testPass) - redisSS, err := c.k8sClient.AppsV1().StatefulSets(namespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{}) + redisSS, err := c.k8sClient.AppsV1().StatefulSets(currentNamespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{}) assert.NoError(err) assert.Len(redisSS.Spec.Template.Spec.Containers, 2) @@ -282,16 +423,16 @@ func (c *clients) testAuth(t *testing.T) { assert.Equal(redisSS.Spec.Template.Spec.Containers[1].Env[4].ValueFrom.SecretKeyRef.LocalObjectReference.Name, authSecretPath) } -func (c *clients) testCustomConfig(t *testing.T) { +func (c *clients) testCustomConfig(t *testing.T, currentNamespace string) { assert := assert.New(t) - redisSS, err := c.k8sClient.AppsV1().StatefulSets(namespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{}) + redisSS, err := c.k8sClient.AppsV1().StatefulSets(currentNamespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{}) assert.NoError(err) listOptions := metav1.ListOptions{ LabelSelector: labels.FormatLabels(redisSS.Spec.Selector.MatchLabels), } - redisPodList, err := c.k8sClient.CoreV1().Pods(namespace).List(context.Background(), listOptions) + redisPodList, err := c.k8sClient.CoreV1().Pods(currentNamespace).List(context.Background(), listOptions) assert.NoError(err) rClient := rediscli.NewClient(&rediscli.Options{