Skip to content

Commit

Permalink
Adding integration tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@samof76
samof76 committed Sep 3, 2024
1 parent a17fcfb commit 10c42ed
Showing 1 changed file with 177 additions and 36 deletions.
213 changes: 177 additions & 36 deletions test/integration/redisfailover/creation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import (

const (
name = "testing"
namespace = "rf-integration-tests"
namespace = "testns"
redisSize = int32(3)
sentinelSize = int32(3)
authSecretPath = "redis-auth"
Expand All @@ -52,23 +52,25 @@ type clients struct {
redisClient redis.Client
}

func (c *clients) prepareNS() error {
func (c *clients) prepareNS(currentNamespace string) error {
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: namespace,
Name: currentNamespace,
},
}
_, err := c.k8sClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
return err
}

func (c *clients) cleanup(stopC chan struct{}) {
c.k8sClient.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{})
func (c *clients) cleanup(stopC chan struct{}, currentNamespace string) {
c.k8sClient.CoreV1().Namespaces().Delete(context.Background(), currentNamespace, metav1.DeleteOptions{})
close(stopC)
}

func TestRedisFailover(t *testing.T) {
require := require.New(t)
disableMyMaster := true
currentNamespace := namespace

// Create signal channels.
stopC := make(chan struct{})
Expand Down Expand Up @@ -97,7 +99,7 @@ func TestRedisFailover(t *testing.T) {
k8sservice := k8s.New(k8sClient, customClient, aeClientset, log.Dummy, metrics.Dummy)

// Prepare namespace
prepErr := clients.prepareNS()
prepErr := clients.prepareNS(currentNamespace)
require.NoError(prepErr)

// Give time to the namespace to be ready
Expand All @@ -112,7 +114,7 @@ func TestRedisFailover(t *testing.T) {
}()

// Prepare cleanup for when the test ends
defer clients.cleanup(stopC)
defer clients.cleanup(stopC, currentNamespace)

// Give time to the operator to start
time.Sleep(15 * time.Second)
Expand All @@ -131,42 +133,170 @@ func TestRedisFailover(t *testing.T) {
require.NoError(err)

// Check that if we create a RedisFailover, it is certainly created and we can get it
ok := t.Run("Check Custom Resource Creation", clients.testCRCreation)
ok := t.Run("Check Custom Resource Creation", func(t *testing.T) {
clients.testCRCreation(t, currentNamespace, disableMyMaster)
})
require.True(ok, "the custom resource has to be created to continue")

// Giving time to the operator to create the resources
time.Sleep(3 * time.Minute)

// Verify that auth is set and actually working
t.Run("Check that auth is set in sentinel and redis configs", clients.testAuth)
t.Run("Check that auth is set in sentinel and redis configs", func(t *testing.T) {
clients.testAuth(t, currentNamespace)
})

// Check custom config is set
t.Run("Check that custom config is behave expected", clients.testCustomConfig)
t.Run("Check that custom config is behave expected", func(t *testing.T) {
clients.testCustomConfig(t, currentNamespace)
})

// Check that a Redis Statefulset is created and the size of it is the one defined by the
// Redis Failover definition created before.
t.Run("Check Redis Statefulset existing and size", clients.testRedisStatefulSet)
t.Run("Check Redis Statefulset existing and size", func(t *testing.T) {
clients.testRedisStatefulSet(t, currentNamespace)
})

// Check that a Sentinel Deployment is created and the size of it is the one defined by the
// Redis Failover definition created before.
t.Run("Check Sentinel Deployment existing and size", clients.testSentinelDeployment)
t.Run("Check Sentinel Deployment existing and size", func(t *testing.T) {
clients.testSentinelDeployment(t, currentNamespace)
})

// Connect to all the Redis pods and, asking to the Redis running inside them, check
// that only one of them is the master of the failover.
t.Run("Check Only One Redis Master", clients.testRedisMaster)
t.Run("Check Only One Redis Master", func(t *testing.T) {
clients.testRedisMaster(t, currentNamespace)
})

// Connect to all the Sentinel pods and, asking to the Sentinel running inside them,
// check that all of them are connected to the same Redis node, and also that that node
// is the master.
t.Run("Check Sentinels Checking the Redis Master", clients.testSentinelMonitoring)
t.Run("Check Sentinels Checking the Redis Master", func(t *testing.T) {
clients.testSentinelMonitoring(t, currentNamespace, disableMyMaster)
})
}

func (c *clients) testCRCreation(t *testing.T) {
func TestRedisFailoverMyMaster(t *testing.T) {
require := require.New(t)
disableMyMaster := false
currentNamespace := "mymaster-" + namespace

// Create signal channels.
stopC := make(chan struct{})
errC := make(chan error)

flags := &utils.CMDFlags{
KubeConfig: filepath.Join(homedir.HomeDir(), ".kube", "config"),
Development: true,
}

// Kubernetes clients.
k8sClient, customClient, aeClientset, err := utils.CreateKubernetesClients(flags)
require.NoError(err)

// Create the redis clients
redisClient := redis.New(metrics.Dummy)

clients := clients{
k8sClient: k8sClient,
rfClient: customClient,
aeClient: aeClientset,
redisClient: redisClient,
}

// Create kubernetes service.
k8sservice := k8s.New(k8sClient, customClient, aeClientset, log.Dummy, metrics.Dummy)

// Prepare namespace
prepErr := clients.prepareNS(currentNamespace)
require.NoError(prepErr)

// Give time to the namespace to be ready
time.Sleep(15 * time.Second)

// Create operator and run.
redisfailoverOperator, err := redisfailover.New(redisfailover.Config{}, k8sservice, k8sClient, currentNamespace, redisClient, metrics.Dummy, log.Dummy)
require.NoError(err)

go func() {
errC <- redisfailoverOperator.Run(context.Background())
}()

// Prepare cleanup for when the test ends
defer clients.cleanup(stopC, currentNamespace)

// Give time to the operator to start
time.Sleep(15 * time.Second)

// Create secret
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: authSecretPath,
Namespace: currentNamespace,
},
Data: map[string][]byte{
"password": []byte(testPass),
},
}
_, err = k8sClient.CoreV1().Secrets(currentNamespace).Create(context.Background(), secret, metav1.CreateOptions{})
require.NoError(err)

// Check that if we create a RedisFailover, it is certainly created and we can get it
ok := t.Run("Check Custom Resource Creation", func(t *testing.T) {
clients.testCRCreation(t, currentNamespace, disableMyMaster)
})
require.True(ok, "the custom resource has to be created to continue")

// Giving time to the operator to create the resources
time.Sleep(3 * time.Minute)

// Verify that auth is set and actually working
t.Run("Check that auth is set in sentinel and redis configs", func(t *testing.T) {
clients.testAuth(t, currentNamespace)
})

// Check custom config is set
t.Run("Check that custom config is behave expected", func(t *testing.T) {
clients.testCustomConfig(t, currentNamespace)
})

// Check that a Redis Statefulset is created and the size of it is the one defined by the
// Redis Failover definition created before.
t.Run("Check Redis Statefulset existing and size", func(t *testing.T) {
clients.testRedisStatefulSet(t, currentNamespace)
})

// Check that a Sentinel Deployment is created and the size of it is the one defined by the
// Redis Failover definition created before.
t.Run("Check Sentinel Deployment existing and size", func(t *testing.T) {
clients.testSentinelDeployment(t, currentNamespace)
})

// Connect to all the Redis pods and, asking to the Redis running inside them, check
// that only one of them is the master of the failover.
t.Run("Check Only One Redis Master", func(t *testing.T) {
clients.testRedisMaster(t, currentNamespace)
})

// Connect to all the Sentinel pods and, asking to the Sentinel running inside them,
// check that all of them are connected to the same Redis node, and also that that node
// is the master.
t.Run("Check Sentinels Checking the Redis Master", func(t *testing.T) {
clients.testSentinelMonitoring(t, currentNamespace, disableMyMaster)
})
}

func (c *clients) testCRCreation(t *testing.T, currentNamespace string, args ...bool) {
disableMyMaster := false
if len(args) > 0 && args[0] {
disableMyMaster = args[0]
}
assert := assert.New(t)
toCreate := &redisfailoverv1.RedisFailover{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Namespace: currentNamespace,
},
Spec: redisfailoverv1.RedisFailoverSpec{
Redis: redisfailoverv1.RedisSettings{
Expand All @@ -177,47 +307,48 @@ func (c *clients) testCRCreation(t *testing.T) {
CustomConfig: []string{`save ""`},
},
Sentinel: redisfailoverv1.SentinelSettings{
Replicas: sentinelSize,
Replicas: sentinelSize,
DisableMyMaster: disableMyMaster,
},
Auth: redisfailoverv1.AuthSettings{
SecretPath: authSecretPath,
},
},
}

c.rfClient.DatabasesV1().RedisFailovers(namespace).Create(context.Background(), toCreate, metav1.CreateOptions{})
gotRF, err := c.rfClient.DatabasesV1().RedisFailovers(namespace).Get(context.Background(), name, metav1.GetOptions{})
c.rfClient.DatabasesV1().RedisFailovers(currentNamespace).Create(context.Background(), toCreate, metav1.CreateOptions{})
gotRF, err := c.rfClient.DatabasesV1().RedisFailovers(currentNamespace).Get(context.Background(), name, metav1.GetOptions{})

assert.NoError(err)
assert.Equal(toCreate.Spec, gotRF.Spec)
}

func (c *clients) testRedisStatefulSet(t *testing.T) {
func (c *clients) testRedisStatefulSet(t *testing.T, currentNamespace string) {
assert := assert.New(t)
redisSS, err := c.k8sClient.AppsV1().StatefulSets(namespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{})
redisSS, err := c.k8sClient.AppsV1().StatefulSets(currentNamespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{})
assert.NoError(err)
assert.Equal(redisSize, int32(redisSS.Status.Replicas))
}

func (c *clients) testSentinelDeployment(t *testing.T) {
func (c *clients) testSentinelDeployment(t *testing.T, currentNamespace string) {
assert := assert.New(t)
sentinelD, err := c.k8sClient.AppsV1().Deployments(namespace).Get(context.Background(), fmt.Sprintf("rfs-%s", name), metav1.GetOptions{})
sentinelD, err := c.k8sClient.AppsV1().Deployments(currentNamespace).Get(context.Background(), fmt.Sprintf("rfs-%s", name), metav1.GetOptions{})
assert.NoError(err)
assert.Equal(3, int(sentinelD.Status.Replicas))
}

func (c *clients) testRedisMaster(t *testing.T) {
func (c *clients) testRedisMaster(t *testing.T, currentNamespace string) {
assert := assert.New(t)
masters := []string{}

redisSS, err := c.k8sClient.AppsV1().StatefulSets(namespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{})
redisSS, err := c.k8sClient.AppsV1().StatefulSets(currentNamespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{})
assert.NoError(err)

listOptions := metav1.ListOptions{
LabelSelector: labels.FormatLabels(redisSS.Spec.Selector.MatchLabels),
}

redisPodList, err := c.k8sClient.CoreV1().Pods(namespace).List(context.Background(), listOptions)
redisPodList, err := c.k8sClient.CoreV1().Pods(currentNamespace).List(context.Background(), listOptions)

assert.NoError(err)

Expand All @@ -231,22 +362,32 @@ func (c *clients) testRedisMaster(t *testing.T) {
assert.Equal(1, len(masters), "only one master expected")
}

func (c *clients) testSentinelMonitoring(t *testing.T) {
func (c *clients) testSentinelMonitoring(t *testing.T, currentNamespace string, args ...bool) {
disableMyMaster := false
if len(args) > 0 {
disableMyMaster = args[0]
}

masterName := "mymaster"
if disableMyMaster {
masterName = name
}

assert := assert.New(t)
masters := []string{}

sentinelD, err := c.k8sClient.AppsV1().Deployments(namespace).Get(context.Background(), fmt.Sprintf("rfs-%s", name), metav1.GetOptions{})
sentinelD, err := c.k8sClient.AppsV1().Deployments(currentNamespace).Get(context.Background(), fmt.Sprintf("rfs-%s", name), metav1.GetOptions{})
assert.NoError(err)

listOptions := metav1.ListOptions{
LabelSelector: labels.FormatLabels(sentinelD.Spec.Selector.MatchLabels),
}
sentinelPodList, err := c.k8sClient.CoreV1().Pods(namespace).List(context.Background(), listOptions)
sentinelPodList, err := c.k8sClient.CoreV1().Pods(currentNamespace).List(context.Background(), listOptions)
assert.NoError(err)

for _, pod := range sentinelPodList.Items {
ip := pod.Status.PodIP
master, _, _ := c.redisClient.GetSentinelMonitor(ip, "mymaster")
master, _, _ := c.redisClient.GetSentinelMonitor(ip, masterName)
masters = append(masters, master)
}

Expand All @@ -259,15 +400,15 @@ func (c *clients) testSentinelMonitoring(t *testing.T) {
assert.True(isMaster, "Sentinel should monitor the Redis master")
}

func (c *clients) testAuth(t *testing.T) {
func (c *clients) testAuth(t *testing.T, currentNamespace string) {
assert := assert.New(t)

redisCfg, err := c.k8sClient.CoreV1().ConfigMaps(namespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{})
redisCfg, err := c.k8sClient.CoreV1().ConfigMaps(currentNamespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{})
assert.NoError(err)
assert.Contains(redisCfg.Data["redis.conf"], "requirepass "+testPass)
assert.Contains(redisCfg.Data["redis.conf"], "masterauth "+testPass)

redisSS, err := c.k8sClient.AppsV1().StatefulSets(namespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{})
redisSS, err := c.k8sClient.AppsV1().StatefulSets(currentNamespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{})
assert.NoError(err)

assert.Len(redisSS.Spec.Template.Spec.Containers, 2)
Expand All @@ -282,16 +423,16 @@ func (c *clients) testAuth(t *testing.T) {
assert.Equal(redisSS.Spec.Template.Spec.Containers[1].Env[4].ValueFrom.SecretKeyRef.LocalObjectReference.Name, authSecretPath)
}

func (c *clients) testCustomConfig(t *testing.T) {
func (c *clients) testCustomConfig(t *testing.T, currentNamespace string) {
assert := assert.New(t)

redisSS, err := c.k8sClient.AppsV1().StatefulSets(namespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{})
redisSS, err := c.k8sClient.AppsV1().StatefulSets(currentNamespace).Get(context.Background(), fmt.Sprintf("rfr-%s", name), metav1.GetOptions{})
assert.NoError(err)

listOptions := metav1.ListOptions{
LabelSelector: labels.FormatLabels(redisSS.Spec.Selector.MatchLabels),
}
redisPodList, err := c.k8sClient.CoreV1().Pods(namespace).List(context.Background(), listOptions)
redisPodList, err := c.k8sClient.CoreV1().Pods(currentNamespace).List(context.Background(), listOptions)
assert.NoError(err)

rClient := rediscli.NewClient(&rediscli.Options{
Expand Down

0 comments on commit 10c42ed

Please sign in to comment.