feat: Redis cluster pod anti affinity #1174

.github/dependabot.yml

@@ -3,8 +3,8 @@ updates:
   - package-ecosystem: gomod
     directory: /
     schedule:
-      interval: daily
+      interval: monthly
   - package-ecosystem: github-actions
     directory: /
     schedule:
-      interval: daily
+      interval: monthly

.github/workflows/ci.yaml

@@ -11,7 +11,7 @@ permissions:
   contents: read
 
 env:
-  GOLANG_VERSION: 1.22
+  GOLANG_VERSION: 1.23.4
   APPLICATION_NAME: redis-operator
   DockerImagName: docker.io/opstree/redis-operator
   BuildDocs: true
@@ -32,7 +32,7 @@ jobs:
       - name: Run GolangCI-Lint
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.54.0
+          version: v1.62.2
 
   gotest:
     needs:

.golangci.yml

@@ -24,7 +24,6 @@ linters:
     - gofmt
     - gofumpt
     - goprintffuncname
-    - gosec
     - gosimple
     - govet
     - grouper
@@ -39,15 +38,14 @@ linters:
     - tenv
     - thelper
     - tparallel
-    - typecheck
     - unconvert
     - unused
     - wastedassign
     - whitespace
 
 run:
   timeout: 15m
-  go: "1.22"
+  go: "1.23.4"
   tests: true
   show-stats: true
   skip-files:

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.22 as builder
+FROM golang:1.23-alpine as builder
 ARG BUILDOS
 ARG BUILDPLATFORM
 ARG BUILDARCH

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [2020] [Opstree Solutions]
+   Copyright [2024] [Opstree Solutions]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

PROJECT

@@ -88,4 +88,11 @@ resources:
   kind: RedisSentinel
   path: redis-operator/api/v1beta1
   version: v1beta1
+- group: core
+  kind: Pod
+  path: k8s.io/api/core/v1
+  version: v1
+  webhooks:
+    defaulting: true
+    webhookVersion: v1
 version: "3"

api/v1beta2/rediscluster_types.go

@@ -31,21 +31,22 @@ type RedisClusterSpec struct {
 	// +kubebuilder:default:=6379
 	Port *int `json:"port,omitempty"`
 	// +kubebuilder:default:=v7
-	ClusterVersion     *string                      `json:"clusterVersion,omitempty"`
-	RedisLeader        RedisLeader                  `json:"redisLeader,omitempty"`
-	RedisFollower      RedisFollower                `json:"redisFollower,omitempty"`
-	RedisExporter      *RedisExporter               `json:"redisExporter,omitempty"`
-	Storage            *ClusterStorage              `json:"storage,omitempty"`
-	PodSecurityContext *corev1.PodSecurityContext   `json:"podSecurityContext,omitempty"`
-	PriorityClassName  string                       `json:"priorityClassName,omitempty"`
-	Resources          *corev1.ResourceRequirements `json:"resources,omitempty"`
-	TLS                *TLSConfig                   `json:"TLS,omitempty"`
-	ACL                *ACLConfig                   `json:"acl,omitempty"`
-	InitContainer      *InitContainer               `json:"initContainer,omitempty"`
-	Sidecars           *[]Sidecar                   `json:"sidecars,omitempty"`
-	ServiceAccountName *string                      `json:"serviceAccountName,omitempty"`
-	PersistenceEnabled *bool                        `json:"persistenceEnabled,omitempty"`
-	EnvVars            *[]corev1.EnvVar             `json:"env,omitempty"`
+	ClusterVersion                *string                      `json:"clusterVersion,omitempty"`
+	RedisLeader                   RedisLeader                  `json:"redisLeader,omitempty"`
+	RedisFollower                 RedisFollower                `json:"redisFollower,omitempty"`
+	RedisExporter                 *RedisExporter               `json:"redisExporter,omitempty"`
+	Storage                       *ClusterStorage              `json:"storage,omitempty"`
+	PodSecurityContext            *corev1.PodSecurityContext   `json:"podSecurityContext,omitempty"`
+	PriorityClassName             string                       `json:"priorityClassName,omitempty"`
+	Resources                     *corev1.ResourceRequirements `json:"resources,omitempty"`
+	TLS                           *TLSConfig                   `json:"TLS,omitempty"`
+	ACL                           *ACLConfig                   `json:"acl,omitempty"`
+	InitContainer                 *InitContainer               `json:"initContainer,omitempty"`
+	Sidecars                      *[]Sidecar                   `json:"sidecars,omitempty"`
+	ServiceAccountName            *string                      `json:"serviceAccountName,omitempty"`
+	PersistenceEnabled            *bool                        `json:"persistenceEnabled,omitempty"`
+	EnvVars                       *[]corev1.EnvVar             `json:"env,omitempty"`
+	LeaderFollowerPodAntiAffinity *bool                        `json:"leaderFollowerPodAntiAffinity,omitempty"`
 }
 
 func (cr *RedisClusterSpec) GetReplicaCounts(t string) int32 {

api/v1beta2/redisreplication_types.go

@@ -1,31 +1,33 @@
 package v1beta2
 
 import (
+	common "github.com/OT-CONTAINER-KIT/redis-operator/api"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 type RedisReplicationSpec struct {
-	Size                          *int32                     `json:"clusterSize"`
-	KubernetesConfig              KubernetesConfig           `json:"kubernetesConfig"`
-	RedisExporter                 *RedisExporter             `json:"redisExporter,omitempty"`
-	RedisConfig                   *RedisConfig               `json:"redisConfig,omitempty"`
-	Storage                       *Storage                   `json:"storage,omitempty"`
-	NodeSelector                  map[string]string          `json:"nodeSelector,omitempty"`
-	PodSecurityContext            *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"`
-	SecurityContext               *corev1.SecurityContext    `json:"securityContext,omitempty"`
-	PriorityClassName             string                     `json:"priorityClassName,omitempty"`
-	Affinity                      *corev1.Affinity           `json:"affinity,omitempty"`
-	Tolerations                   *[]corev1.Toleration       `json:"tolerations,omitempty"`
-	TLS                           *TLSConfig                 `json:"TLS,omitempty"`
-	ACL                           *ACLConfig                 `json:"acl,omitempty"`
-	ReadinessProbe                *corev1.Probe              `json:"readinessProbe,omitempty" protobuf:"bytes,11,opt,name=readinessProbe"`
-	LivenessProbe                 *corev1.Probe              `json:"livenessProbe,omitempty" protobuf:"bytes,12,opt,name=livenessProbe"`
-	InitContainer                 *InitContainer             `json:"initContainer,omitempty"`
-	Sidecars                      *[]Sidecar                 `json:"sidecars,omitempty"`
-	ServiceAccountName            *string                    `json:"serviceAccountName,omitempty"`
-	TerminationGracePeriodSeconds *int64                     `json:"terminationGracePeriodSeconds,omitempty" protobuf:"varint,4,opt,name=terminationGracePeriodSeconds"`
-	EnvVars                       *[]corev1.EnvVar           `json:"env,omitempty"`
+	Size                          *int32                           `json:"clusterSize"`
+	KubernetesConfig              KubernetesConfig                 `json:"kubernetesConfig"`
+	RedisExporter                 *RedisExporter                   `json:"redisExporter,omitempty"`
+	RedisConfig                   *RedisConfig                     `json:"redisConfig,omitempty"`
+	Storage                       *Storage                         `json:"storage,omitempty"`
+	NodeSelector                  map[string]string                `json:"nodeSelector,omitempty"`
+	PodSecurityContext            *corev1.PodSecurityContext       `json:"podSecurityContext,omitempty"`
+	SecurityContext               *corev1.SecurityContext          `json:"securityContext,omitempty"`
+	PriorityClassName             string                           `json:"priorityClassName,omitempty"`
+	Affinity                      *corev1.Affinity                 `json:"affinity,omitempty"`
+	Tolerations                   *[]corev1.Toleration             `json:"tolerations,omitempty"`
+	TLS                           *TLSConfig                       `json:"TLS,omitempty"`
+	PodDisruptionBudget           *common.RedisPodDisruptionBudget `json:"pdb,omitempty"`
+	ACL                           *ACLConfig                       `json:"acl,omitempty"`
+	ReadinessProbe                *corev1.Probe                    `json:"readinessProbe,omitempty" protobuf:"bytes,11,opt,name=readinessProbe"`
+	LivenessProbe                 *corev1.Probe                    `json:"livenessProbe,omitempty" protobuf:"bytes,12,opt,name=livenessProbe"`
+	InitContainer                 *InitContainer                   `json:"initContainer,omitempty"`
+	Sidecars                      *[]Sidecar                       `json:"sidecars,omitempty"`
+	ServiceAccountName            *string                          `json:"serviceAccountName,omitempty"`
+	TerminationGracePeriodSeconds *int64                           `json:"terminationGracePeriodSeconds,omitempty" protobuf:"varint,4,opt,name=terminationGracePeriodSeconds"`
+	EnvVars                       *[]corev1.EnvVar                 `json:"env,omitempty"`
 }
 
 func (cr *RedisReplicationSpec) GetReplicationCounts(t string) int32 {
@@ -41,6 +43,8 @@ type RedisReplicationStatus struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:storageversion
+// +kubebuilder:printcolumn:name="Master",type="string",JSONPath=".status.masterNode"
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 
 // Redis is the Schema for the redis API
 type RedisReplication struct {

config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml

@@ -6598,6 +6598,8 @@ spec:
                 required:
                 - image
                 type: object
+              leaderFollowerPodAntiAffinity:
+                type: boolean
               persistenceEnabled:
                 type: boolean
               podSecurityContext:

config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml

@@ -4397,7 +4397,14 @@ spec:
     storage: false
     subresources:
       status: {}
-  - name: v1beta2
+  - additionalPrinterColumns:
+    - jsonPath: .status.masterNode
+      name: Master
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
     schema:
       openAPIV3Schema:
         description: Redis is the Schema for the redis API
@@ -6233,6 +6240,19 @@ spec:
                 additionalProperties:
                   type: string
                 type: object
+              pdb:
+                description: RedisPodDisruptionBudget configure a PodDisruptionBudget
+                  on the resource (leader/follower)
+                properties:
+                  enabled:
+                    type: boolean
+                  maxUnavailable:
+                    format: int32
+                    type: integer
+                  minAvailable:
+                    format: int32
+                    type: integer
+                type: object
               podSecurityContext:
                 description: |-
                   PodSecurityContext holds pod-level security attributes and common container settings.

config/default/kustomization.yaml

@@ -18,9 +18,9 @@ bases:
 - ../manager
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
-#- ../webhook
+- ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
-#- ../certmanager
+- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
 
@@ -36,12 +36,12 @@ bases:
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
-#- manager_webhook_patch.yaml
+- manager_webhook_patch.yaml
 
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
 # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
 # 'CERTMANAGER' needs to be enabled to use ca injection
-#- webhookcainjection_patch.yaml
+- webhookcainjection_patch.yaml
 
 # the following config is for teaching kustomize how to do var substitution
 #vars:

go.mod

@@ -1,13 +1,13 @@
 module github.com/OT-CONTAINER-KIT/redis-operator
 
-go 1.22
+go 1.23.4
 
 require (
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/banzaicloud/k8s-objectmatcher v1.8.0
 	github.com/go-logr/logr v1.4.2
 	github.com/go-redis/redismock/v9 v9.2.0
-	github.com/onsi/ginkgo/v2 v2.20.1
+	github.com/onsi/ginkgo/v2 v2.22.0
 	github.com/onsi/gomega v1.35.1
 	github.com/pkg/errors v0.9.1
 	github.com/redis/go-redis/v9 v9.7.0
@@ -40,7 +40,7 @@ require (
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
+	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
@@ -67,7 +67,7 @@ require (
 	golang.org/x/term v0.25.0 // indirect
 	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.24.0 // indirect
+	golang.org/x/tools v0.26.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.35.1 // indirect

go.sum

@@ -100,8 +100,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA=
-github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -153,8 +153,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo=
-github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI=
+github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
+github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
@@ -269,8 +269,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
-golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
+golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
+golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

main.go

@@ -19,6 +19,7 @@ package main
 import (
 	"flag"
 	"os"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 	"strings"
 
 	redisv1beta1 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta1"
@@ -29,6 +30,7 @@ import (
 	"github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllers/redissentinel"
 	intctrlutil "github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllerutil"
 	"github.com/OT-CONTAINER-KIT/redis-operator/pkg/k8sutils"
+	coreWebhook "github.com/OT-CONTAINER-KIT/redis-operator/pkg/webhook"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -176,6 +178,10 @@ func main() {
 			setupLog.Error(err, "unable to create webhook", "webhook", "RedisSentinel")
 			os.Exit(1)
 		}
+
+		wblog := ctrl.Log.WithName("webhook").WithName("PodAffiniytMutate")
+		mgr.GetWebhookServer().Register("/mutate-core-v1-pod", &webhook.Admission{
+			Handler: coreWebhook.NewPodAffiniytMutate(mgr.GetClient(), admission.NewDecoder(scheme), wblog)})
 	}
 	// +kubebuilder:scaffold:builder
 

pkg/controllers/rediscluster/rediscluster_controller.go

@@ -73,6 +73,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 
 	// Check if the cluster is downscaled
 	if leaderCount := k8sutils.CheckRedisNodeCount(ctx, r.K8sClient, instance, "leader"); leaderReplicas < leaderCount {
+		if !(r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") && r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-follower")) {
+			return intctrlutil.Reconciled()
+		}
+
 		logger.Info("Redis cluster is downscaling...", "Current.LeaderReplicas", leaderCount, "Desired.LeaderReplicas", leaderReplicas)
 		for shardIdx := leaderCount - 1; shardIdx >= leaderReplicas; shardIdx-- {
 			logger.Info("Remove the shard", "Shard.Index", shardIdx)
@@ -83,7 +87,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 				// lastLeaderPod is slaving right now Make it the master Pod
 				// We have to bring a manual failover here to make it a leaderPod
 				// clusterFailover should also include the clusterReplicate since we have to map the followers to new leader
-				k8sutils.ClusterFailover(ctx, r.K8sClient, instance)
+				logger.Info("Cluster Failover is initiated", "Shard.Index", shardIdx)
+				if err = k8sutils.ClusterFailover(ctx, r.K8sClient, instance); err != nil {
+					logger.Error(err, "Failed to initiate cluster failover")
+					return intctrlutil.RequeueWithError(ctx, err, "")
+				}
 			}
 			// Step 1 Remove the Follower Node
 			k8sutils.RemoveRedisFollowerNodesFromCluster(ctx, r.K8sClient, instance)