[Development][Add] Added recreation logic for statefulset (#411)

* Added code for recreate statefulsets

* Added logic for standalone and cluster recreate

* Added push command for buildx

* Added example for recreation of sts

Signed-off-by: iamabhishek-dubey <[email protected]>

Makefile

@@ -82,7 +82,7 @@ docker-build:
 
 # Push the docker image
 docker-push:
-	docker push ${IMG}
+	docker buildx build --push --platform="linux/arm64,linux/amd64" -t ${IMG} .
 
 # Download controller-gen locally if necessary
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen

example/eks-cluster.yaml

@@ -19,6 +19,8 @@ addons:
   - name: vpc-cni
   - name: coredns
   - name: kube-proxy
-  - name: ebs-csi-driver
+  - name: aws-ebs-csi-driver
+    attachPolicyARNs:
+      - arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy
 iam:
   withOIDC: true

example/recreate-statefulset/clusterd.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta1
+kind: RedisCluster
+metadata:
+  name: redis-cluster
+  annotations:
+    redis.opstreelabs.in/recreate-statefulset: "true"
+spec:
+  clusterSize: 3
+  clusterVersion: v7
+  securityContext:
+    runAsUser: 1000
+    fsGroup: 1000
+  persistenceEnabled: true
+  kubernetesConfig:
+    image: quay.io/opstree/redis:v7.0.5
+    imagePullPolicy: IfNotPresent
+  redisExporter:
+    enabled: false
+    image: quay.io/opstree/redis-exporter:v1.44.0
+  storage:
+    volumeClaimTemplate:
+      spec:
+        # storageClassName: standard
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi
+  priorityClassName: priority-100

example/recreate-statefulset/standalone.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta1
+kind: Redis
+metadata:
+  name: redis-standalone
+  annotations:
+    redis.opstreelabs.in/recreate-statefulset: "true"
+spec:
+  kubernetesConfig:
+    image: quay.io/opstree/redis:v7.0.5
+    imagePullPolicy: IfNotPresent
+  securityContext:
+    runAsUser: 1000
+    fsGroup: 1000
+  storage:
+    volumeClaimTemplate:
+      spec:
+        # storageClassName: standard
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi
+  redisExporter:
+    enabled: false
+    image: quay.io/opstree/redis-exporter:v1.44.0
+  priorityClassName: system-cluster-critical

k8sutils/redis-cluster.go

@@ -47,6 +47,9 @@ func generateRedisClusterParams(cr *redisv1beta1.RedisCluster, replicas int32, e
 	if externalConfig != nil {
 		res.ExternalConfig = externalConfig
 	}
+	if _, found := cr.ObjectMeta.GetAnnotations()["redis.opstreelabs.in/recreate-statefulset"]; found {
+		res.RecreateStatefulSet = true
+	}
 	return res
 }
 

k8sutils/redis-standalone.go

@@ -93,6 +93,9 @@ func generateRedisStandaloneParams(cr *redisv1beta1.Redis) statefulSetParameters
 	if cr.Spec.ServiceAccountName != nil {
 		res.ServiceAccountName = cr.Spec.ServiceAccountName
 	}
+	if _, found := cr.ObjectMeta.GetAnnotations()["redis.opstreelabs.in/recreate-statefulset"]; found {
+		res.RecreateStatefulSet = true
+	}
 	return res
 }
 

k8sutils/statefulset.go

@@ -7,13 +7,15 @@ import (
 	redisv1beta1 "redis-operator/api/v1beta1"
 	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/banzaicloud/k8s-objectmatcher/patch"
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	"k8s.io/apimachinery/pkg/api/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 )
@@ -37,6 +39,7 @@ type statefulSetParameters struct {
 	ExternalConfig        *string
 	ServiceAccountName    *string
 	UpdateStrategy        appsv1.StatefulSetUpdateStrategy
+	RecreateStatefulSet   bool
 }
 
 // containerParameters will define container input params
@@ -70,16 +73,16 @@ func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, params
 			logger.Error(err, "Unable to patch redis statefulset with comparison object")
 			return err
 		}
-		if errors.IsNotFound(err) {
+		if apierrors.IsNotFound(err) {
 			return createStatefulSet(namespace, statefulSetDef)
 		}
 		return err
 	}
-	return patchStatefulSet(storedStateful, statefulSetDef, namespace)
+	return patchStatefulSet(storedStateful, statefulSetDef, namespace, params.RecreateStatefulSet)
 }
 
 // patchStateFulSet will patch Redis Kubernetes StateFulSet
-func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.StatefulSet, namespace string) error {
+func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.StatefulSet, namespace string, recreateStateFulSet bool) error {
 	logger := statefulSetLogger(namespace, storedStateful.Name)
 
 	// We want to try and keep this atomic as possible.
@@ -176,7 +179,7 @@ func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St
 			logger.Error(err, "Unable to patch redis statefulset with comparison object")
 			return err
 		}
-		return updateStatefulSet(namespace, newStateful)
+		return updateStatefulSet(namespace, newStateful, recreateStateFulSet)
 	}
 	logger.Info("Reconciliation Complete, no Changes required.")
 	return nil
@@ -520,15 +523,28 @@ func createStatefulSet(namespace string, stateful *appsv1.StatefulSet) error {
 }
 
 // updateStatefulSet is a method to update statefulset in Kubernetes
-func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet) error {
+func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet, recreateStateFulSet bool) error {
 	logger := statefulSetLogger(namespace, stateful.Name)
-	// logger.Info(fmt.Sprintf("Setting Statefulset to the following: %s", stateful))
 	_, err := generateK8sClient().AppsV1().StatefulSets(namespace).Update(context.TODO(), stateful, metav1.UpdateOptions{})
+	if recreateStateFulSet {
+		sErr, ok := err.(*apierrors.StatusError)
+		if ok && sErr.ErrStatus.Code == 422 && sErr.ErrStatus.Reason == metav1.StatusReasonInvalid {
+			failMsg := make([]string, len(sErr.ErrStatus.Details.Causes))
+			for messageCount, cause := range sErr.ErrStatus.Details.Causes {
+				failMsg[messageCount] = cause.Message
+			}
+			logger.Info("recreating StatefulSet because the update operation wasn't possible", "reason", strings.Join(failMsg, ", "))
+			propagationPolicy := metav1.DeletePropagationForeground
+			if err := generateK8sClient().AppsV1().StatefulSets(namespace).Delete(context.TODO(), stateful.GetName(), metav1.DeleteOptions{PropagationPolicy: &propagationPolicy}); err != nil {
+				return errors.Wrap(err, "failed to delete StatefulSet to avoid forbidden action")
+			}
+		}
+	}
 	if err != nil {
-		logger.Error(err, "Redis stateful update failed")
+		logger.Error(err, "Redis statefulset update failed")
 		return err
 	}
-	logger.Info("Redis stateful successfully updated ")
+	logger.Info("Redis statefulset successfully updated ")
 	return nil
 }