From d00d8938d8b3149dc2cd1c6b80f1e51a935cb3ef Mon Sep 17 00:00:00 2001 From: iamabhishek-dubey Date: Mon, 16 Jan 2023 23:36:47 +0530 Subject: [PATCH 1/4] Added code for recreate statefulsets Signed-off-by: iamabhishek-dubey --- example/eks-cluster.yaml | 2 ++ k8sutils/statefulset.go | 23 ++++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/example/eks-cluster.yaml b/example/eks-cluster.yaml index c2481d46c..360fdc0dc 100644 --- a/example/eks-cluster.yaml +++ b/example/eks-cluster.yaml @@ -20,5 +20,7 @@ addons: - name: coredns - name: kube-proxy - name: ebs-csi-driver + attachPolicyARNs: + - arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy iam: withOIDC: true diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index 7635ace88..eadb55d89 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -7,13 +7,15 @@ import ( redisv1beta1 "redis-operator/api/v1beta1" "sort" "strconv" + "strings" "github.com/banzaicloud/k8s-objectmatcher/patch" "github.com/go-logr/logr" + "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ) @@ -70,7 +72,7 @@ func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, params logger.Error(err, "Unable to patch redis statefulset with comparison object") return err } - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return createStatefulSet(namespace, statefulSetDef) } return err @@ -522,13 +524,24 @@ func createStatefulSet(namespace string, stateful *appsv1.StatefulSet) error { // updateStatefulSet is a method to update statefulset in Kubernetes func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet) error { logger := statefulSetLogger(namespace, stateful.Name) - // logger.Info(fmt.Sprintf("Setting Statefulset to the following: %s", stateful)) _, err := generateK8sClient().AppsV1().StatefulSets(namespace).Update(context.TODO(), stateful, metav1.UpdateOptions{}) + sErr, ok := err.(*apierrors.StatusError) + if ok && sErr.ErrStatus.Code == 422 && sErr.ErrStatus.Reason == metav1.StatusReasonInvalid { + failMsg := make([]string, len(sErr.ErrStatus.Details.Causes)) + for messageCount, cause := range sErr.ErrStatus.Details.Causes { + failMsg[messageCount] = cause.Message + } + logger.Info("recreating StatefulSet because the update operation wasn't possible", "reason", strings.Join(failMsg, ", ")) + propagationPolicy := metav1.DeletePropagationForeground + if err := generateK8sClient().AppsV1().StatefulSets(namespace).Delete(context.TODO(), stateful.GetName(), metav1.DeleteOptions{PropagationPolicy: &propagationPolicy}); err != nil { + return errors.Wrap(err, "failed to delete StatefulSet to avoid forbidden action") + } + } if err != nil { - logger.Error(err, "Redis stateful update failed") + logger.Error(err, "Redis statefulset update failed") return err } - logger.Info("Redis stateful successfully updated ") + logger.Info("Redis statefulset successfully updated ") return nil } From e99bfb075ce401ef95fc68ff624fff663e8ddf36 Mon Sep 17 00:00:00 2001 From: iamabhishek-dubey Date: Mon, 16 Jan 2023 23:50:25 +0530 Subject: [PATCH 2/4] Added logic for standalone and cluster recreate Signed-off-by: iamabhishek-dubey --- k8sutils/redis-cluster.go | 3 +++ k8sutils/redis-standalone.go | 3 +++ k8sutils/statefulset.go | 31 +++++++++++++++++-------------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/k8sutils/redis-cluster.go b/k8sutils/redis-cluster.go index 2ba427436..2a5047525 100644 --- a/k8sutils/redis-cluster.go +++ b/k8sutils/redis-cluster.go @@ -47,6 +47,9 @@ func generateRedisClusterParams(cr *redisv1beta1.RedisCluster, replicas int32, e if externalConfig != nil { res.ExternalConfig = externalConfig } + if _, found := cr.ObjectMeta.GetAnnotations()["redis.opstreelabs.in/recreate-statefulset"]; found { + res.RecreateStatefulSet = true + } return res } diff --git a/k8sutils/redis-standalone.go b/k8sutils/redis-standalone.go index 9c51f2751..9b9c02336 100644 --- a/k8sutils/redis-standalone.go +++ b/k8sutils/redis-standalone.go @@ -93,6 +93,9 @@ func generateRedisStandaloneParams(cr *redisv1beta1.Redis) statefulSetParameters if cr.Spec.ServiceAccountName != nil { res.ServiceAccountName = cr.Spec.ServiceAccountName } + if _, found := cr.ObjectMeta.GetAnnotations()["redis.opstreelabs.in/recreate-statefulset"]; found { + res.RecreateStatefulSet = true + } return res } diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index eadb55d89..0cdfa8c70 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -39,6 +39,7 @@ type statefulSetParameters struct { ExternalConfig *string ServiceAccountName *string UpdateStrategy appsv1.StatefulSetUpdateStrategy + RecreateStatefulSet bool } // containerParameters will define container input params @@ -77,11 +78,11 @@ func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, params } return err } - return patchStatefulSet(storedStateful, statefulSetDef, namespace) + return patchStatefulSet(storedStateful, statefulSetDef, namespace, params.RecreateStatefulSet) } // patchStateFulSet will patch Redis Kubernetes StateFulSet -func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.StatefulSet, namespace string) error { +func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.StatefulSet, namespace string, recreateStateFulSet bool) error { logger := statefulSetLogger(namespace, storedStateful.Name) // We want to try and keep this atomic as possible. @@ -178,7 +179,7 @@ func patchStatefulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St logger.Error(err, "Unable to patch redis statefulset with comparison object") return err } - return updateStatefulSet(namespace, newStateful) + return updateStatefulSet(namespace, newStateful, recreateStateFulSet) } logger.Info("Reconciliation Complete, no Changes required.") return nil @@ -522,19 +523,21 @@ func createStatefulSet(namespace string, stateful *appsv1.StatefulSet) error { } // updateStatefulSet is a method to update statefulset in Kubernetes -func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet) error { +func updateStatefulSet(namespace string, stateful *appsv1.StatefulSet, recreateStateFulSet bool) error { logger := statefulSetLogger(namespace, stateful.Name) _, err := generateK8sClient().AppsV1().StatefulSets(namespace).Update(context.TODO(), stateful, metav1.UpdateOptions{}) - sErr, ok := err.(*apierrors.StatusError) - if ok && sErr.ErrStatus.Code == 422 && sErr.ErrStatus.Reason == metav1.StatusReasonInvalid { - failMsg := make([]string, len(sErr.ErrStatus.Details.Causes)) - for messageCount, cause := range sErr.ErrStatus.Details.Causes { - failMsg[messageCount] = cause.Message - } - logger.Info("recreating StatefulSet because the update operation wasn't possible", "reason", strings.Join(failMsg, ", ")) - propagationPolicy := metav1.DeletePropagationForeground - if err := generateK8sClient().AppsV1().StatefulSets(namespace).Delete(context.TODO(), stateful.GetName(), metav1.DeleteOptions{PropagationPolicy: &propagationPolicy}); err != nil { - return errors.Wrap(err, "failed to delete StatefulSet to avoid forbidden action") + if recreateStateFulSet { + sErr, ok := err.(*apierrors.StatusError) + if ok && sErr.ErrStatus.Code == 422 && sErr.ErrStatus.Reason == metav1.StatusReasonInvalid { + failMsg := make([]string, len(sErr.ErrStatus.Details.Causes)) + for messageCount, cause := range sErr.ErrStatus.Details.Causes { + failMsg[messageCount] = cause.Message + } + logger.Info("recreating StatefulSet because the update operation wasn't possible", "reason", strings.Join(failMsg, ", ")) + propagationPolicy := metav1.DeletePropagationForeground + if err := generateK8sClient().AppsV1().StatefulSets(namespace).Delete(context.TODO(), stateful.GetName(), metav1.DeleteOptions{PropagationPolicy: &propagationPolicy}); err != nil { + return errors.Wrap(err, "failed to delete StatefulSet to avoid forbidden action") + } } } if err != nil { From 561ca742c7cc46e02f1024c515149ebe297685ba Mon Sep 17 00:00:00 2001 From: iamabhishek-dubey Date: Tue, 17 Jan 2023 00:15:11 +0530 Subject: [PATCH 3/4] Added push command for buildx Signed-off-by: iamabhishek-dubey --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 28b9a94a2..986346ca3 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ docker-build: # Push the docker image docker-push: - docker push ${IMG} + docker buildx build --push --platform="linux/arm64,linux/amd64" -t ${IMG} . # Download controller-gen locally if necessary CONTROLLER_GEN = $(shell pwd)/bin/controller-gen From 257434ce6faa9f2ba1c7887f1e3d659e037e13b1 Mon Sep 17 00:00:00 2001 From: iamabhishek-dubey Date: Tue, 17 Jan 2023 14:59:49 +0530 Subject: [PATCH 4/4] Added example for recreation of sts Signed-off-by: iamabhishek-dubey --- example/eks-cluster.yaml | 2 +- example/recreate-statefulset/clusterd.yaml | 29 ++++++++++++++++++++ example/recreate-statefulset/standalone.yaml | 26 ++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 example/recreate-statefulset/clusterd.yaml create mode 100644 example/recreate-statefulset/standalone.yaml diff --git a/example/eks-cluster.yaml b/example/eks-cluster.yaml index 360fdc0dc..4474537e3 100644 --- a/example/eks-cluster.yaml +++ b/example/eks-cluster.yaml @@ -19,7 +19,7 @@ addons: - name: vpc-cni - name: coredns - name: kube-proxy - - name: ebs-csi-driver + - name: aws-ebs-csi-driver attachPolicyARNs: - arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy iam: diff --git a/example/recreate-statefulset/clusterd.yaml b/example/recreate-statefulset/clusterd.yaml new file mode 100644 index 000000000..f944f8b2c --- /dev/null +++ b/example/recreate-statefulset/clusterd.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta1 +kind: RedisCluster +metadata: + name: redis-cluster + annotations: + redis.opstreelabs.in/recreate-statefulset: "true" +spec: + clusterSize: 3 + clusterVersion: v7 + securityContext: + runAsUser: 1000 + fsGroup: 1000 + persistenceEnabled: true + kubernetesConfig: + image: quay.io/opstree/redis:v7.0.5 + imagePullPolicy: IfNotPresent + redisExporter: + enabled: false + image: quay.io/opstree/redis-exporter:v1.44.0 + storage: + volumeClaimTemplate: + spec: + # storageClassName: standard + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + priorityClassName: priority-100 diff --git a/example/recreate-statefulset/standalone.yaml b/example/recreate-statefulset/standalone.yaml new file mode 100644 index 000000000..03b62796f --- /dev/null +++ b/example/recreate-statefulset/standalone.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta1 +kind: Redis +metadata: + name: redis-standalone + annotations: + redis.opstreelabs.in/recreate-statefulset: "true" +spec: + kubernetesConfig: + image: quay.io/opstree/redis:v7.0.5 + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 1000 + fsGroup: 1000 + storage: + volumeClaimTemplate: + spec: + # storageClassName: standard + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + redisExporter: + enabled: false + image: quay.io/opstree/redis-exporter:v1.44.0 + priorityClassName: system-cluster-critical