From 86d80070c81d72da727d5f52369f8c86581c9497 Mon Sep 17 00:00:00 2001 From: Madhav Bhargava Date: Mon, 22 Jul 2024 16:16:25 +0200 Subject: [PATCH] Fixes pre-deploy step having incorrect desired labels and adds a temp fix for race condition (#835) * check if operation reconcile is not present and only then reconcile in custodian * desired pods labels were incorrect, it should only check new + old labels --- controllers/custodian/reconciler.go | 8 ++++++++ controllers/predicate/predicate.go | 15 ++++++--------- controllers/utils/reconciler.go | 8 ++++++++ pkg/component/etcd/statefulset/statefulset.go | 2 +- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/controllers/custodian/reconciler.go b/controllers/custodian/reconciler.go index 948a31b7d..a42ce9677 100644 --- a/controllers/custodian/reconciler.go +++ b/controllers/custodian/reconciler.go @@ -20,6 +20,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" + ctrlutils "github.com/gardener/etcd-druid/controllers/utils" "github.com/gardener/etcd-druid/pkg/health/status" "github.com/gardener/etcd-druid/pkg/utils" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" @@ -74,6 +75,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String()) + if ctrlutils.HasOperationAnnotation(etcd) { + logger.Info("Skipping custodian-reconcile as etcd still has a reconcile annotation set on it. Will retry after 10s.") + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, nil + } + if etcd.Status.LastError != nil && *etcd.Status.LastError != "" { logger.Info("Requeue item because of last error", "namespace", etcd.Namespace, "name", etcd.Name, "lastError", *etcd.Status.LastError) return ctrl.Result{ diff --git a/controllers/predicate/predicate.go b/controllers/predicate/predicate.go index 260f69587..e61c23311 100644 --- a/controllers/predicate/predicate.go +++ b/controllers/predicate/predicate.go @@ -18,7 +18,8 @@ import ( "reflect" "strings" - v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/etcd-druid/controllers/utils" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" coordinationv1 "k8s.io/api/coordination/v1" @@ -31,21 +32,17 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" ) -func hasOperationAnnotation(obj client.Object) bool { - return obj.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile -} - // HasOperationAnnotation is a predicate for the operation annotation. func HasOperationAnnotation() predicate.Predicate { return predicate.Funcs{ CreateFunc: func(event event.CreateEvent) bool { - return hasOperationAnnotation(event.Object) + return utils.HasOperationAnnotation(event.Object) }, UpdateFunc: func(event event.UpdateEvent) bool { - return hasOperationAnnotation(event.ObjectNew) + return utils.HasOperationAnnotation(event.ObjectNew) }, GenericFunc: func(event event.GenericEvent) bool { - return hasOperationAnnotation(event.Object) + return utils.HasOperationAnnotation(event.Object) }, DeleteFunc: func(event event.DeleteEvent) bool { return true @@ -127,7 +124,7 @@ func EtcdReconciliationFinished(ignoreOperationAnnotation bool) predicate.Predic condition := *etcd.Status.ObservedGeneration == etcd.Generation if !ignoreOperationAnnotation { - condition = condition && !hasOperationAnnotation(etcd) + condition = condition && !utils.HasOperationAnnotation(etcd) } return condition diff --git a/controllers/utils/reconciler.go b/controllers/utils/reconciler.go index bfdfd5366..d77035ac7 100644 --- a/controllers/utils/reconciler.go +++ b/controllers/utils/reconciler.go @@ -17,6 +17,9 @@ package utils import ( "path/filepath" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/gardener/etcd-druid/pkg/common" "github.com/gardener/gardener/pkg/utils/imagevector" ) @@ -41,3 +44,8 @@ func CreateImageVector() (imagevector.ImageVector, error) { } return imageVector, nil } + +// HasOperationAnnotation checks if the given object has the operation annotation and its value is set to Reconcile. +func HasOperationAnnotation(obj client.Object) bool { + return obj.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile +} diff --git a/pkg/component/etcd/statefulset/statefulset.go b/pkg/component/etcd/statefulset/statefulset.go index fddc16bb6..e0153a1ae 100644 --- a/pkg/component/etcd/statefulset/statefulset.go +++ b/pkg/component/etcd/statefulset/statefulset.go @@ -299,7 +299,7 @@ func (c *component) waitUntilPodsHaveDesiredLabels(ctx context.Context, etcd *dr if err := c.client.Get(ctx, client.ObjectKey{Name: podName, Namespace: etcd.Namespace}, pod); err != nil { return false, err } - if !utils.ContainsAllDesiredLabels(pod.Labels, utils.MergeStringMaps(c.values.PodLabels, c.values.AdditionalPodLabels)) { + if !utils.ContainsAllDesiredLabels(pod.Labels, c.values.PodLabels) { return false, nil } }