Fixes pre-deploy step having incorrect desired labels and adds a temp…

… fix for race condition (#835)

* check if operation reconcile is not present and only then reconcile in custodian

* desired pods labels were incorrect, it should only check new + old labels

controllers/custodian/reconciler.go

@@ -20,6 +20,7 @@ import (
 
 	druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1"
 
+	ctrlutils "github.com/gardener/etcd-druid/controllers/utils"
 	"github.com/gardener/etcd-druid/pkg/health/status"
 	"github.com/gardener/etcd-druid/pkg/utils"
 	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
@@ -74,6 +75,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 
 	logger := r.logger.WithValues("etcd", kutil.Key(etcd.Namespace, etcd.Name).String())
 
+	if ctrlutils.HasOperationAnnotation(etcd) {
+		logger.Info("Skipping custodian-reconcile as etcd still has a reconcile annotation set on it. Will retry after 10s.")
+		return ctrl.Result{
+			RequeueAfter: 10 * time.Second,
+		}, nil
+	}
+
 	if etcd.Status.LastError != nil && *etcd.Status.LastError != "" {
 		logger.Info("Requeue item because of last error", "namespace", etcd.Namespace, "name", etcd.Name, "lastError", *etcd.Status.LastError)
 		return ctrl.Result{

controllers/predicate/predicate.go

@@ -18,7 +18,8 @@ import (
 	"reflect"
 	"strings"
 
-	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	"github.com/gardener/etcd-druid/controllers/utils"
+
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	coordinationv1 "k8s.io/api/coordination/v1"
@@ -31,21 +32,17 @@ import (
 	druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1"
 )
 
-func hasOperationAnnotation(obj client.Object) bool {
-	return obj.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile
-}
-
 // HasOperationAnnotation is a predicate for the operation annotation.
 func HasOperationAnnotation() predicate.Predicate {
 	return predicate.Funcs{
 		CreateFunc: func(event event.CreateEvent) bool {
-			return hasOperationAnnotation(event.Object)
+			return utils.HasOperationAnnotation(event.Object)
 		},
 		UpdateFunc: func(event event.UpdateEvent) bool {
-			return hasOperationAnnotation(event.ObjectNew)
+			return utils.HasOperationAnnotation(event.ObjectNew)
 		},
 		GenericFunc: func(event event.GenericEvent) bool {
-			return hasOperationAnnotation(event.Object)
+			return utils.HasOperationAnnotation(event.Object)
 		},
 		DeleteFunc: func(event event.DeleteEvent) bool {
 			return true
@@ -127,7 +124,7 @@ func EtcdReconciliationFinished(ignoreOperationAnnotation bool) predicate.Predic
 		condition := *etcd.Status.ObservedGeneration == etcd.Generation
 
 		if !ignoreOperationAnnotation {
-			condition = condition && !hasOperationAnnotation(etcd)
+			condition = condition && !utils.HasOperationAnnotation(etcd)
 		}
 
 		return condition

controllers/utils/reconciler.go

@@ -17,6 +17,9 @@ package utils
 import (
 	"path/filepath"
 
+	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"github.com/gardener/etcd-druid/pkg/common"
 	"github.com/gardener/gardener/pkg/utils/imagevector"
 )
@@ -41,3 +44,8 @@ func CreateImageVector() (imagevector.ImageVector, error) {
 	}
 	return imageVector, nil
 }
+
+// HasOperationAnnotation checks if the given object has the operation annotation and its value is set to Reconcile.
+func HasOperationAnnotation(obj client.Object) bool {
+	return obj.GetAnnotations()[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationReconcile
+}

pkg/component/etcd/statefulset/statefulset.go

@@ -299,7 +299,7 @@ func (c *component) waitUntilPodsHaveDesiredLabels(ctx context.Context, etcd *dr
 			if err := c.client.Get(ctx, client.ObjectKey{Name: podName, Namespace: etcd.Namespace}, pod); err != nil {
 				return false, err
 			}
-			if !utils.ContainsAllDesiredLabels(pod.Labels, utils.MergeStringMaps(c.values.PodLabels, c.values.AdditionalPodLabels)) {
+			if !utils.ContainsAllDesiredLabels(pod.Labels, c.values.PodLabels) {
 				return false, nil
 			}
 		}