Skip to content

Commit

Permalink
Merge pull request #3919 from fabriziopandini/fix-reconcile-etcd
Browse files Browse the repository at this point in the history
🐛 Prevents reconcileEtcdMember to remove etcd members when etcd starts slowly
  • Loading branch information
k8s-ci-robot authored Nov 16, 2020
2 parents e9f3ec5 + dd2b013 commit f926f16
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 1 deletion.
7 changes: 7 additions & 0 deletions controlplane/kubeadm/controllers/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,13 @@ func (r *KubeadmControlPlaneReconciler) reconcileEtcdMembers(ctx context.Context
return ctrl.Result{}, nil
}

// If there are provisioning machines (machines without a node yet), return.
for _, machine := range controlPlane.Machines {
if machine.Status.NodeRef == nil {
return ctrl.Result{}, nil
}
}

// Potential inconsistencies between the list of members and the list of machines/nodes are
// surfaced using the EtcdClusterHealthyCondition; if this condition is true, meaning no inconsistencies exists, return early.
if conditions.IsTrue(controlPlane.KCP, controlplanev1.EtcdClusterHealthyCondition) {
Expand Down
8 changes: 7 additions & 1 deletion controlplane/kubeadm/internal/workload_cluster_etcd.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package internal

import (
"context"
"fmt"

"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -59,6 +60,11 @@ func (w *Workload) ReconcileEtcdMembers(ctx context.Context) ([]string, error) {
// Check if any member's node is missing from workload cluster
// If any, delete it with best effort
for _, member := range members {
// If this member is just added, it has a empty name until the etcd pod starts. Ignore it.
if member.Name == "" {
continue
}

isFound := false
for _, node := range controlPlaneNodes.Items {
if member.Name == node.Name {
Expand All @@ -70,7 +76,7 @@ func (w *Workload) ReconcileEtcdMembers(ctx context.Context) ([]string, error) {
if isFound {
continue
}
removedMembers = append(removedMembers, member.Name)
removedMembers = append(removedMembers, fmt.Sprintf("%d (Name: %s)", member.ID, member.Name))
if err := w.removeMemberForNode(ctx, member.Name); err != nil {
errs = append(errs, err)
}
Expand Down

0 comments on commit f926f16

Please sign in to comment.