Cluster reconciliation should reconcile exited ELB

actuators/actuators.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 
+	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -29,6 +30,10 @@ import (
 	"sigs.k8s.io/kind/pkg/cluster/nodes"
 )
 
+const (
+	containerRunningStatus = "running"
+)
+
 func getRole(machine *clusterv1.Machine) string {
 	// Figure out what kind of node we're making
 	labels := machine.GetLabels()
@@ -39,10 +44,12 @@ func getRole(machine *clusterv1.Machine) string {
 	return setValue
 }
 
-func getExternalLoadBalancerNode(clusterName string) (*nodes.Node, error) {
+func getExternalLoadBalancerNode(clusterName string, log logr.Logger) (*nodes.Node, error) {
+	log.Info("Getting external load balancer node for cluster", "cluster-name", clusterName)
 	elb, err := nodes.List(
 		fmt.Sprintf("label=%s=%s", constants.NodeRoleKey, constants.ExternalLoadBalancerNodeRoleValue),
 		fmt.Sprintf("label=%s=%s", constants.ClusterLabelKey, clusterName),
+		fmt.Sprintf("status=%s", containerRunningStatus),
 	)
 	if err != nil {
 		return nil, err
@@ -53,6 +60,7 @@ func getExternalLoadBalancerNode(clusterName string) (*nodes.Node, error) {
 	if len(elb) > 1 {
 		return nil, errors.New("too many external load balancers")
 	}
+	log.Info("External loadbalancer node for cluster", "cluster-name", clusterName, "elb", elb[0])
 	return &elb[0], nil
 }
 

actuators/cluster.go

@@ -29,7 +29,8 @@ type Cluster struct {
 
 // Reconcile setups an external load balancer for the cluster if needed
 func (c *Cluster) Reconcile(cluster *clusterv1.Cluster) error {
-	elb, err := getExternalLoadBalancerNode(cluster.Name)
+	c.Log.Info("Reconciling cluster", "cluster-namespace", cluster.Namespace, "cluster-name", cluster.Name)
+	elb, err := getExternalLoadBalancerNode(cluster.Name, c.Log)
 	if err != nil {
 		c.Log.Error(err, "Error getting external load balancer node")
 		return err

actuators/machine.go

@@ -92,7 +92,7 @@ func (m *Machine) Create(ctx context.Context, c *clusterv1.Cluster, machine *clu
 		}
 
 		m.Log.Info("Creating a brand new cluster")
-		elb, err := getExternalLoadBalancerNode(c.Name)
+		elb, err := getExternalLoadBalancerNode(c.Name, m.Log)
 		if err != nil {
 			m.Log.Error(err, "Error getting external load balancer node")
 			return err

kind/actions/kind.go

@@ -71,10 +71,34 @@ func AddControlPlane(clusterName, machineName, version string) (*nodes.Node, err
 	return controlPlane, nil
 }
 
+func removeExitedELBWithNameConflict(name string) error {
+	exitedLB, err := nodes.List(
+		fmt.Sprintf("label=%s=%s", constants.NodeRoleKey, constants.ExternalLoadBalancerNodeRoleValue),
+		fmt.Sprintf("name=%s", name),
+		fmt.Sprintf("status=exited"),
+	)
+
+	if err != nil {
+		return errors.Wrapf(err, "failed to list exited external load balancer nodes with name %q", name)
+	}
+
+	if len(exitedLB) > 0 {
+		// only one container with given name should exist, if any.
+		fmt.Printf("Removing exited ELB %q\n", exitedLB[0].Name())
+		return nodes.Delete(exitedLB[0])
+	}
+	return nil
+}
+
 // SetUpLoadBalancer creates a load balancer but does not configure it.
 func SetUpLoadBalancer(clusterName string) (*nodes.Node, error) {
 	clusterLabel := fmt.Sprintf("%s=%s", constants.ClusterLabelKey, clusterName)
 	name := fmt.Sprintf("%s-%s", clusterName, constants.ExternalLoadBalancerNodeRoleValue)
+	err := removeExitedELBWithNameConflict(name)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to delete exited load balancer node %q", name)
+	}
+
 	return nodes.CreateExternalLoadBalancerNode(
 		name,
 		loadbalancer.Image,