diff --git a/actuators/actuators.go b/actuators/actuators.go
index 025f1d8..7c26003 100644
--- a/actuators/actuators.go
+++ b/actuators/actuators.go
@@ -29,6 +29,10 @@ import (
 	"sigs.k8s.io/kind/pkg/cluster/nodes"
 )
 
+const (
+	containerRunningStatus = "running"
+)
+
 func getRole(machine *clusterv1.Machine) string {
 	// Figure out what kind of node we're making
 	labels := machine.GetLabels()
@@ -40,9 +44,11 @@ func getRole(machine *clusterv1.Machine) string {
 }
 
 func getExternalLoadBalancerNode(clusterName string) (*nodes.Node, error) {
+	fmt.Printf("Getting external load balancer node for cluster %q\n", clusterName)
 	elb, err := nodes.List(
 		fmt.Sprintf("label=%s=%s", constants.NodeRoleKey, constants.ExternalLoadBalancerNodeRoleValue),
 		fmt.Sprintf("label=%s=%s", constants.ClusterLabelKey, clusterName),
+		fmt.Sprintf("status=%s", containerRunningStatus),
 	)
 	if err != nil {
 		return nil, err
@@ -53,6 +59,7 @@ func getExternalLoadBalancerNode(clusterName string) (*nodes.Node, error) {
 	if len(elb) > 1 {
 		return nil, errors.New("too many external load balancers")
 	}
+	fmt.Printf("External loadbalancer node for cluster %q is %v\n", clusterName, elb[0])
 	return &elb[0], nil
 }
 
diff --git a/actuators/cluster.go b/actuators/cluster.go
index 9f92eb3..1d338cf 100644
--- a/actuators/cluster.go
+++ b/actuators/cluster.go
@@ -29,6 +29,7 @@ type Cluster struct {
 
 // Reconcile setups an external load balancer for the cluster if needed
 func (c *Cluster) Reconcile(cluster *clusterv1.Cluster) error {
+	fmt.Printf("Reconciling cluster %s/%s\n", cluster.Namespace, cluster.Name)
 	elb, err := getExternalLoadBalancerNode(cluster.Name)
 	if err != nil {
 		c.Log.Error(err, "Error getting external load balancer node")
diff --git a/kind/actions/kind.go b/kind/actions/kind.go
index 3eb689a..eb96d10 100644
--- a/kind/actions/kind.go
+++ b/kind/actions/kind.go
@@ -71,10 +71,34 @@ func AddControlPlane(clusterName, machineName, version string) (*nodes.Node, err
 	return controlPlane, nil
 }
 
+func removeExitedELBWithNameConflict(name string) error {
+	exitedLB, err := nodes.List(
+		fmt.Sprintf("label=%s=%s", constants.NodeRoleKey, constants.ExternalLoadBalancerNodeRoleValue),
+		fmt.Sprintf("name=%s", name),
+		fmt.Sprintf("status=exited"),
+	)
+
+	if err != nil {
+		return errors.Wrapf(err, "failed to list exited external load balancer nodes with name %q", name)
+	}
+
+	if len(exitedLB) > 0 {
+		// only one container with given name should exist, if any.
+		fmt.Printf("Removing exited ELB %q\n", exitedLB[0].Name())
+		return nodes.Delete(exitedLB[0])
+	}
+	return nil
+}
+
 // SetUpLoadBalancer creates a load balancer but does not configure it.
 func SetUpLoadBalancer(clusterName string) (*nodes.Node, error) {
 	clusterLabel := fmt.Sprintf("%s=%s", constants.ClusterLabelKey, clusterName)
 	name := fmt.Sprintf("%s-%s", clusterName, constants.ExternalLoadBalancerNodeRoleValue)
+	err := removeExitedELBWithNameConflict(name)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to delete exited load balancer node %q", name)
+	}
+
 	return nodes.CreateExternalLoadBalancerNode(
 		name,
 		loadbalancer.Image,