Protect against initialization race and switch ETCD images (#388)

* goimports fix ordering

* Make sure we can list clusters to ensure storage is up

* Use k8s' ETCD versioned image, and use cached images if possible

clusterctl/clusterdeployer/clusterapiservertemplate.go

@@ -169,8 +169,7 @@ spec:
       terminationGracePeriodSeconds: 10
       containers:
       - name: etcd
-        image: quay.io/coreos/etcd:latest
-        imagePullPolicy: Always
+        image: k8s.gcr.io/etcd:3.1.12
         resources:
           requests:
             cpu: 100m

clusterctl/clusterdeployer/clusterclient.go

@@ -22,15 +22,16 @@ import (
 	"os/exec"
 
 	"fmt"
+	"strings"
+	"time"
+
 	"github.com/golang/glog"
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 	"sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset"
 	"sigs.k8s.io/cluster-api/pkg/clientcmd"
 	"sigs.k8s.io/cluster-api/pkg/util"
-	"strings"
-	"time"
 )
 
 const (
@@ -203,6 +204,7 @@ func (c *clusterClient) waitForKubectlApply(manifest string) error {
 }
 
 func waitForClusterResourceReady(cs clientset.Interface) error {
+	deadline := time.Now().Add(TimeoutResourceReady)
 	err := util.PollImmediate(RetryIntervalResourceReady, TimeoutResourceReady, func() (bool, error) {
 		glog.V(2).Info("Waiting for Cluster v1alpha resources to become available...")
 		_, err := cs.Discovery().ServerResourcesForGroupVersion("cluster.k8s.io/v1alpha1")
@@ -212,7 +214,18 @@ func waitForClusterResourceReady(cs clientset.Interface) error {
 		return false, nil
 	})
 
-	return err
+	if err != nil {
+		return err
+	}
+	timeout := time.Until(deadline)
+	return util.PollImmediate(RetryIntervalResourceReady, timeout, func() (bool, error) {
+		glog.V(2).Info("Waiting for Cluster v1alpha resources to be listable...")
+		_, err := cs.ClusterV1alpha1().Clusters(apiv1.NamespaceDefault).List(metav1.ListOptions{})
+		if err == nil {
+			return true, nil
+		}
+		return false, nil
+	})
 }
 
 func waitForMachineReady(cs clientset.Interface, machine *clusterv1.Machine) error {