Working vsphere clusterctl example

cloud/vsphere/cmd/vsphere-machine-controller/Makefile

@@ -18,7 +18,7 @@ GCR_BUCKET = k8s-cluster-api
 PREFIX = gcr.io/$(GCR_BUCKET)
 DEV_PREFIX ?= gcr.io/$(shell gcloud config get-value project)
 NAME = vsphere-machine-controller
-TAG = 0.0.2
+TAG = 0.0.3
 
 image:
 	docker build -t "$(PREFIX)/$(NAME):$(TAG)" -f ./Dockerfile ../../../..

clusterctl/clusterdeployer/clusterapiserver.go

@@ -29,7 +29,7 @@ import (
 	"k8s.io/client-go/util/cert/triple"
 )
 
-var apiServerImage = "gcr.io/k8s-cluster-api/cluster-apiserver:0.0.3"
+var apiServerImage = "gcr.io/k8s-cluster-api/cluster-apiserver:0.0.5"
 
 func init() {
 	if img, ok := os.LookupEnv("CLUSTER_API_SERVER_IMAGE"); ok {

clusterctl/clusterdeployer/clusterclient.go

@@ -172,6 +172,11 @@ func (c *clusterClient) waitForKubectlApply(manifest string) error {
 		err := c.kubectlApply(manifest)
 		if err != nil {
 			if strings.Contains(err.Error(), "connection refused") {
+				glog.V(4).Infof("Waiting for kubectl apply... server not yet available: %v", err)
+				return false, nil
+			}
+			if strings.Contains(err.Error(), "unable to recognize") {
+				glog.V(4).Infof("Waiting for kubectl apply... api not yet available: %v", err)
 				return false, nil
 			}
 			return false, err
@@ -197,7 +202,7 @@ func waitForClusterResourceReady(cs clientset.Interface) error {
 }
 
 func waitForMachineReady(cs clientset.Interface, machine *clusterv1.Machine) error {
-	err := util.Poll(500*time.Millisecond, 120*time.Second, func() (bool, error) {
+	err := util.Poll(time.Second, 5*time.Minute, func() (bool, error) {
 		glog.V(2).Infof("Waiting for Machine %v to become ready...", machine.Name)
 		m, err := cs.ClusterV1alpha1().Machines(apiv1.NamespaceDefault).Get(machine.Name, metav1.GetOptions{})
 		if err != nil {

clusterctl/cmd/create_cluster.go

@@ -23,9 +23,9 @@ import (
 	"github.com/spf13/cobra"
 	"io/ioutil"
 	"sigs.k8s.io/cluster-api/cloud/google"
+	"sigs.k8s.io/cluster-api/cloud/vsphere"
 	"sigs.k8s.io/cluster-api/clusterctl/clusterdeployer"
 	"sigs.k8s.io/cluster-api/clusterctl/clusterdeployer/minikube"
-	"sigs.k8s.io/cluster-api/errors"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 	"sigs.k8s.io/cluster-api/util"
 )
@@ -95,7 +95,7 @@ func init() {
 	createClusterCmd.Flags().StringVarP(&co.Machine, "machines", "m", "", "A yaml file containing machine object definition(s)")
 	createClusterCmd.Flags().StringVarP(&co.ProviderComponents, "provider-components", "p", "", "A yaml file containing cluster api provider controllers and supporting objects")
 	// TODO: Remove as soon as code allows https://github.com/kubernetes-sigs/cluster-api/issues/157
-	createClusterCmd.Flags().StringVarP(&co.Provider, "provider", "", "", "Which provider deployment logic to use (google/terraform)")
+	createClusterCmd.Flags().StringVarP(&co.Provider, "provider", "", "", "Which provider deployment logic to use (google/vsphere)")
 
 	// Optional flags
 	createClusterCmd.Flags().BoolVarP(&co.CleanupExternalCluster, "cleanup-external-cluster", "", true, "Whether to cleanup the external cluster after bootstrap")
@@ -143,10 +143,28 @@ func getProvider(provider string) (clusterdeployer.ProviderDeployer, error) {
 	switch provider {
 	case "google":
 		return google.NewMachineActuator(google.MachineActuatorParams{})
-	case "terraform":
-		// TODO: Actually hook up terraform
-		return nil, errors.NotImplementedError
+	case "vsphere":
+		t, err := vsphere.NewMachineActuator("", nil, "")
+		if err != nil {
+			return nil, err
+		}
+		return &vsphereAdapter{t}, nil
 	default:
 		return nil, fmt.Errorf("Unrecognized provider %v", provider)
 	}
 }
+
+// Adapt the vsphere methods calls since gcp/vsphere are not on the same page.
+// Long term, these providers should converge or the need for a provider will go away.
+// Whichever comes first.
+type vsphereAdapter struct {
+	*vsphere.VsphereClient
+}
+
+func (a *vsphereAdapter) GetIP(cluster *clusterv1.Cluster, machine *clusterv1.Machine) (string, error) {
+	return a.VsphereClient.GetIP(machine)
+}
+
+func (a *vsphereAdapter) GetKubeConfig(cluster *clusterv1.Cluster, master *clusterv1.Machine) (string, error) {
+	return a.VsphereClient.GetKubeConfig(master)
+}

clusterctl/examples/vsphere/.gitignore

@@ -0,0 +1,5 @@
+machines.yaml
+cluster.yaml
+provider-components.yaml
+vsphere_tmp
+vsphere_tmp.pub

clusterctl/examples/vsphere/README.md

@@ -0,0 +1,27 @@
+# Vsphere Example Files
+## Contents
+* *.yaml files - concrete example files that can be used as is.
+* *.yaml.template files - template example files that need values filled in before use.
+
+## Generation
+For convenience, a generation script which populates templates where possible.
+
+1. Run the generation script. This wil produce ```provider-components.yaml```
+```
+./generate-yaml.sh
+```
+2. Copy machines.yaml.template to machines.yaml and
+Manually edit ```terraformVariables``` for machines in machines.yaml
+```
+cp machines.yaml.template machines.yaml
+```
+
+3. Copy cluster.yaml.template to cluster.yaml and
+Manually edit ```providerConfig``` for the cluster in cluster.yaml
+```
+cp cluster.yaml.template cluster.yaml
+```
+
+## Manual Modification
+You may always manually curate files based on the examples provided.
+

clusterctl/examples/vsphere/cluster.yaml.template

@@ -0,0 +1,18 @@
+apiVersion: "cluster.k8s.io/v1alpha1"
+kind: Cluster
+metadata:
+  name: test1
+spec:
+    clusterNetwork:
+        services:
+            cidrBlocks: ["10.96.0.0/12"]
+        pods:
+            cidrBlocks: ["192.168.0.0/16"]
+        serviceDomain: "cluster.local"
+    providerConfig:
+      value:
+        apiVersion: "vsphereproviderconfig/v1alpha1"
+        kind: "VsphereClusterProviderConfig"
+        vsphereUser: ""
+        vspherePassword: ""
+        vsphereServer: ""

clusterctl/examples/vsphere/generate-yaml.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+set -e
+
+PROVIDERCOMPONENT_TEMPLATE_FILE=provider-components.yaml.template
+PROVIDERCOMPONENT_GENERATED_FILE=provider-components.yaml
+
+MACHINE_CONTROLLER_SSH_PUBLIC_FILE=vsphere_tmp.pub
+MACHINE_CONTROLLER_SSH_PUBLIC=
+MACHINE_CONTROLLER_SSH_PRIVATE_FILE=vsphere_tmp
+MACHINE_CONTROLLER_SSH_PRIVATE=
+MACHINE_CONTROLLER_SSH_HOME=~/.ssh/
+
+OVERWRITE=0
+
+SCRIPT=$(basename $0)
+while test $# -gt 0; do
+        case "$1" in
+          -h|--help)
+            echo "$SCRIPT - generates input yaml files for Cluster API on vSphere"
+            echo " "
+            echo "$SCRIPT [options]"
+            echo " "
+            echo "options:"
+            echo "-h, --help                show brief help"
+            echo "-f, --force-overwrite     if file to be generated already exists, force script to overwrite it"
+            exit 0
+            ;;
+          -f)
+            OVERWRITE=1
+            shift
+            ;;
+          --force-overwrite)
+            OVERWRITE=1
+            shift
+            ;;
+          *)
+            break
+            ;;
+        esac
+done
+
+if [ $OVERWRITE -ne 1 ] && [ -f $PROVIDERCOMPONENT_GENERATED_FILE ]; then
+  echo File $PROVIDERCOMPONENT_GENERATED_FILE already exists. Delete it manually before running this script.
+  exit 1
+fi
+
+if [ ! -f $MACHINE_CONTROLLER_SSH_PRIVATE_FILE ]; then
+  echo Generate SSH key files fo machine controller
+  ssh-keygen -t rsa -f $MACHINE_CONTROLLER_SSH_PRIVATE_FILE  -N ""
+fi
+
+# Copy file to home ssh directory till using vsphere GetIP logic that
+# does not assume the file at this location
+cp $MACHINE_CONTROLLER_SSH_PUBLIC_FILE $MACHINE_CONTROLLER_SSH_HOME
+cp $MACHINE_CONTROLLER_SSH_PRIVATE_FILE $MACHINE_CONTROLLER_SSH_HOME
+
+MACHINE_CONTROLLER_SSH_PUBLIC=$(cat $MACHINE_CONTROLLER_SSH_PUBLIC_FILE|base64 -w0)
+MACHINE_CONTROLLER_SSH_PRIVATE=$(cat $MACHINE_CONTROLLER_SSH_PRIVATE_FILE|base64 -w0)
+
+cat $PROVIDERCOMPONENT_TEMPLATE_FILE \
+  | sed -e "s/\$MACHINE_CONTROLLER_SSH_PUBLIC/$MACHINE_CONTROLLER_SSH_PUBLIC/" \
+  | sed -e "s/\$MACHINE_CONTROLLER_SSH_PRIVATE/$MACHINE_CONTROLLER_SSH_PRIVATE/" \
+  > $PROVIDERCOMPONENT_GENERATED_FILE

clusterctl/examples/vsphere/machines.yaml.template

@@ -0,0 +1,57 @@
+items:
+- apiVersion: "cluster.k8s.io/v1alpha1"
+  kind: Machine
+  metadata:
+    generateName: tf-master-
+    labels:
+      set: master
+  spec:
+    providerConfig:
+      value:
+        apiVersion: "vsphereproviderconfig/v1alpha1"
+        kind: "VsphereMachineProviderConfig"
+        vsphereMachine: "standard-master"
+        terraformVariables: [
+          "datacenter = \"\"",
+          "datastore = \"\"",
+          "resource_pool = \"\"",
+          "network = \"\"",
+          "num_cpus = \"2\"",
+          "memory = \"2048\"",
+          "vm_template = \"\"",
+          "disk_label = \"\"",
+          "disk_size = \"\"",
+          "virtual_machine_domain = \"\"",
+		]
+    versions:
+      kubelet: 1.10.1
+      controlPlane: 1.10.1
+    roles:
+    - Master
+- apiVersion: "cluster.k8s.io/v1alpha1"
+  kind: Machine
+  metadata:
+    generateName: tf-node-
+  spec:
+    providerConfig:
+      value:
+        apiVersion: "vsphereproviderconfig/v1alpha1"
+        kind: "VsphereMachineProviderConfig"
+        vsphereMachine: "standard-node"
+        terraformVariables: [
+          "datacenter = \"\"",
+          "datastore = \"\"",
+          "resource_pool = \"\"",
+          "network = \"\"",
+          "num_cpus = \"2\"",
+          "memory = \"2048\"",
+          "vm_template = \"\"",
+          "disk_label = \"\"",
+          "disk_size = \"\"",
+          "virtual_machine_domain = \"\"",
+	    ]
+    versions:
+      kubelet: 1.10.1
+      controlPlane: 1.10.1
+    roles:
+    - Node