diff --git a/clusterctl/clusterdeployer/clusterapiserver.go b/clusterctl/clusterdeployer/clusterapiserver.go index d4fe83e1ff46..25997097eea6 100644 --- a/clusterctl/clusterdeployer/clusterapiserver.go +++ b/clusterctl/clusterdeployer/clusterapiserver.go @@ -29,7 +29,7 @@ import ( "k8s.io/client-go/util/cert/triple" ) -var apiServerImage = "gcr.io/k8s-cluster-api/cluster-apiserver:0.0.3" +var apiServerImage = "gcr.io/k8s-cluster-api/cluster-apiserver:0.0.5" func init() { if img, ok := os.LookupEnv("CLUSTER_API_SERVER_IMAGE"); ok { diff --git a/clusterctl/clusterdeployer/clusterclient.go b/clusterctl/clusterdeployer/clusterclient.go index 1cada709b59f..edc465b2ae19 100644 --- a/clusterctl/clusterdeployer/clusterclient.go +++ b/clusterctl/clusterdeployer/clusterclient.go @@ -172,6 +172,11 @@ func (c *clusterClient) waitForKubectlApply(manifest string) error { err := c.kubectlApply(manifest) if err != nil { if strings.Contains(err.Error(), "connection refused") { + glog.V(4).Infof("Waiting for kubectl apply... server not yet available: %v", err) + return false, nil + } + if strings.Contains(err.Error(), "unable to recognize") { + glog.V(4).Infof("Waiting for kubectl apply... api not yet available: %v", err) return false, nil } return false, err @@ -197,7 +202,7 @@ func waitForClusterResourceReady(cs clientset.Interface) error { } func waitForMachineReady(cs clientset.Interface, machine *clusterv1.Machine) error { - err := util.Poll(500*time.Millisecond, 120*time.Second, func() (bool, error) { + err := util.Poll(time.Second, 5*time.Minute, func() (bool, error) { glog.V(2).Infof("Waiting for Machine %v to become ready...", machine.Name) m, err := cs.ClusterV1alpha1().Machines(apiv1.NamespaceDefault).Get(machine.Name, metav1.GetOptions{}) if err != nil { diff --git a/clusterctl/cmd/create_cluster.go b/clusterctl/cmd/create_cluster.go index 27bf9f636a03..c5224f36fe90 100644 --- a/clusterctl/cmd/create_cluster.go +++ b/clusterctl/cmd/create_cluster.go @@ -23,9 +23,9 @@ import ( "github.com/spf13/cobra" "io/ioutil" "sigs.k8s.io/cluster-api/cloud/google" + "sigs.k8s.io/cluster-api/cloud/vsphere" "sigs.k8s.io/cluster-api/clusterctl/clusterdeployer" "sigs.k8s.io/cluster-api/clusterctl/clusterdeployer/minikube" - "sigs.k8s.io/cluster-api/errors" clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" "sigs.k8s.io/cluster-api/util" ) @@ -95,7 +95,7 @@ func init() { createClusterCmd.Flags().StringVarP(&co.Machine, "machines", "m", "", "A yaml file containing machine object definition(s)") createClusterCmd.Flags().StringVarP(&co.ProviderComponents, "provider-components", "p", "", "A yaml file containing cluster api provider controllers and supporting objects") // TODO: Remove as soon as code allows https://github.com/kubernetes-sigs/cluster-api/issues/157 - createClusterCmd.Flags().StringVarP(&co.Provider, "provider", "", "", "Which provider deployment logic to use (google/terraform)") + createClusterCmd.Flags().StringVarP(&co.Provider, "provider", "", "", "Which provider deployment logic to use (google/vsphere)") // Optional flags createClusterCmd.Flags().BoolVarP(&co.CleanupExternalCluster, "cleanup-external-cluster", "", true, "Whether to cleanup the external cluster after bootstrap") @@ -143,10 +143,28 @@ func getProvider(provider string) (clusterdeployer.ProviderDeployer, error) { switch provider { case "google": return google.NewMachineActuator(google.MachineActuatorParams{}) - case "terraform": - // TODO: Actually hook up terraform - return nil, errors.NotImplementedError + case "vsphere": + t, err := vsphere.NewMachineActuator("", nil, "") + if err != nil { + return nil, err + } + return &vsphereAdapter{t}, nil default: return nil, fmt.Errorf("Unrecognized provider %v", provider) } } + +// Adapt the terraform methods calls since gcp/vsphere are not on the same page. +// Long term, these providers should converge or the need for a provider will go away. +// Whichever comes first. +type vsphereAdapter struct { + *vsphere.VsphereClient +} + +func (a *vsphereAdapter) GetIP(cluster *clusterv1.Cluster, machine *clusterv1.Machine) (string, error) { + return a.VsphereClient.GetIP(machine) +} + +func (a *vsphereAdapter) GetKubeConfig(cluster *clusterv1.Cluster, master *clusterv1.Machine) (string, error) { + return a.VsphereClient.GetKubeConfig(master) +} diff --git a/clusterctl/examples/vsphere/.gitignore b/clusterctl/examples/vsphere/.gitignore new file mode 100644 index 000000000000..1a6ceab92a2b --- /dev/null +++ b/clusterctl/examples/vsphere/.gitignore @@ -0,0 +1,4 @@ +machines.yaml +provider-components.yaml +vsphere_tmp +vsphere_tmp.pub \ No newline at end of file diff --git a/clusterctl/examples/vsphere/README.md b/clusterctl/examples/vsphere/README.md new file mode 100644 index 000000000000..81773ea9c290 --- /dev/null +++ b/clusterctl/examples/vsphere/README.md @@ -0,0 +1,22 @@ +# Vsphere Example Files +## Contents +* *.yaml files - concrete example files that can be used as is. +* *.yaml.template files - template example files that need values filled in before use. + +## Generation +For convenience, a generation script which populates templates where possible. + +1. Run the generation script. This wil produce ```provider-components.yaml``` +``` +./generate-yaml.sh +``` +2. Copy machines.yaml.template to machines.yaml. +``` +cp machines.yaml.template machines.yaml +``` + +3. Manually edit ```terraformVariables``` for machines in machines.yaml + +## Manual Modification +You may always manually curate files based on the examples provided. + diff --git a/clusterctl/examples/vsphere/cluster.yaml b/clusterctl/examples/vsphere/cluster.yaml new file mode 100644 index 000000000000..7d211634e0f7 --- /dev/null +++ b/clusterctl/examples/vsphere/cluster.yaml @@ -0,0 +1,15 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: Cluster +metadata: + name: test1 +spec: + clusterNetwork: + services: + cidrBlocks: ["10.96.0.0/12"] + pods: + cidrBlocks: ["192.168.0.0/16"] + serviceDomain: "cluster.local" + providerConfig: + value: + apiVersion: "terraformproviderconfig/v1alpha1" + kind: "TerraformProviderConfig" \ No newline at end of file diff --git a/clusterctl/examples/vsphere/generate-yaml.sh b/clusterctl/examples/vsphere/generate-yaml.sh new file mode 100755 index 000000000000..fc8edc9ac629 --- /dev/null +++ b/clusterctl/examples/vsphere/generate-yaml.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -e + +PROVIDERCOMPONENT_TEMPLATE_FILE=provider-components.yaml.template +PROVIDERCOMPONENT_GENERATED_FILE=provider-components.yaml + +MACHINE_CONTROLLER_SSH_PUBLIC_FILE=vsphere_tmp.pub +MACHINE_CONTROLLER_SSH_PUBLIC= +MACHINE_CONTROLLER_SSH_PRIVATE_FILE=vsphere_tmp +MACHINE_CONTROLLER_SSH_PRIVATE= +MACHINE_CONTROLLER_SSH_HOME=~/.ssh/ + +OVERWRITE=0 + +SCRIPT=$(basename $0) +while test $# -gt 0; do + case "$1" in + -h|--help) + echo "$SCRIPT - generates input yaml files for Cluster API on vSphere" + echo " " + echo "$SCRIPT [options]" + echo " " + echo "options:" + echo "-h, --help show brief help" + echo "-f, --force-overwrite if file to be generated already exists, force script to overwrite it" + exit 0 + ;; + -f) + OVERWRITE=1 + shift + ;; + --force-overwrite) + OVERWRITE=1 + shift + ;; + *) + break + ;; + esac +done + +if [ $OVERWRITE -ne 1 ] && [ -f $PROVIDERCOMPONENT_GENERATED_FILE ]; then + echo File $PROVIDERCOMPONENT_GENERATED_FILE already exists. Delete it manually before running this script. + exit 1 +fi + +if [ ! -f $MACHINE_CONTROLLER_SSH_PRIVATE_FILE ]; then + echo Generate SSH key files fo machine controller + ssh-keygen -t rsa -f $MACHINE_CONTROLLER_SSH_PRIVATE_FILE -N "" +fi + +# Copy file to home ssh directory till using vsphere GetIP logic that +# does not assume the file at this location +cp $MACHINE_CONTROLLER_SSH_PUBLIC_FILE $MACHINE_CONTROLLER_SSH_HOME +cp $MACHINE_CONTROLLER_SSH_PRIVATE_FILE $MACHINE_CONTROLLER_SSH_HOME + +MACHINE_CONTROLLER_SSH_PUBLIC=$(cat $MACHINE_CONTROLLER_SSH_PUBLIC_FILE|base64 -w0) +MACHINE_CONTROLLER_SSH_PRIVATE=$(cat $MACHINE_CONTROLLER_SSH_PRIVATE_FILE|base64 -w0) + +cat $PROVIDERCOMPONENT_TEMPLATE_FILE \ + | sed -e "s/\$MACHINE_CONTROLLER_SSH_PUBLIC/$MACHINE_CONTROLLER_SSH_PUBLIC/" \ + | sed -e "s/\$MACHINE_CONTROLLER_SSH_PRIVATE/$MACHINE_CONTROLLER_SSH_PRIVATE/" \ + > $PROVIDERCOMPONENT_GENERATED_FILE \ No newline at end of file diff --git a/clusterctl/examples/vsphere/machines.yaml.template b/clusterctl/examples/vsphere/machines.yaml.template new file mode 100644 index 000000000000..2e1b0ac2cd10 --- /dev/null +++ b/clusterctl/examples/vsphere/machines.yaml.template @@ -0,0 +1,62 @@ +items: +- apiVersion: "cluster.k8s.io/v1alpha1" + kind: Machine + metadata: + name: tf-test-master + labels: + set: master + spec: + providerConfig: + value: + apiVersion: "terraformproviderconfig/v1alpha1" + kind: "TerraformProviderConfig" + terraformMachine: "standard-master" + terraformVariables: [ + "user = \"\"", + "password = \"\"", + "vsphere_server = \"", + "datacenter = \"\"", + "datastore = \"\"", + "resource_pool = \"\"", + "network = \"\"", + "num_cpus = \"2\"", + "memory = \"2048\"", + "vm_template = \"\"", + "disk_label = \"\"", + "disk_size = \"\"", + "virtual_machine_domain = \"\"", + ] + versions: + kubelet: 1.10.1 + controlPlane: 1.10.1 + roles: + - Master +- apiVersion: "cluster.k8s.io/v1alpha1" + kind: Machine + metadata: + generateName: jesschen-tf-test- + spec: + providerConfig: + value: + apiVersion: "terraformproviderconfig/v1alpha1" + kind: "TerraformProviderConfig" + terraformMachine: "standard-node" + terraformVariables: [ + "user = \"\"", + "password = \"\"", + "vsphere_server = \"", + "datacenter = \"\"", + "datastore = \"\"", + "resource_pool = \"\"", + "network = \"\"", + "num_cpus = \"2\"", + "memory = \"2048\"", + "vm_template = \"\"", + "disk_label = \"\"", + "disk_size = \"\"", + "virtual_machine_domain = \"\"", + ] + versions: + kubelet: 1.9.7 + roles: + - Node diff --git a/clusterctl/examples/vsphere/provider-components.yaml.template b/clusterctl/examples/vsphere/provider-components.yaml.template new file mode 100644 index 000000000000..3515a77ad358 --- /dev/null +++ b/clusterctl/examples/vsphere/provider-components.yaml.template @@ -0,0 +1,397 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: clusterapi-controllers + namespace: default + labels: + api: clusterapi +spec: + replicas: 1 + template: + metadata: + labels: + api: clusterapi + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + key: node.alpha.kubernetes.io/notReady + operator: Exists + - effect: NoExecute + key: node.alpha.kubernetes.io/unreachable + operator: Exists + containers: + - name: controller-manager + image: gcr.io/k8s-cluster-api/controller-manager:0.0.5 + volumeMounts: + - name: config + mountPath: /etc/kubernetes + - name: certs + mountPath: /etc/ssl/certs + command: + - "./controller-manager" + args: + - --kubeconfig=/etc/kubernetes/admin.conf + resources: + requests: + cpu: 100m + memory: 20Mi + limits: + cpu: 100m + memory: 30Mi + - name: terraform-machine-controller + image: gcr.io/k8s-cluster-api/terraform-machine-controller:0.0.6 + volumeMounts: + - name: config + mountPath: /etc/kubernetes + - name: certs + mountPath: /etc/ssl/certs + - name: terraform-config + mountPath: /root/.terraform.d + - name: sshkeys + mountPath: /root/.ssh/vsphere_tmp + subPath: vsphere_tmp + - name: sshkeys + mountPath: /root/.ssh/vsphere_tmp.pub + subPath: vsphere_tmp.pub + - name: named-machines + mountPath: /etc/named-machines + command: + - "./terraform-machine-controller" + args: + - --kubeconfig=/etc/kubernetes/admin.conf + # Hardedcoded token can be removed as part of https://github.com/kubernetes-sigs/cluster-api/issues/159 + - --token=e6uqfr.5zkanzwfdclsbn7p + - --namedmachines=/etc/named-machines/vsphere_named_machines.yaml + resources: + requests: + cpu: 200m + memory: 200Mi + limits: + cpu: 400m + memory: 500Mi + volumes: + - name: config + hostPath: + path: /etc/kubernetes + - name: certs + hostPath: + path: /etc/ssl/certs + - name: terraform-config + emptyDir: {} + - name: sshkeys + secret: + secretName: sshkeys + - name: named-machines + configMap: + name: named-machines +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: sshkeys + namespace: default +data: + vsphere_tmp: $MACHINE_CONTROLLER_SSH_PRIVATE + vsphere_tmp.pub: $MACHINE_CONTROLLER_SSH_PUBLIC +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: named-machines +data: + vsphere_named_machines.yaml: |- + items: + - machineName: standard-master + machineHcl: | + variable "user" {} + variable "password" {} + variable "vsphere_server" {} + + variable "datacenter" {} + variable "datastore" {} + variable "resource_pool" {} + variable "num_cpus" {} + variable "memory" {} + variable "vm_template" {} + variable "network" { default = "VM Network"} + variable "disk_label" { default = "disk0" } + variable "disk_size" { default = 10} + + variable "vm_name" { + type = "string" + } + + provider "vsphere" { + version = "~> 1.4" + user = "${var.user}" + password = "${var.password}" + vsphere_server = "${var.vsphere_server}" + + # if you have a self-signed cert + allow_unverified_ssl = true + } + + data "vsphere_datacenter" "dc" { + name = "${var.datacenter}" + } + + data "vsphere_datastore" "datastore" { + name = "${var.datastore}" + datacenter_id = "${data.vsphere_datacenter.dc.id}" + } + + data "vsphere_resource_pool" "pool" { + name = "${var.resource_pool}" + datacenter_id = "${data.vsphere_datacenter.dc.id}" + } + + data "vsphere_network" "network" { + name = "${var.network}" + datacenter_id = "${data.vsphere_datacenter.dc.id}" + } + + data "vsphere_virtual_machine" "template" { + name = "${var.vm_template}" + datacenter_id = "${data.vsphere_datacenter.dc.id}" + } + + data "template_file" "cloud_provider_config" { + template = <