From 5d5cdec6cf1208bfb4f63e3b2cd48154a12d52a5 Mon Sep 17 00:00:00 2001 From: krousey Date: Mon, 7 May 2018 15:49:51 -0700 Subject: [PATCH] Use ubuntu cloud image with OVF (#140) * Use ubuntu cloud image with OVF Also build docker image with Terraform instead of transfering it over. Added ssh options to prevent host file checking. * fixup! Use ubuntu cloud image with OVF --- .../terraform-machine-controller/Dockerfile | 5 +- .../cmd/terraform-machine-controller/Makefile | 2 +- cloud/terraform/machineactuator.go | 34 +-- cloud/terraform/templates.go | 4 +- tf-deployer/vsphere_named_machines.yaml | 233 +++++++----------- 5 files changed, 112 insertions(+), 166 deletions(-) diff --git a/cloud/terraform/cmd/terraform-machine-controller/Dockerfile b/cloud/terraform/cmd/terraform-machine-controller/Dockerfile index 07fe21afc716..1c132fcd2685 100644 --- a/cloud/terraform/cmd/terraform-machine-controller/Dockerfile +++ b/cloud/terraform/cmd/terraform-machine-controller/Dockerfile @@ -24,7 +24,6 @@ RUN CGO_ENABLED=0 GOOS=linux go install -a -ldflags '-extldflags "-static"' sigs # Final container FROM alpine:3.7 -RUN apk --no-cache add --update ca-certificates bash openssh -RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config -RUN echo "UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config +RUN apk --no-cache add --update ca-certificates bash openssh terraform +RUN echo 'plugin_cache_dir = "$HOME/.terraform.d/plugin-cache"' >> ~/.terraformrc COPY --from=builder /go/bin/terraform-machine-controller . diff --git a/cloud/terraform/cmd/terraform-machine-controller/Makefile b/cloud/terraform/cmd/terraform-machine-controller/Makefile index 0859325d806d..800a7f53c0ef 100644 --- a/cloud/terraform/cmd/terraform-machine-controller/Makefile +++ b/cloud/terraform/cmd/terraform-machine-controller/Makefile @@ -17,7 +17,7 @@ PREFIX = gcr.io/k8s-cluster-api DEV_PREFIX ?= gcr.io/$(shell gcloud config get-value project) NAME = terraform-machine-controller -TAG = 0.0.1 +TAG = 0.0.2 image: docker build -t "$(PREFIX)/$(NAME):$(TAG)" -f ./Dockerfile ../../../.. diff --git a/cloud/terraform/machineactuator.go b/cloud/terraform/machineactuator.go index 3baebddda36b..8b16d7dd9ebd 100644 --- a/cloud/terraform/machineactuator.go +++ b/cloud/terraform/machineactuator.go @@ -259,19 +259,17 @@ func (tf *TerraformClient) Create(cluster *clusterv1.Cluster, machine *clusterv1 args = append(args, fmt.Sprintf("vm_name=%s", machine.ObjectMeta.Name)) args = append(args, fmt.Sprintf("-var-file=%s", tfVarsPath)) - out, cmdErr := runTerraformCmd(false, tfConfigDir, args...) + _, cmdErr := runTerraformCmd(false, tfConfigDir, args...) if cmdErr != nil { - return errors.New(fmt.Sprintf("Could not run terraform: ", cmdErr)) + return errors.New(fmt.Sprintf("Could not run terraform: %s", cmdErr)) } // Get the IP address - kubeadmJoinIpRe := regexp.MustCompile("kubeadm join .* ([0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}:[0-9]{1,5})") - parts := kubeadmJoinIpRe.FindStringSubmatch(out.String()) // [full match, ip addr] - if len(parts) < 2 { - return errors.New(fmt.Sprintf("Could not get master IP address. You will need to manually modify the cluster object's status with the endpoint if the master was created successfully.")) + out, cmdErr := runTerraformCmd(false, tfConfigDir, "output", "ip_address") + if cmdErr != nil { + return fmt.Errorf("could not obtain 'ip_address' output variable: %s", cmdErr) } - ipPortParts := strings.Split(parts[1], ":") - masterEndpointIp := ipPortParts[0] + masterEndpointIp := strings.TrimSpace(out.String()) glog.Infof("Master created with ip address %s", masterEndpointIp) // If we have a machineClient, then annotate the machine so that we @@ -396,8 +394,10 @@ func (tf *TerraformClient) GetKubeConfig(master *clusterv1.Machine) (string, err cmd := exec.Command( // TODO: this is taking my private key and username for now. "ssh", "-i", "~/.ssh/vsphere_tmp", + "-o", "StrictHostKeyChecking no", + "-o", "UserKnownHostsFile /dev/null", fmt.Sprintf("ubuntu@%s", ip), - "echo STARTFILE; cat /etc/kubernetes/admin.conf") + "echo STARTFILE; sudo cat /etc/kubernetes/admin.conf") cmd.Stdout = &out cmd.Stderr = os.Stderr cmd.Run() @@ -432,6 +432,8 @@ func (tf *TerraformClient) SetupRemoteMaster(master *clusterv1.Machine) error { } cmd := exec.Command( "scp", "-i", "~/.ssh/vsphere_tmp", + "-o", "StrictHostKeyChecking no", + "-o", "UserKnownHostsFile /dev/null", "-r", path.Join(homedir, ".terraform.d"), fmt.Sprintf("ubuntu@%s:~/", ip)) @@ -439,22 +441,12 @@ func (tf *TerraformClient) SetupRemoteMaster(master *clusterv1.Machine) error { cmd.Stderr = os.Stderr cmd.Run() - // TODO: Bake this into the controller image instead of this hacky thing. - glog.Infof("Copying the terraform binary to master.") - cmd = exec.Command( - // TODO: this is taking my private key and username for now. - "scp", "-i", "~/.ssh/vsphere_tmp", - // TODO: this should be a flag? - "-r", "/Users/karangoel/.gvm/pkgsets/go1.9.2/global/src/sigs.k8s.io/cluster-api/cloud/terraform/bin/", - fmt.Sprintf("ubuntu@%s:~/.terraform.d/", ip)) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Run() - glog.Infof("Setting up terraform on remote master.") cmd = exec.Command( // TODO: this is taking my private key and username for now. "ssh", "-i", "~/.ssh/vsphere_tmp", + "-o", "StrictHostKeyChecking no", + "-o", "UserKnownHostsFile /dev/null", fmt.Sprintf("ubuntu@%s", ip), fmt.Sprintf("source ~/.profile; cd ~/.terraform.d/kluster/machines/%s; ~/.terraform.d/terraform init; cp -r ~/.terraform.d/kluster/machines/%s/.terraform/plugins/* ~/.terraform.d/plugins/", machineName, machineName)) cmd.Stdout = os.Stdout diff --git a/cloud/terraform/templates.go b/cloud/terraform/templates.go index 6b7cc9b97d28..873ee264f0a3 100644 --- a/cloud/terraform/templates.go +++ b/cloud/terraform/templates.go @@ -167,7 +167,7 @@ deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main EOF apt-get update -apt-get install -y docker-engine=1.11.2-0~xenial +apt-get install -y docker.io curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - @@ -313,7 +313,7 @@ kubeadm init --apiserver-bind-port ${PORT} --token ${TOKEN} --kubernetes-version # install weavenet sysctl net.bridge.bridge-nf-call-iptables=1 export kubever=$(kubectl version --kubeconfig /etc/kubernetes/admin.conf | base64 | tr -d '\n') -kubectl apply --kubeconfig /etc/kubernetes/admin.conf -f "https://cloud.weave.works/k8s/net?k8s-version=$kubever" +kubectl apply --kubeconfig /etc/kubernetes/admin.conf -f "https://cloud.weave.works/k8s/net?env.CHECKPOINT_DISABLE=1&env.IPALLOC_RANGE=${POD_CIDR}&disable-npc=true&k8s-version=$kubever" for tries in $(seq 1 60); do kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break diff --git a/tf-deployer/vsphere_named_machines.yaml b/tf-deployer/vsphere_named_machines.yaml index 7429b4572fc0..238d5bfa793b 100644 --- a/tf-deployer/vsphere_named_machines.yaml +++ b/tf-deployer/vsphere_named_machines.yaml @@ -8,38 +8,19 @@ items: variable "datacenter" {} variable "datastore" {} variable "resource_pool" {} - variable "network" {} variable "num_cpus" {} variable "memory" {} variable "vm_template" {} - variable "disk_label" {} - variable "disk_size" {} - - // The domain name to set up each virtual machine as. - variable "virtual_machine_domain" {} - - // The network address for the virtual machines, in the form of 10.0.0.0/24. - variable "virtual_machine_network_address" {} - - // The last octect that serves as the start of the IP addresses for the virtual - // machines. Given the default value here of 100, if the network address is - // 10.0.0.0/24, the 3 virtual machines will be assigned addresses 10.0.0.100, - // 10.0.0.101, and 10.0.0.102. - variable "virtual_machine_ip_address_start" {} - - // The default gateway for the network the virtual machines reside in. - variable "virtual_machine_gateway" {} - - // The DNS servers for the network the virtual machines reside in. - variable "virtual_machine_dns_servers" { - type = "list" - } + variable "network" { default = "VM Network"} + variable "disk_label" { default = "disk0" } + variable "disk_size" { default = 10} variable "vm_name" { type = "string" } provider "vsphere" { + version = "~> 1.4" user = "${var.user}" password = "${var.password}" vsphere_server = "${var.vsphere_server}" @@ -72,6 +53,22 @@ items: datacenter_id = "${data.vsphere_datacenter.dc.id}" } + data "template_file" "user_data" { + template = <