Set up node and storage management for vsphere (kubernetes-sigs#212)

* Set up node and storage management for vsphere * Bump the terraform controller version
spew · May 25, 2018 · 2461d32 · 2461d32
1 parent d560c7b
commit 2461d32
Show file tree

Hide file tree

Showing 4 changed files with 87 additions and 11 deletions.
diff --git a/cloud/terraform/cmd/terraform-machine-controller/Makefile b/cloud/terraform/cmd/terraform-machine-controller/Makefile
@@ -18,7 +18,7 @@ GCR_BUCKET = k8s-cluster-api
 PREFIX = gcr.io/$(GCR_BUCKET)
 DEV_PREFIX ?= gcr.io/$(shell gcloud config get-value project)
 NAME = terraform-machine-controller
-TAG = 0.0.3
+TAG = 0.0.4
 
 image:
 	docker build -t "$(PREFIX)/$(NAME):$(TAG)" -f ./Dockerfile ../../../..

diff --git a/cloud/terraform/pods.go b/cloud/terraform/pods.go
@@ -34,7 +34,7 @@ import (
 
 var apiServerImage = "gcr.io/k8s-cluster-api/cluster-apiserver:0.0.3"
 var controllerManagerImage = "gcr.io/k8s-cluster-api/controller-manager:0.0.3"
-var machineControllerImage = "gcr.io/k8s-cluster-api/terraform-machine-controller:0.0.3"
+var machineControllerImage = "gcr.io/k8s-cluster-api/terraform-machine-controller:0.0.4"
 
 func init() {
 	if img, ok := os.LookupEnv("MACHINE_CONTROLLER_IMAGE"); ok {

diff --git a/cloud/terraform/templates.go b/cloud/terraform/templates.go
@@ -213,7 +213,11 @@ sysctl net.bridge.bridge-nf-call-iptables=1
 # kubeadm uses 10th IP as DNS server
 CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1)
 
-sed -i "s/KUBELET_DNS_ARGS=[^\"]*/KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+cat > /etc/systemd/system/kubelet.service.d/20-cloud.conf << EOF
+[Service]
+Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
+Environment="KUBELET_EXTRA_ARGS=--cloud-provider=vsphere"
+EOF
 systemctl daemon-reload
 systemctl restart kubelet.service
 
@@ -295,7 +299,11 @@ chmod a+rx /usr/bin/kubeadm
 
 systemctl enable docker
 systemctl start docker
-sed -i "s/KUBELET_DNS_ARGS=[^\"]*/KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+cat > /etc/systemd/system/kubelet.service.d/20-cloud.conf << EOF
+[Service]
+Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
+Environment="KUBELET_EXTRA_ARGS=--cloud-provider=vsphere --cloud-config=/etc/kubernetes/cloud-config/cloud-config.yaml"
+EOF
 systemctl daemon-reload
 systemctl restart kubelet.service
 ` +
@@ -304,10 +312,37 @@ echo $PRIVATEIP > /tmp/.ip
 ` +
 	"PUBLICIP=`ip route get 8.8.8.8 | awk '{printf \"%s\", $NF; exit}'`" + `
 
+# Set up kubeadm config file to pass parameters to kubeadm init.
+cat > /etc/kubernetes/kubeadm_config.yaml <<EOF
+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: ${PUBLICIP}
+  bindPort: ${PORT}
+networking:
+  serviceSubnet: ${SERVICE_CIDR}
+kubernetesVersion: v${CONTROL_PLANE_VERSION}
+token: ${TOKEN}
+apiServerCertSANs:
+- ${PUBLICIP}
+- ${PRIVATEIP}
+apiServerExtraArgs:
+  cloud-provider: vsphere
+  cloud-config: /etc/kubernetes/cloud-config/cloud-config.yaml
+apiServerExtraVolumes:
+  - name: cloud-config
+    hostPath: /etc/kubernetes/cloud-config
+    mountPath: /etc/kubernetes/cloud-config
+controllerManagerExtraArgs:
+  cloud-provider: vsphere
+  cloud-config: /etc/kubernetes/cloud-config/cloud-config.yaml
+controllerManagerExtraVolumes:
+  - name: cloud-config
+    hostPath: /etc/kubernetes/cloud-config
+    mountPath: /etc/kubernetes/cloud-config
+EOF
 
-kubeadm init --apiserver-bind-port ${PORT} --token ${TOKEN} --kubernetes-version v${CONTROL_PLANE_VERSION} \
-             --apiserver-advertise-address ${PUBLICIP} --apiserver-cert-extra-sans ${PUBLICIP} ${PRIVATEIP} \
-             --service-cidr ${SERVICE_CIDR}
+kubeadm init --config /etc/kubernetes/kubeadm_config.yaml
 
 # install weavenet
 sysctl net.bridge.bridge-nf-call-iptables=1

diff --git a/tf-deployer/vsphere_named_machines.yaml b/tf-deployer/vsphere_named_machines.yaml
@@ -53,19 +53,60 @@ items:
       datacenter_id = "${data.vsphere_datacenter.dc.id}"
     }
 
+    data "template_file" "cloud_provider_config" {
+      template = <<EOF
+    [Global]
+    datacenters = "$${datacenter}"
+    insecure-flag = "1" #set to 1 if the vCenter uses a self-signed cert
+
+    [VirtualCenter "$${vsphere_server}"]
+            user = "$${user}"
+            password = "$${password}"
+
+    [Workspace]
+            server = "$${vsphere_server}"
+            datacenter = "$${datacenter}"
+            folder = "$${resource_pool}"
+            default-datastore = "$${datastore}"
+            resourcepool-path = "$${resource_pool}"
+
+    [Disk]
+            scsicontrollertype = pvscsi
+
+    [Network]
+            public-network = "$${network}"
+    EOF
+      vars {
+        vsphere_server = "${var.vsphere_server}"
+        datacenter     = "${var.datacenter}"
+        user           = "${var.user}"
+        password       = "${var.password}"
+        datastore      = "${var.datastore}"
+        resource_pool  = "${var.resource_pool}"
+        network        = "${var.network}"
+      }
+    }
+
     data "template_file" "user_data" {
       template = <<EOF
     #cloud-config
     write_files:
-      - content: |
-          $${startup_script}
-        path: /tmp/master.sh
+      - path: /tmp/master.sh
         permissions: '0755'
+        encoding: base64
+        content: |
+          $${startup_script}
+      - path: /etc/kubernetes/cloud-config/cloud-config.yaml
+        permissions: '0600'
+        encoding: base64
+        content: |
+          $${cloud_provider_config}
     runcmd:
       - /tmp/master.sh
     EOF
       vars {
-        startup_script = "${indent(6, file("/tmp/machine-startup.sh"))}"
+        startup_script = "${base64encode(file("/tmp/machine-startup.sh"))}"
+        cloud_provider_config = "${base64encode(data.template_file.cloud_provider_config.rendered)}"
       }
     }