From cf10c9cffb87a0a71159d541de299cd8ad0b171a Mon Sep 17 00:00:00 2001 From: Abrar Shivani Date: Wed, 5 Apr 2017 22:30:04 -0700 Subject: [PATCH] Add vm-uuid in cloud-config file required by Kubernetes vSphere CloudProvider --- nodeup/pkg/model/cloudconfig.go | 95 +++++++++++++------ upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go | 31 ++++++ upup/pkg/fi/cloudup/vspheretasks/attachiso.go | 17 +++- .../pkg/fi/cloudup/vspheretasks/cloud_init.go | 5 + 4 files changed, 117 insertions(+), 31 deletions(-) diff --git a/nodeup/pkg/model/cloudconfig.go b/nodeup/pkg/model/cloudconfig.go index 2e940dc3f67f1..6727c9021e5c3 100644 --- a/nodeup/pkg/model/cloudconfig.go +++ b/nodeup/pkg/model/cloudconfig.go @@ -17,15 +17,22 @@ limitations under the License. package model import ( + "bufio" "fmt" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "os" "strings" ) const CloudConfigFilePath = "/etc/kubernetes/cloud.config" +// Required for vSphere CloudProvider +const MinimumVersionForVMUUID = "1.5.3" +const VM_UUID_FILE_PATH = "/root/vm_uuid" + // CloudConfigBuilder creates the cloud configuration file type CloudConfigBuilder struct { *NodeupModelContext @@ -59,37 +66,41 @@ func (b *CloudConfigBuilder) Build(c *fi.ModelBuilderContext) error { if cloudConfig.DisableSecurityGroupIngress != nil { lines = append(lines, fmt.Sprintf("DisableSecurityGroupIngress = %t", *cloudConfig.DisableSecurityGroupIngress)) } - } - - if cloudConfig.VSphereUsername != nil { - lines = append(lines, "user = "+*cloudConfig.VSphereUsername) - } - - if cloudConfig.VSpherePassword != nil { - lines = append(lines, "password = "+*cloudConfig.VSpherePassword) - } - - if cloudConfig.VSphereServer != nil { - lines = append(lines, "server = "+*cloudConfig.VSphereServer) - lines = append(lines, "port = 443") - lines = append(lines, fmt.Sprintf("insecure-flag = %t", true)) - } - - if cloudConfig.VSphereDatacenter != nil { - lines = append(lines, "datacenter = "+*cloudConfig.VSphereDatacenter) - } - - if cloudConfig.VSphereDatastore != nil { - lines = append(lines, "datastore = "+*cloudConfig.VSphereDatastore) + case "vsphere": + vm_uuid, err := getVMUUID(b.Cluster.Spec.KubernetesVersion) + if err != nil { + return err + } + // Note: Segregate configuration for different sections as below + // Global Config for vSphere CloudProvider + if cloudConfig.VSphereUsername != nil { + lines = append(lines, "user = "+*cloudConfig.VSphereUsername) + } + if cloudConfig.VSpherePassword != nil { + lines = append(lines, "password = "+*cloudConfig.VSpherePassword) + } + if cloudConfig.VSphereServer != nil { + lines = append(lines, "server = "+*cloudConfig.VSphereServer) + lines = append(lines, "port = 443") + lines = append(lines, fmt.Sprintf("insecure-flag = %t", true)) + } + if cloudConfig.VSphereDatacenter != nil { + lines = append(lines, "datacenter = "+*cloudConfig.VSphereDatacenter) + } + if cloudConfig.VSphereDatastore != nil { + lines = append(lines, "datastore = "+*cloudConfig.VSphereDatastore) + } + if vm_uuid != "" { + lines = append(lines, "vm-uuid = "+strings.Trim(vm_uuid, "\n")) + } + // Disk Config for vSphere CloudProvider + // We need this to support Kubernetes vSphere CloudProvider < v1.5.3 + lines = append(lines, "[disk]") + lines = append(lines, "scsicontrollertype = pvscsi") } config := "[global]\n" + strings.Join(lines, "\n") + "\n" - // We need this to support Kubernetes vSphere CloudProvider < v1.5.3 - if fi.CloudProviderID(b.Cluster.Spec.CloudProvider) == fi.CloudProviderVSphere { - config += "[disk]\n" + "scsicontrollertype = pvscsi" + "\n" - } - t := &nodetasks.File{ Path: CloudConfigFilePath, Contents: fi.NewStringResource(config), @@ -99,3 +110,33 @@ func (b *CloudConfigBuilder) Build(c *fi.ModelBuilderContext) error { return nil } + +// We need this for vSphere CloudProvider +// getVMUUID gets instance uuid of the VM from the file written by cloud-init +func getVMUUID(kubernetesVersion string) (string, error) { + + actualKubernetesVersion, err := util.ParseKubernetesVersion(kubernetesVersion) + if err != nil { + return "", err + } + minimumVersionForUUID, err := util.ParseKubernetesVersion(MinimumVersionForVMUUID) + if err != nil { + return "", err + } + + // VM UUID is required only for Kubernetes version greater than 1.5.3 + if actualKubernetesVersion.GTE(*minimumVersionForUUID) { + file, err := os.Open(VM_UUID_FILE_PATH) + defer file.Close() + if err != nil { + return "", err + } + vm_uuid, err := bufio.NewReader(file).ReadString('\n') + if err != nil { + return "", err + } + return vm_uuid, err + } + + return "", err +} diff --git a/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go b/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go index 6bffab333cad0..2de9e3ff64aa2 100644 --- a/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go +++ b/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go @@ -298,6 +298,37 @@ func (c *VSphereCloud) UploadAndAttachISO(vm *string, isoFile string) error { } +// Returns VM's instance uuid +func (c *VSphereCloud) FindVMUUID(vm *string) (string, error) { + f := find.NewFinder(c.Client.Client, true) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dc, err := f.Datacenter(ctx, c.Datacenter) + if err != nil { + return "", err + } + f.SetDatacenter(dc) + + vmRef, err := f.VirtualMachine(ctx, *vm) + if err != nil { + return "", err + } + + var refs []types.ManagedObjectReference + refs = append(refs, vmRef.Reference()) + var vmResult mo.VirtualMachine + + pc := property.DefaultCollector(c.Client.Client) + err = pc.RetrieveOne(ctx, vmRef.Reference(), []string{"config.uuid"}, &vmResult) + if err != nil { + return "", err + } + glog.V(4).Infof("vm property collector result :%+v\n", vmResult) + glog.V(3).Infof("retrieved vm uuid as %q for vm %q", vmResult.Config.Uuid, *vm) + return vmResult.Config.Uuid, nil +} + func getCloudInitFileName(vmName string) string { return vmName + "/" + cloudInitFile } diff --git a/upup/pkg/fi/cloudup/vspheretasks/attachiso.go b/upup/pkg/fi/cloudup/vspheretasks/attachiso.go index c9a45efec9277..823133c40125c 100644 --- a/upup/pkg/fi/cloudup/vspheretasks/attachiso.go +++ b/upup/pkg/fi/cloudup/vspheretasks/attachiso.go @@ -91,7 +91,13 @@ func (_ *AttachISO) RenderVSphere(t *vsphere.VSphereAPITarget, a, e, changes *At dir, err := ioutil.TempDir("", *changes.VM.Name) defer os.RemoveAll(dir) - isoFile, err := createISO(changes, startupStr, dir, t.Cloud.CoreDNSServer) + // Need this in cloud config file for vSphere CloudProvider + vmUUID, err := t.Cloud.FindVMUUID(changes.VM.Name) + if err != nil { + return err + } + + isoFile, err := createISO(changes, startupStr, dir, t.Cloud.CoreDNSServer, vmUUID) if err != nil { glog.Errorf("Failed to createISO for vspheretasks, err: %v", err) return err @@ -105,7 +111,7 @@ func (_ *AttachISO) RenderVSphere(t *vsphere.VSphereAPITarget, a, e, changes *At return nil } -func createUserData(startupStr string, dir string, dnsServer string) error { +func createUserData(startupStr string, dir string, dnsServer string, vmUUID string) error { // Update the startup script to add the extra spaces for // indentation when copied to the user-data file. strArray := strings.Split(startupStr, "\n") @@ -131,6 +137,9 @@ func createUserData(startupStr string, dir string, dnsServer string) error { dnsUpdateStr := strings.Join(lines, "\n") data = strings.Replace(data, "$DNS_SCRIPT", dnsUpdateStr, -1) + vmUUIDStr := " " + vmUUID + "\n" + data = strings.Replace(data, "$VM_UUID", vmUUIDStr, -1) + userDataFile := filepath.Join(dir, "user-data") glog.V(4).Infof("User data file content: %s", data) @@ -156,8 +165,8 @@ func createMetaData(dir string, vmName string) error { return nil } -func createISO(changes *AttachISO, startupStr string, dir string, dnsServer string) (string, error) { - err := createUserData(startupStr, dir, dnsServer) +func createISO(changes *AttachISO, startupStr string, dir string, dnsServer string, vmUUID string) (string, error) { + err := createUserData(startupStr, dir, dnsServer, vmUUID) if err != nil { return "", err } diff --git a/upup/pkg/fi/cloudup/vspheretasks/cloud_init.go b/upup/pkg/fi/cloudup/vspheretasks/cloud_init.go index 69980e0ef8656..3efc1f006a2c5 100644 --- a/upup/pkg/fi/cloudup/vspheretasks/cloud_init.go +++ b/upup/pkg/fi/cloudup/vspheretasks/cloud_init.go @@ -29,6 +29,11 @@ $DNS_SCRIPT owner: root:root path: /root/update_dns.sh permissions: "0644" + - content: | +$VM_UUID + owner: root:root + path: /root/vm_uuid + permissions: "0644" runcmd: - bash /root/update_dns.sh 2>&1 > /var/log/update_dns.log