Removed hardcode from protokube logic. Fixes kubernetes#15. (kubernet…

…es#46)

docs/development/vsphere-dev.md

@@ -93,7 +93,7 @@ Please note that dns-controller has also been modified to support vSphere. You c
 Execute following command to launch cluster.
 
 ```bash
-.build/dist/darwin/amd64/kops create cluster kubernetes.skydns.local  --cloud=vsphere --zones=${AWS_REGION}a --dns-zone=skydns.local --networking=flannel
+.build/dist/darwin/amd64/kops create cluster kubernetes.skydns.local  --cloud=vsphere --zones=vmware-zone --dns-zone=skydns.local --networking=flannel
  --vsphere-server=10.160.97.44 --vsphere-datacenter=VSAN-DC --vsphere-resource-pool=VSAN-Cluster --vsphere-datastore=vsanDatastore --dns private --vsphere-coredns-server=http://10.192.217.24:2379 --image="ubuntu_16_04" 
 ```
 
@@ -102,7 +102,7 @@ Use .build/dist/linux/amd64/kops if working on a linux machine, instead of mac.
 **Notes**
 
 1. ```clustername``` should end with **skydns.local**. Example: ```kubernetes.cluster.skydns.local```.
-2. ```zones``` should end with ```a```. Example: ```us-west-2a``` or as the above command sets ```--zones=${AWS_REGION}a```.
+2. For ```zones``` any string will do, for now. It's only getting used for the construction of names of various entities. But it's a mandatory argument.
 3. Make sure following parameters have these values,
     * ```--dns-zone=skydns.local```
     * ```--networking=flannel```

pkg/model/vspheremodel/autoscalinggroup.go

@@ -57,6 +57,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 				VM:              createVmTask,
 				IG:              ig,
 				BootstrapScript: b.BootstrapScript,
+				EtcdClusters:    b.Cluster.Spec.EtcdClusters,
 			}
 			attachISOTask.BootstrapScript.AddAwsEnvironmentVariables = true
 

protokube/pkg/protokube/vsphere_volume.go

@@ -18,37 +18,26 @@ package protokube
 
 import (
 	"errors"
+	"fmt"
 	"github.com/golang/glog"
+	"io/ioutil"
+	"k8s.io/kops/upup/pkg/fi/cloudup/vsphere"
 	"net"
+	"os/exec"
+	"runtime"
+	"strings"
 )
 
-const EtcdDataKey = "01"
-const EtcdDataVolPath = "/mnt/master-" + EtcdDataKey
-const EtcdEventKey = "02"
-const EtcdEventVolPath = "/mnt/master-" + EtcdEventKey
-
-// TODO Use lsblk or counterpart command to find the actual device details.
-const LocalDeviceForDataVol = "/dev/sdb1"
-const LocalDeviceForEventsVol = "/dev/sdc1"
+const VolumeMetaDataFile = "/vol-metadata/metadata.json"
 const VolStatusValue = "attached"
-const EtcdNodeName = "a"
-const EtcdClusterName = "main"
-const EtcdEventsClusterName = "events"
 
-type VSphereVolumes struct {
-	// Dummy property. Not getting used any where for now.
-	paths map[string]string
-}
+type VSphereVolumes struct{}
 
 var _ Volumes = &VSphereVolumes{}
 var machineIp net.IP
 
 func NewVSphereVolumes() (*VSphereVolumes, error) {
-	vsphereVolumes := &VSphereVolumes{
-		paths: make(map[string]string),
-	}
-	vsphereVolumes.paths[EtcdDataKey] = EtcdDataVolPath
-	vsphereVolumes.paths[EtcdEventKey] = EtcdEventVolPath
+	vsphereVolumes := &VSphereVolumes{}
 	return vsphereVolumes, nil
 }
 
@@ -60,49 +49,80 @@ func (v *VSphereVolumes) FindVolumes() ([]*Volume, error) {
 		attachedTo = ip.String()
 	}
 
-	// etcd data volume and etcd cluster spec.
-	{
+	etcdClusters, err := getVolMetadata()
+
+	if err != nil {
+		return nil, err
+	}
+
+	for _, etcd := range etcdClusters {
+		mountPoint := vsphere.GetMountPoint(etcd.VolumeId)
+		localDevice, err := getDevice(mountPoint)
+		if err != nil {
+			return nil, err
+		}
 		vol := &Volume{
-			ID:          EtcdDataKey,
-			LocalDevice: LocalDeviceForDataVol,
+			ID:          etcd.VolumeId,
+			LocalDevice: localDevice,
 			AttachedTo:  attachedTo,
-			Mountpoint:  EtcdDataVolPath,
+			Mountpoint:  mountPoint,
 			Status:      VolStatusValue,
 			Info: VolumeInfo{
-				Description: EtcdClusterName,
+				Description: etcd.EtcdClusterName,
 			},
 		}
+
 		etcdSpec := &EtcdClusterSpec{
-			ClusterKey: EtcdClusterName,
-			NodeName:   EtcdNodeName,
-			NodeNames:  []string{EtcdNodeName},
+			ClusterKey: etcd.EtcdClusterName,
+			NodeName:   etcd.EtcdNodeName,
+		}
+
+		var nodeNames []string
+		for _, member := range etcd.Members {
+			nodeNames = append(nodeNames, member.Name)
 		}
+		etcdSpec.NodeNames = nodeNames
 		vol.Info.EtcdClusters = []*EtcdClusterSpec{etcdSpec}
 		volumes = append(volumes, vol)
 	}
+	glog.V(4).Infof("Found volumes: %v", volumes)
+	return volumes, nil
+}
 
-	// etcd events volume and etcd events cluster spec.
-	{
-		vol := &Volume{
-			ID:          EtcdEventKey,
-			LocalDevice: LocalDeviceForEventsVol,
-			AttachedTo:  attachedTo,
-			Mountpoint:  EtcdEventVolPath,
-			Status:      VolStatusValue,
-			Info: VolumeInfo{
-				Description: EtcdEventsClusterName,
-			},
+func getDevice(mountPoint string) (string, error) {
+	if runtime.GOOS == "linux" {
+		cmd := "lsblk"
+		arg := "-l"
+		out, err := exec.Command(cmd, arg).Output()
+		if err != nil {
+			return "", err
 		}
-		etcdSpec := &EtcdClusterSpec{
-			ClusterKey: EtcdEventsClusterName,
-			NodeName:   EtcdNodeName,
-			NodeNames:  []string{EtcdNodeName},
+
+		if Containerized {
+			mountPoint = PathFor(mountPoint)
 		}
-		vol.Info.EtcdClusters = []*EtcdClusterSpec{etcdSpec}
-		volumes = append(volumes, vol)
+		lines := strings.Split(string(out), "\n")
+		for _, line := range lines {
+			if strings.Contains(line, mountPoint) {
+				lsblkOutput := strings.Split(line, " ")
+				glog.V(4).Infof("Found device: %v ", lsblkOutput[0])
+				return "/dev/" + lsblkOutput[0], nil
+			}
+		}
+	} else {
+		return "", fmt.Errorf("Failed to find device. OS %v is not supported for vSphere.", runtime.GOOS)
 	}
-	glog.Infof("Found volumes: %v", volumes)
-	return volumes, nil
+	return "", fmt.Errorf("No device has been mounted on mountPoint %v.", mountPoint)
+}
+
+func getVolMetadata() ([]vsphere.VolumeMetadata, error) {
+	rawData, err := ioutil.ReadFile(PathFor(VolumeMetaDataFile))
+
+	if err != nil {
+		return nil, err
+	}
+
+	return vsphere.UnmarshalVolumeMetadata(string(rawData))
 }
 
 func (v *VSphereVolumes) AttachVolume(volume *Volume) error {

upup/pkg/fi/cloudup/vsphere/vsphere_volume_metadata.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"encoding/json"
+	"strconv"
+)
+
+type VolumeMetadata struct {
+	// EtcdClusterName is the name of the etcd cluster (main, events etc)
+	EtcdClusterName string `json:"etcdClusterName,omitempty"`
+	// EtcdNodeName is the name of a node in etcd cluster for which this volume will be used
+	EtcdNodeName string `json:"etcdNodeName,omitempty"`
+	// EtcdMember stores the configurations for each member of the cluster
+	Members []EtcdMemberSpec `json:"etcdMembers,omitempty"`
+	// Volume id
+	VolumeId string `json:"volumeId,omitempty"`
+}
+
+type EtcdMemberSpec struct {
+	// Name is the name of the member within the etcd cluster
+	Name          string `json:"name,omitempty"`
+	InstanceGroup string `json:"instanceGroup,omitempty"`
+}
+
+func MarshalVolumeMetadata(v []VolumeMetadata) (string, error) {
+	metadata, err := json.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+
+	return string(metadata), nil
+}
+
+func UnmarshalVolumeMetadata(text string) ([]VolumeMetadata, error) {
+	var v []VolumeMetadata
+	err := json.Unmarshal([]byte(text), &v)
+	return v, err
+}
+
+func GetVolumeId(i int) string {
+	return "0" + strconv.Itoa(i)
+}
+
+/*
+ * GetMountPoint will return the mount point where the volume is expected to be mounted.
+ * This path would be /mnt/master-<volumeId>, eg: /mnt/master-01.
+ */
+func GetMountPoint(volumeId string) string {
+	return "/mnt/master-" + volumeId
+}

upup/pkg/fi/cloudup/vspheretasks/attachiso.go

@@ -42,6 +42,7 @@ type AttachISO struct {
 	VM              *VirtualMachine
 	IG              *kops.InstanceGroup
 	BootstrapScript *model.BootstrapScript
+	EtcdClusters    []*kops.EtcdClusterSpec
 }
 
 var _ fi.HasName = &AttachISO{}
@@ -111,7 +112,7 @@ func (_ *AttachISO) RenderVSphere(t *vsphere.VSphereAPITarget, a, e, changes *At
 	return nil
 }
 
-func createUserData(startupStr string, dir string, dnsServer string, vmUUID string) error {
+func createUserData(changes *AttachISO, startupStr string, dir string, dnsServer string, vmUUID string) error {
 	// Update the startup script to add the extra spaces for
 	// indentation when copied to the user-data file.
 	strArray := strings.Split(startupStr, "\n")
@@ -140,17 +141,76 @@ func createUserData(startupStr string, dir string, dnsServer string, vmUUID stri
 	vmUUIDStr := "       " + vmUUID + "\n"
 	data = strings.Replace(data, "$VM_UUID", vmUUIDStr, -1)
 
+	data, err = createVolumeScript(changes, data)
+	if err != nil {
+		return err
+	}
+
 	userDataFile := filepath.Join(dir, "user-data")
 	glog.V(4).Infof("User data file content: %s", data)
 
-	if err := ioutil.WriteFile(userDataFile, []byte(data), 0644); err != nil {
+	if err = ioutil.WriteFile(userDataFile, []byte(data), 0644); err != nil {
 		glog.Errorf("Unable to write user-data into file %s", userDataFile)
 		return err
 	}
 
 	return nil
 }
 
+func createVolumeScript(changes *AttachISO, data string) (string, error) {
+	if changes.IG.Spec.Role != kops.InstanceGroupRoleMaster {
+		return strings.Replace(data, "$VOLUME_SCRIPT", "       No volume metadata needed for "+string(changes.IG.Spec.Role)+".", -1), nil
+	}
+
+	volsString, err := getVolMetadata(changes)
+
+	if err != nil {
+		return "", err
+	}
+
+	return strings.Replace(data, "$VOLUME_SCRIPT", "       "+volsString, -1), nil
+}
+
+func getVolMetadata(changes *AttachISO) (string, error) {
+	var volsMetadata []vsphere.VolumeMetadata
+
+	// Creating vsphere.VolumeMetadata using clusters EtcdClusterSpec
+	for i, etcd := range changes.EtcdClusters {
+		volMetadata := vsphere.VolumeMetadata{}
+		volMetadata.EtcdClusterName = etcd.Name
+		volMetadata.VolumeId = vsphere.GetVolumeId(i + 1)
+
+		var members []vsphere.EtcdMemberSpec
+		var thisNode string
+		for _, member := range etcd.Members {
+			if *member.InstanceGroup == changes.IG.Name {
+				thisNode = member.Name
+			}
+			etcdMember := vsphere.EtcdMemberSpec{
+				Name:          member.Name,
+				InstanceGroup: *member.InstanceGroup,
+			}
+			members = append(members, etcdMember)
+		}
+
+		if thisNode == "" {
+			return "", fmt.Errorf("Failed to construct volume metadata for %v InstanceGroup.", changes.IG.Name)
+		}
+
+		volMetadata.EtcdNodeName = thisNode
+		volMetadata.Members = members
+		volsMetadata = append(volsMetadata, volMetadata)
+	}
+
+	glog.V(4).Infof("Marshaling master vol metadata : %v", volsMetadata)
+	volsString, err := vsphere.MarshalVolumeMetadata(volsMetadata)
+	glog.V(4).Infof("Marshaled master vol metadata: %v", volsString)
+	if err != nil {
+		return "", err
+	}
+	return volsString, nil
+}
+
 func createMetaData(dir string, vmName string) error {
 	data := strings.Replace(metaDataTemplate, "$INSTANCE_ID", uuid.NewUUID().String(), -1)
 	data = strings.Replace(data, "$LOCAL_HOST_NAME", vmName, -1)
@@ -165,8 +225,9 @@ func createMetaData(dir string, vmName string) error {
 	return nil
 }
 
-func createISO(changes *AttachISO, startupStr string, dir string, dnsServer string, vmUUID string) (string, error) {
-	err := createUserData(startupStr, dir, dnsServer, vmUUID)
+func createISO(changes *AttachISO, startupStr string, dir string, dnsServer, vmUUID string) (string, error) {
+	err := createUserData(changes, startupStr, dir, dnsServer, vmUUID)
+
 	if err != nil {
 		return "", err
 	}

upup/pkg/fi/cloudup/vspheretasks/cloud_init.go

@@ -34,6 +34,11 @@ $VM_UUID
     owner: root:root
     path: /root/vm_uuid
     permissions: "0644"
+  - content: |
+$VOLUME_SCRIPT
+    owner: root:root
+    path: /vol-metadata/metadata.json
+    permissions: "0644"
 
 runcmd:
   - bash /root/update_dns.sh 2>&1 > /var/log/update_dns.log