Handling configurable machine setup/installation

cluster-api/cloud/google/cmd/gce-machine-controller/main.go

@@ -31,7 +31,8 @@ import (
 )
 
 var (
-	kubeadmToken = pflag.String("token", "", "Kubeadm token to use to join new machines")
+	kubeadmToken            = pflag.String("token", "", "Kubeadm token to use to join new machines")
+	machineSetupConfigsPath = pflag.String("machinesetup", "", "path to machine setup configs file")
 )
 
 func init() {
@@ -54,7 +55,7 @@ func main() {
 		glog.Fatalf("Could not create client for talking to the apiserver: %v", err)
 	}
 
-	actuator, err := google.NewMachineActuator(*kubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault))
+	actuator, err := google.NewMachineActuator(*kubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault), *machineSetupConfigsPath)
 	if err != nil {
 		glog.Fatalf("Could not create Google machine actuator: %v", err)
 	}

cluster-api/cloud/google/cmd/generate-image/main.go

@@ -21,13 +21,12 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
-	"k8s.io/kube-deploy/cluster-api/cloud/google"
+	"io/ioutil"
+	"os"
 )
 
 type options struct {
-	version      string
-	role         string
-	dockerImages []string
+	script string
 }
 
 var opts options
@@ -36,37 +35,31 @@ var generateCmd = &cobra.Command{
 	Use:   "generate_image",
 	Short: "Outputs a script to generate a preloaded image",
 	Run: func(cmd *cobra.Command, args []string) {
+		if opts.script == "" {
+			glog.Error("Please provide a startup script.")
+			cmd.Help()
+			os.Exit(1)
+		}
+
 		if err := runGenerate(opts); err != nil {
 			glog.Exit(err)
 		}
 	},
 }
 
 func init() {
-	generateCmd.Flags().StringVar(&opts.version, "version", "1.7.3", "The version of kubernetes to install")
-	generateCmd.Flags().StringVar(&opts.role, "role", "master", "The role of the machine (master or node)")
-	generateCmd.Flags().StringArrayVar(&opts.dockerImages, "extra-docker-images", []string{}, "extra docker images to preload")
+	generateCmd.Flags().StringVar(&opts.script, "script", "", "The path to the machine's startup script")
 }
 
 func runGenerate(o options) error {
-	var script string
-	var err error
-	switch o.role {
-	case "master":
-		script, err = google.PreloadMasterScript(o.version, o.dockerImages)
-	case "node":
-		script, err = google.PreloadMasterScript(o.version, o.dockerImages)
-	default:
-		return fmt.Errorf("unrecognized role: %q", o.role)
-	}
-
+	bytes, err := ioutil.ReadFile(o.script)
 	if err != nil {
 		return err
 	}
 
 	// just print the script for now
 	// TODO actually start a VM, let it run the script, stop the VM, then create the image
-	fmt.Println(script)
+	fmt.Println(string(bytes))
 	return nil
 }
 

cluster-api/cloud/google/config/configtemplate.go

@@ -139,6 +139,8 @@ spec:
             mountPath: /etc/credentials
           - name: sshkeys
             mountPath: /etc/sshkeys
+          - name: machine-setup
+            mountPath: /etc/machinesetup
         env:
           - name: GOOGLE_APPLICATION_CREDENTIALS
             value: /etc/credentials/service-account.json
@@ -147,6 +149,7 @@ spec:
         args:
         - --kubeconfig=/etc/kubernetes/admin.conf
         - --token={{ .Token }}
+        - --machinesetup=/etc/machinesetup/machine_setup_configs.yaml
         resources:
           requests:
             cpu: 100m
@@ -171,6 +174,9 @@ spec:
       - name: credentials
         secret:
           secretName: machine-controller-credential
+      - name: machine-setup
+        configMap: 
+          name: machine-setup
 ---
 apiVersion: apps/v1beta1
 kind: StatefulSet

cluster-api/cloud/google/gceproviderconfig/types.go

@@ -27,5 +27,7 @@ type GCEProviderConfig struct {
 	Project     string `json:"project"`
 	Zone        string `json:"zone"`
 	MachineType string `json:"machineType"`
-	Image       string `json:"image"`
+
+	// The name of the OS to be installed on the machine.
+	OS string `json:"os"`
 }

cluster-api/cloud/google/gceproviderconfig/v1alpha1/types.go

@@ -27,5 +27,7 @@ type GCEProviderConfig struct {
 	Project     string `json:"project"`
 	Zone        string `json:"zone"`
 	MachineType string `json:"machineType"`
-	Image       string `json:"image"`
+
+	// The name of the OS to be installed on the machine.
+	OS string `json:"os"`
 }

cluster-api/cloud/google/machineactuator.go

@@ -37,8 +37,12 @@ import (
 
 	"regexp"
 
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	gceconfig "k8s.io/kube-deploy/cluster-api/cloud/google/gceproviderconfig"
 	gceconfigv1 "k8s.io/kube-deploy/cluster-api/cloud/google/gceproviderconfig/v1alpha1"
+	"k8s.io/kube-deploy/cluster-api/cloud/google/machinesetup"
 	apierrors "k8s.io/kube-deploy/cluster-api/errors"
 	clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1"
 	client "k8s.io/kube-deploy/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1"
@@ -52,6 +56,10 @@ const (
 
 	UIDLabelKey       = "machine-crd-uid"
 	BootstrapLabelKey = "boostrap"
+
+	// This file is a yaml that will be used to create the machine-setup configmap on the machine controller.
+	// It contains the supported machine configurations along with the startup scripts and OS image paths that correspond to each supported configuration.
+	MachineSetupConfigsFilename = "machine_setup_configs.yaml"
 )
 
 type SshCreds struct {
@@ -66,14 +74,15 @@ type GCEClient struct {
 	kubeadmToken  string
 	sshCreds      SshCreds
 	machineClient client.MachineInterface
+	configWatch   *machinesetup.ConfigWatch
 }
 
 const (
 	gceTimeout   = time.Minute * 10
 	gceWaitSleep = time.Second * 5
 )
 
-func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface) (*GCEClient, error) {
+func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface, configListPath string) (*GCEClient, error) {
 	// The default GCP client expects the environment variable
 	// GOOGLE_APPLICATION_CREDENTIALS to point to a file with service credentials.
 	client, err := google.DefaultClient(context.TODO(), compute.ComputeScope)
@@ -104,6 +113,15 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa
 		}
 	}
 
+	// TODO: get rid of empty string check when we switch to the new bootstrapping method.
+	var configWatch *machinesetup.ConfigWatch
+	if configListPath != "" {
+		configWatch, err = machinesetup.NewConfigWatch(configListPath)
+		if err != nil {
+			glog.Errorf("Error creating config watch: %v", err)
+		}
+	}
+
 	return &GCEClient{
 		service:      service,
 		scheme:       scheme,
@@ -114,10 +132,11 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa
 			user:           user,
 		},
 		machineClient: machineClient,
+		configWatch:   configWatch,
 	}, nil
 }
 
-func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error {
+func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine, clientSet kubernetes.Clientset) error {
 	if err := gce.CreateMachineControllerServiceAccount(cluster, initialMachines); err != nil {
 		return err
 	}
@@ -131,6 +150,27 @@ func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initia
 		return err
 	}
 
+	// Create the configmap so the machine setup configs can be mounted into the node.
+	// TODO: create the configmap during bootstrapping instead of being buried in the machine actuator code.
+	machineSetupConfigs, err := gce.configWatch.ValidConfigs()
+	if err != nil {
+		return err
+	}
+	yaml, err := machineSetupConfigs.GetYaml()
+	if err != nil {
+		return err
+	}
+	configMap := corev1.ConfigMap{
+		ObjectMeta: v1.ObjectMeta{Name: "machine-setup"},
+		Data: map[string]string{
+			MachineSetupConfigsFilename: yaml,
+		},
+	}
+	configMaps := clientSet.CoreV1().ConfigMaps(corev1.NamespaceDefault)
+	if _, err := configMaps.Create(&configMap); err != nil {
+		return err
+	}
+
 	if err := CreateApiServerAndController(gce.kubeadmToken); err != nil {
 		return err
 	}
@@ -153,22 +193,32 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 		return errors.New("invalid master configuration: missing Machine.Spec.Versions.Kubelet")
 	}
 
-	image, preloaded := gce.getImage(machine, config)
+	machineSetupConfigs, err := gce.configWatch.ValidConfigs()
+	if err != nil {
+		return err
+	}
+	configParams := &machinesetup.ConfigParams{
+		OS:       config.OS,
+		Roles:    machine.Spec.Roles,
+		Versions: machine.Spec.Versions,
+	}
+	image, err := machineSetupConfigs.GetImage(configParams)
+	if err != nil {
+		return err
+	}
+	imagePath := gce.getImagePath(image)
 
+	machineSetupMetadata, err := machineSetupConfigs.GetMetadata(configParams)
+	if err != nil {
+		return err
+	}
 	if util.IsMaster(machine) {
 		if machine.Spec.Versions.ControlPlane == "" {
 			return gce.handleMachineError(machine, apierrors.InvalidMachineConfiguration(
 				"invalid master configuration: missing Machine.Spec.Versions.ControlPlane"))
 		}
 		var err error
-		metadata, err = masterMetadata(
-			templateParams{
-				Token:     gce.kubeadmToken,
-				Cluster:   cluster,
-				Machine:   machine,
-				Preloaded: preloaded,
-			},
-		)
+		metadata, err = masterMetadata(gce.kubeadmToken, cluster, machine, config.Project, &machineSetupMetadata)
 		if err != nil {
 			return err
 		}
@@ -177,14 +227,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 			return errors.New("invalid cluster state: cannot create a Kubernetes node without an API endpoint")
 		}
 		var err error
-		metadata, err = nodeMetadata(
-			templateParams{
-				Token:     gce.kubeadmToken,
-				Cluster:   cluster,
-				Machine:   machine,
-				Preloaded: preloaded,
-			},
-		)
+		metadata, err = nodeMetadata(gce.kubeadmToken, cluster, machine, &machineSetupMetadata)
 		if err != nil {
 			return err
 		}
@@ -207,13 +250,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 	name := machine.ObjectMeta.Name
 	project := config.Project
 	zone := config.Zone
-	diskSize := int64(10)
-
-	// Our preloaded image already has a lot stored on it, so increase the
-	// disk size to have more free working space.
-	if preloaded {
-		diskSize = 30
-	}
+	diskSize := int64(30)
 
 	if instance == nil {
 		labels := map[string]string{
@@ -242,7 +279,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 					AutoDelete: true,
 					Boot:       true,
 					InitializeParams: &compute.AttachedDiskInitializeParams{
-						SourceImage: image,
+						SourceImage: imagePath,
 						DiskSizeGb:  diskSize,
 					},
 				},
@@ -653,42 +690,29 @@ func (gce *GCEClient) handleMachineError(machine *clusterv1.Machine, err *apierr
 	return err
 }
 
-func (gce *GCEClient) getImage(machine *clusterv1.Machine, config *gceconfig.GCEProviderConfig) (image string, isPreloaded bool) {
+func (gce *GCEClient) getImagePath(img string) (imagePath string) {
 	defaultImg := "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710"
-	project := config.Project
-	img := config.Image
 
-	// A full image path must match the regex format. If it doesn't, we'll assume it's just the image name and try to get it.
-	// If that doesn't work, we will fall back to a default base image.
+	// A full image path must match the regex format. If it doesn't, we will fall back to a default base image.
 	matches := regexp.MustCompile("projects/(.+)/global/images/(family/)*(.+)").FindStringSubmatch(img)
-	if matches == nil {
-		// Only the image name was specified in config, so check if it is preloaded in the project specified in config.
-		fullPath := fmt.Sprintf("projects/%s/global/images/%s", project, img)
-		if _, err := gce.service.Images.Get(project, img).Do(); err == nil {
-			return fullPath, false
+	if matches != nil {
+		// Check to see if the image exists in the given path. The presence of "family" in the path dictates which API call we need to make.
+		project, family, name := matches[1], matches[2], matches[3]
+		var err error
+		if family == "" {
+			_, err = gce.service.Images.Get(project, name).Do()
+		} else {
+			_, err = gce.service.Images.GetFromFamily(project, name).Do()
 		}
 
-		// Otherwise, fall back to the non-preloaded base image.
-		glog.Infof("Could not find image at %s. Defaulting to %s.", fullPath, defaultImg)
-		return defaultImg, false
-	}
-
-	// Check to see if the image exists in the given path. The presence of "family" in the path dictates which API call we need to make.
-	project, family, name := matches[1], matches[2], matches[3]
-	var err error
-	if family == "" {
-		_, err = gce.service.Images.Get(project, name).Do()
-	} else {
-		_, err = gce.service.Images.GetFromFamily(project, name).Do()
-	}
-
-	if err == nil {
-		return img, false
+		if err == nil {
+			return img
+		}
 	}
 
-	// Otherwise, fall back to the non-preloaded base image.
+	// Otherwise, fall back to the base image.
 	glog.Infof("Could not find image at %s. Defaulting to %s.", img, defaultImg)
-	return defaultImg, false
+	return defaultImg
 }
 
 // Just a temporary hack to grab a single range from the config.