diff --git a/cloud/google/cmd/gce-machine-controller/main.go b/cloud/google/cmd/gce-machine-controller/main.go index fb5adf62aef6..e9a60ae0e256 100644 --- a/cloud/google/cmd/gce-machine-controller/main.go +++ b/cloud/google/cmd/gce-machine-controller/main.go @@ -55,7 +55,12 @@ func main() { glog.Fatalf("Could not create client for talking to the apiserver: %v", err) } - actuator, err := google.NewMachineActuator(*kubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault), *machineSetupConfigsPath) + params := google.MachineActuatorParams{ + ConfigListPath: *machineSetupConfigsPath, + KubeadmToken: *kubeadmToken, + MachineClient: client.ClusterV1alpha1().Machines(corev1.NamespaceDefault), + } + actuator, err := google.NewMachineActuator(params) if err != nil { glog.Fatalf("Could not create Google machine actuator: %v", err) } diff --git a/cloud/google/gceproviderconfig/types.go b/cloud/google/gceproviderconfig/types.go index 1ca0f76f6304..229dd89fec72 100644 --- a/cloud/google/gceproviderconfig/types.go +++ b/cloud/google/gceproviderconfig/types.go @@ -30,4 +30,15 @@ type GCEProviderConfig struct { // The name of the OS to be installed on the machine. OS string `json:"os"` + Image string `json:"image"` + Disks []Disk `json:"disks"` +} + +type Disk struct { + InitializeParams DiskInitializeParams `json:"initializeParams"` +} + +type DiskInitializeParams struct { + DiskSizeGb int64 `json:"diskSizeGb"` + DiskType string `json:"diskType"` } diff --git a/cloud/google/gceproviderconfig/v1alpha1/types.go b/cloud/google/gceproviderconfig/v1alpha1/types.go index b81885cf49e4..6431a086f8b4 100644 --- a/cloud/google/gceproviderconfig/v1alpha1/types.go +++ b/cloud/google/gceproviderconfig/v1alpha1/types.go @@ -30,4 +30,15 @@ type GCEProviderConfig struct { // The name of the OS to be installed on the machine. OS string `json:"os"` + Image string `json:"image"` + Disks []Disk `json:"disks"` +} + +type Disk struct { + InitializeParams DiskInitializeParams `json:"initializeParams"` +} + +type DiskInitializeParams struct { + DiskSizeGb int64 `json:"diskSizeGb"` + DiskType string `json:"diskType"` } diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go index a75a0a8f8609..2f7c3664ad14 100644 --- a/cloud/google/machineactuator.go +++ b/cloud/google/machineactuator.go @@ -86,20 +86,20 @@ type GCEClient struct { configWatch *machinesetup.ConfigWatch } +type MachineActuatorParams struct { + ComputeService GCEClientComputeService + ConfigListPath string + KubeadmToken string + MachineClient client.MachineInterface +} + const ( gceTimeout = time.Minute * 10 gceWaitSleep = time.Second * 5 ) -func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface, configListPath string) (*GCEClient, error) { - // The default GCP client expects the environment variable - // GOOGLE_APPLICATION_CREDENTIALS to point to a file with service credentials. - client, err := google.DefaultClient(context.TODO(), compute.ComputeScope) - if err != nil { - return nil, err - } - - computeService, err := clients.NewComputeService(client) +func NewMachineActuator(params MachineActuatorParams) (*GCEClient, error) { + computeService, err := getOrCreateComputeService(params) if err != nil { return nil, err } @@ -124,8 +124,8 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa // TODO: get rid of empty string check when we switch to the new bootstrapping method. var configWatch *machinesetup.ConfigWatch - if configListPath != "" { - configWatch, err = machinesetup.NewConfigWatch(configListPath) + if params.ConfigListPath != "" { + configWatch, err = machinesetup.NewConfigWatch(params.ConfigListPath) if err != nil { glog.Errorf("Error creating config watch: %v", err) } @@ -135,13 +135,13 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa computeService: computeService, scheme: scheme, codecFactory: codecFactory, - kubeadmToken: kubeadmToken, + kubeadmToken: params.KubeadmToken, sshCreds: SshCreds{ privateKeyPath: privateKeyPath, user: user, }, - machineClient: machineClient, configWatch: configWatch, + machineClient: params.MachineClient, }, nil } @@ -259,7 +259,6 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach name := machine.ObjectMeta.Name project := config.Project zone := config.Zone - diskSize := int64(30) if instance == nil { labels := map[string]string{} @@ -281,16 +280,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach }, }, }, - Disks: []*compute.AttachedDisk{ - { - AutoDelete: true, - Boot: true, - InitializeParams: &compute.AttachedDiskInitializeParams{ - SourceImage: imagePath, - DiskSizeGb: diskSize, - }, - }, - }, + Disks: buildDisks(config, zone, imagePath, int64(30)), Metadata: &compute.Metadata{ Items: metadataItems, }, @@ -709,6 +699,27 @@ func (gce *GCEClient) getImagePath(img string) (imagePath string) { return defaultImg } +func buildDisks(config *gceconfig.GCEProviderConfig, zone string, imagePath string, minDiskSizeGb int64) []*compute.AttachedDisk { + var disks []*compute.AttachedDisk + for idx, disk := range config.Disks { + diskSizeGb := disk.InitializeParams.DiskSizeGb + if diskSizeGb < minDiskSizeGb { + diskSizeGb = minDiskSizeGb + } + d := compute.AttachedDisk{ + AutoDelete: true, + Boot: idx == 0, + InitializeParams: &compute.AttachedDiskInitializeParams{ + SourceImage: imagePath, + DiskSizeGb: diskSizeGb, + DiskType: fmt.Sprintf("zones/%s/diskTypes/%s", zone, disk.InitializeParams.DiskType), + }, + } + disks = append(disks, &d) + } + return disks +} + // Just a temporary hack to grab a single range from the config. func getSubnet(netRange clusterv1.NetworkRanges) string { if len(netRange.CIDRBlocks) == 0 { @@ -717,6 +728,23 @@ func getSubnet(netRange clusterv1.NetworkRanges) string { return netRange.CIDRBlocks[0] } +func getOrCreateComputeService(params MachineActuatorParams) (GCEClientComputeService, error) { + if params.ComputeService != nil { + return params.ComputeService, nil + } + // The default GCP client expects the environment variable + // GOOGLE_APPLICATION_CREDENTIALS to point to a file with service credentials. + client, err := google.DefaultClient(context.TODO(), compute.ComputeScope) + if err != nil { + return nil, err + } + computeService, err := clients.NewComputeService(client) + if err != nil { + return nil, err + } + return computeService, nil +} + // TODO: We need to change this when we create dedicated service account for apiserver/controller // pod. // diff --git a/cloud/google/machineactuator_test.go b/cloud/google/machineactuator_test.go new file mode 100644 index 000000000000..7db609d507cc --- /dev/null +++ b/cloud/google/machineactuator_test.go @@ -0,0 +1,119 @@ +package google_test + +import ( + "github.com/stretchr/testify/assert" + compute "google.golang.org/api/compute/v1" + "sigs.k8s.io/cluster-api/cloud/google" + "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" + "strings" + "testing" +) + +type GCEClientComputeServiceMock struct { + mockImagesGet func(project string, image string) (*compute.Image, error) + mockImagesGetFromFamily func(project string, family string) (*compute.Image, error) + mockInstancesDelete func(project string, zone string, targetInstance string) (*compute.Operation, error) + mockInstancesGet func(project string, zone string, instance string) (*compute.Instance, error) + mockInstancesInsert func(project string, zone string, instance *compute.Instance) (*compute.Operation, error) + mockZoneOperationsGet func(project string, zone string, operation string) (*compute.Operation, error) +} + +func (c *GCEClientComputeServiceMock) ImagesGet(project string, image string) (*compute.Image, error) { + if c.mockImagesGet == nil { + return nil, nil + } + return c.mockImagesGet(project, image) +} + +func (c *GCEClientComputeServiceMock) ImagesGetFromFamily(project string, family string) (*compute.Image, error) { + if c.mockImagesGetFromFamily == nil { + return nil, nil + } + return c.mockImagesGetFromFamily(project, family) +} + +func (c *GCEClientComputeServiceMock) InstancesDelete(project string, zone string, targetInstance string) (*compute.Operation, error) { + if c.mockInstancesDelete == nil { + return nil, nil + } + return c.mockInstancesDelete(project, zone, targetInstance) +} + +func (c *GCEClientComputeServiceMock) InstancesGet(project string, zone string, instance string) (*compute.Instance, error) { + if c.mockInstancesGet == nil { + return nil, nil + } + return c.mockInstancesGet(project, zone, instance) +} + +func (c *GCEClientComputeServiceMock) InstancesInsert(project string, zone string, instance *compute.Instance) (*compute.Operation, error) { + if c.mockInstancesInsert == nil { + return nil, nil + } + return c.mockInstancesInsert(project, zone, instance) +} + +func (c *GCEClientComputeServiceMock) ZoneOperationsGet(project string, zone string, operation string) (*compute.Operation, error) { + if c.mockZoneOperationsGet == nil { + return nil, nil + } + return c.mockZoneOperationsGet(project, zone, operation) +} + +func TestOneDisk(t *testing.T) { + receivedInstance, computeServiceMock := createInsertInstanceCapturingMock() + err := createCluster("testdata/cluster.yaml", "testdata/machine-one.yaml", computeServiceMock) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if receivedInstance == nil { + t.Error("expected a valid instance") + } + if len(receivedInstance.Disks) != 1 { + t.Errorf("invalid disk count: expected %v got %v", 1, len(receivedInstance.Disks)) + } + checkDiskValues(t, receivedInstance.Disks[0], true, 17, "pd-ssd") +} + +func TestTwoDisks(t *testing.T) { + receivedInstance, computeServiceMock := createInsertInstanceCapturingMock() + err := createCluster("testdata/cluster.yaml", "testdata/machine-two.yaml", computeServiceMock) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if receivedInstance == nil { + t.Error("expected a valid instance") + } + if len(receivedInstance.Disks) != 2 { + t.Errorf("invalid disk count: expected %v got %v", 2, len(receivedInstance.Disks)) + } + checkDiskValues(t, receivedInstance.Disks[0], true, 12, "pd-ssd") + checkDiskValues(t, receivedInstance.Disks[1], false, 15, "pd-standard") +} + +func checkDiskValues(t *testing.T, disk *compute.AttachedDisk, boot bool, sizeGb int64, diskType string) { + assert.Equal(t, boot, disk.Boot) + assert.Equal(t, sizeGb, disk.InitializeParams.DiskSizeGb) + assert.True(t, strings.Contains(disk.InitializeParams.DiskType, diskType)) +} + +func createCluster(clusterYamlFile string, machineYamlFile string, computeServiceMock *GCEClientComputeServiceMock) error { + cluster, _ := v1alpha1.ParseClusterYaml(clusterYamlFile) + machine, _ := v1alpha1.ParseMachineYaml(machineYamlFile) + params := google.MachineActuatorParams{ComputeService: computeServiceMock} + gce, _ := google.NewMachineActuator(params) + return gce.Create(cluster, machine) +} + +func createInsertInstanceCapturingMock() (*compute.Instance, *GCEClientComputeServiceMock) { + var receivedInstance compute.Instance + computeServiceMock := GCEClientComputeServiceMock{ + mockInstancesInsert: func(project string, zone string, instance *compute.Instance) (*compute.Operation, error) { + receivedInstance = *instance + return &compute.Operation{ + Status: "DONE", + }, nil + }, + } + return &receivedInstance, &computeServiceMock +} diff --git a/cloud/google/testdata/cluster.yaml b/cloud/google/testdata/cluster.yaml new file mode 100644 index 000000000000..ab39261cd81d --- /dev/null +++ b/cloud/google/testdata/cluster.yaml @@ -0,0 +1,15 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: Cluster +metadata: + name: test10 +spec: + clusterNetwork: + services: + cidrBlocks: ["10.96.0.0/12"] + pods: + cidrBlocks: ["192.168.0.0/16"] + serviceDomain: "cluster.local" + providerConfig: + value: + apiVersion: "gceproviderconfig/v1alpha1" + kind: "GCEProviderConfig" \ No newline at end of file diff --git a/cloud/google/testdata/machine-one.yaml b/cloud/google/testdata/machine-one.yaml new file mode 100644 index 000000000000..966fa52e3464 --- /dev/null +++ b/cloud/google/testdata/machine-one.yaml @@ -0,0 +1,27 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: Machine +metadata: + generateName: gce-master- + labels: + set: master +spec: + providerConfig: + value: + apiVersion: "gceproviderconfig/v1alpha1" + kind: "GCEProviderConfig" + project: "project-name-1000" + zone: "us-west1-c" + machineType: "n1-standard-2" + image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts" + disks: + - initializeParams: + diskSizeGb: 17 + diskType: "pd-ssd" + versions: + kubelet: 1.8.3 + controlPlane: 1.8.3 + containerRuntime: + name: docker + version: 1.12.0 + roles: + - Master \ No newline at end of file diff --git a/cloud/google/testdata/machine-two.yaml b/cloud/google/testdata/machine-two.yaml new file mode 100644 index 000000000000..778dddbff293 --- /dev/null +++ b/cloud/google/testdata/machine-two.yaml @@ -0,0 +1,30 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: Machine +metadata: + generateName: gce-master- + labels: + set: master +spec: + providerConfig: + value: + apiVersion: "gceproviderconfig/v1alpha1" + kind: "GCEProviderConfig" + project: "project-name-1000" + zone: "us-west1-c" + machineType: "n1-standard-2" + image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts" + disks: + - initializeParams: + diskSizeGb: 12 + diskType: "pd-ssd" + - initializeParams: + diskSizeGb: 15 + diskType: "pd-standard" + versions: + kubelet: 1.8.3 + controlPlane: 1.8.3 + containerRuntime: + name: docker + version: 1.12.0 + roles: + - Master \ No newline at end of file diff --git a/gcp-deployer/cmd/create.go b/gcp-deployer/cmd/create.go index 2a2be289c4f3..0237459a591a 100644 --- a/gcp-deployer/cmd/create.go +++ b/gcp-deployer/cmd/create.go @@ -22,6 +22,7 @@ import ( "github.com/golang/glog" "github.com/spf13/cobra" "sigs.k8s.io/cluster-api/gcp-deployer/deploy" + "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" ) type CreateOptions struct { @@ -59,7 +60,7 @@ var createCmd = &cobra.Command{ } func RunCreate(co *CreateOptions) error { - cluster, err := parseClusterYaml(co.Cluster) + cluster, err := v1alpha1.ParseClusterYaml(co.Cluster) if err != nil { return err } diff --git a/gcp-deployer/cmd/root.go b/gcp-deployer/cmd/root.go index b9161679955b..8f62e54db57b 100644 --- a/gcp-deployer/cmd/root.go +++ b/gcp-deployer/cmd/root.go @@ -49,21 +49,6 @@ func Execute() { } } -func parseClusterYaml(file string) (*clusterv1.Cluster, error) { - bytes, err := ioutil.ReadFile(file) - if err != nil { - return nil, err - } - - cluster := &clusterv1.Cluster{} - err = yaml.Unmarshal(bytes, cluster) - if err != nil { - return nil, err - } - - return cluster, nil -} - func parseMachinesYaml(file string) ([]*clusterv1.Machine, error) { bytes, err := ioutil.ReadFile(file) if err != nil { diff --git a/gcp-deployer/deploy/deploy.go b/gcp-deployer/deploy/deploy.go index ae87f20ff2d8..7943803e0c6d 100644 --- a/gcp-deployer/deploy/deploy.go +++ b/gcp-deployer/deploy/deploy.go @@ -56,7 +56,11 @@ func NewDeployer(provider string, kubeConfigPath string, machineSetupConfigPath glog.Exit(fmt.Sprintf("Failed to set Kubeconfig path err %v\n", err)) } } - ma, err := google.NewMachineActuator(token, nil, machineSetupConfigPath) + params := google.MachineActuatorParams{ + ConfigListPath: machineSetupConfigPath, + KubeadmToken: token, + } + ma, err := google.NewMachineActuator(params) if err != nil { glog.Exit(err) } diff --git a/gcp-deployer/machines.yaml.template b/gcp-deployer/machines.yaml.template index 62c0edeb921b..5d299bdab89d 100644 --- a/gcp-deployer/machines.yaml.template +++ b/gcp-deployer/machines.yaml.template @@ -14,6 +14,10 @@ items: zone: "$ZONE" machineType: "n1-standard-2" os: "ubuntu-1710-weave" + disks: + - initializeParams: + diskSizeGb: 30 + diskType: "pd-standard" versions: kubelet: 1.9.4 controlPlane: 1.9.4 @@ -37,6 +41,10 @@ items: zone: "$ZONE" machineType: "n1-standard-1" os: "ubuntu-1710-weave" + disks: + - initializeParams: + diskSizeGb: 30 + diskType: "pd-standard" versions: kubelet: 1.9.4 containerRuntime: diff --git a/pkg/apis/cluster/v1alpha1/cluster_types.go b/pkg/apis/cluster/v1alpha1/cluster_types.go index 2d3ee939ec0d..b091dcbd6ce8 100644 --- a/pkg/apis/cluster/v1alpha1/cluster_types.go +++ b/pkg/apis/cluster/v1alpha1/cluster_types.go @@ -25,6 +25,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" + "github.com/ghodss/yaml" + "io/ioutil" "sigs.k8s.io/cluster-api/pkg/apis/cluster" "sigs.k8s.io/cluster-api/pkg/apis/cluster/common" ) @@ -141,3 +143,16 @@ func (ClusterSchemeFns) DefaultingFunction(o interface{}) { // set default field values here log.Printf("Defaulting fields for Cluster %s\n", obj.Name) } + +func ParseClusterYaml(file string) (*Cluster, error) { + bytes, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + cluster := &Cluster{} + err = yaml.Unmarshal(bytes, cluster) + if err != nil { + return nil, err + } + return cluster, nil +} diff --git a/pkg/apis/cluster/v1alpha1/machine_types.go b/pkg/apis/cluster/v1alpha1/machine_types.go index 77f02e0308ea..8ef13697117c 100644 --- a/pkg/apis/cluster/v1alpha1/machine_types.go +++ b/pkg/apis/cluster/v1alpha1/machine_types.go @@ -26,6 +26,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" + "github.com/ghodss/yaml" + "io/ioutil" "sigs.k8s.io/cluster-api/pkg/apis/cluster" clustercommon "sigs.k8s.io/cluster-api/pkg/apis/cluster/common" ) @@ -183,3 +185,16 @@ func (MachineSchemeFns) DefaultingFunction(o interface{}) { // set default field values here log.Printf("Defaulting fields for Machine %s\n", obj.Name) } + +func ParseMachineYaml(file string) (*Machine, error) { + bytes, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + machine := &Machine{} + err = yaml.Unmarshal(bytes, &machine) + if err != nil { + return nil, err + } + return machine, nil +}