Skip to content

Commit

Permalink
Add the ability to configure disk size and type for GCP clusters
Browse files Browse the repository at this point in the history 
This change adds the ability for GCP clusters to have custom sized disks
and types.
  • Loading branch information
@spew
spew committed May 9, 2018
1 parent fc4dc6a commit aae9762
Show file tree
Hide file tree
Showing 9 changed files with 459 additions and 47 deletions.
8 changes: 6 additions & 2 deletions cloud/google/cmd/gce-machine-controller/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,12 @@ func main() {
if err != nil {
glog.Fatalf("Could not create config watch: %v", err)
}

actuator, err := google.NewMachineActuator(*kubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault), configWatch)
params := google.MachineActuatorParams{
MachineSetupConfigGetter: configWatch,
KubeadmToken: *kubeadmToken,
MachineClient: client.ClusterV1alpha1().Machines(corev1.NamespaceDefault),
}
actuator, err := google.NewMachineActuator(params)
if err != nil {
glog.Fatalf("Could not create Google machine actuator: %v", err)
}
Expand Down
12 changes: 11 additions & 1 deletion cloud/google/gceproviderconfig/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,15 @@ type GCEProviderConfig struct {
MachineType string `json:"machineType"`

// The name of the OS to be installed on the machine.
OS string `json:"os"`
OS string `json:"os"`
Disks []Disk `json:"disks"`
}

type Disk struct {
InitializeParams DiskInitializeParams `json:"initializeParams"`
}

type DiskInitializeParams struct {
DiskSizeGb int64 `json:"diskSizeGb"`
DiskType string `json:"diskType"`
}
66 changes: 61 additions & 5 deletions cloud/google/gceproviderconfig/v1alpha1/register.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,20 @@ limitations under the License.
package v1alpha1

import (
"bytes"
"fmt"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"sigs.k8s.io/cluster-api/cloud/google/gceproviderconfig"
clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
)

type GCEProviderConfigCodec struct {
encoder runtime.Encoder
decoder runtime.Decoder
}

const GroupName = "gceproviderconfig"

var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"}
Expand All @@ -44,14 +52,62 @@ func addKnownTypes(scheme *runtime.Scheme) error {
return nil
}

func NewSchemeAndCodecs() (*runtime.Scheme, *serializer.CodecFactory, error) {
func NewScheme() (*runtime.Scheme, error) {
scheme := runtime.NewScheme()
if err := AddToScheme(scheme); err != nil {
return nil, nil, err
return nil, err
}
if err := gceproviderconfig.AddToScheme(scheme); err != nil {
return nil, nil, err
return nil, err
}
return scheme, nil
}

func NewCodec() (*GCEProviderConfigCodec, error) {
scheme, err := NewScheme()
if err != nil {
return nil, err
}
codecFactory := serializer.NewCodecFactory(scheme)
encoder, err := newEncoder(&codecFactory)
if err != nil {
return nil, err
}
codec := GCEProviderConfigCodec{
encoder: encoder,
decoder: codecFactory.UniversalDecoder(SchemeGroupVersion),
}
return &codec, nil
}

func (codec *GCEProviderConfigCodec) DecodeFromProviderConfig(providerConfig clusterv1.ProviderConfig) (*GCEProviderConfig, error) {
obj, gvk, err := codec.decoder.Decode(providerConfig.Value.Raw, nil, nil)
if err != nil {
return nil, fmt.Errorf("decoding failure: %v", err)
}
config, ok := obj.(*GCEProviderConfig)
if !ok {
return nil, fmt.Errorf("failure to cast to gce; type: %v", gvk)
}
return config, nil
}

func (codec *GCEProviderConfigCodec) EncodeToProviderConfig(gceProviderConfig *GCEProviderConfig) (*clusterv1.ProviderConfig, error) {
var buf bytes.Buffer
if err := codec.encoder.Encode(gceProviderConfig, &buf); err != nil {
return nil, fmt.Errorf("encoding failed: %v", err)
}
providerConfig := clusterv1.ProviderConfig{
Value: &runtime.RawExtension{Raw: buf.Bytes()},
}
return &providerConfig, nil
}

func newEncoder(codecFactory *serializer.CodecFactory) (runtime.Encoder, error) {
serializerInfos := codecFactory.SupportedMediaTypes()
if len(serializerInfos) == 0 {
return nil, fmt.Errorf("unable to find any serlializers")
}
codecs := serializer.NewCodecFactory(scheme)
return scheme, &codecs, nil
encoder := codecFactory.EncoderForVersion(serializerInfos[0].Serializer, SchemeGroupVersion)
return encoder, nil
}
12 changes: 11 additions & 1 deletion cloud/google/gceproviderconfig/v1alpha1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,15 @@ type GCEProviderConfig struct {
MachineType string `json:"machineType"`

// The name of the OS to be installed on the machine.
OS string `json:"os"`
OS string `json:"os"`
Disks []Disk `json:"disks"`
}

type Disk struct {
InitializeParams DiskInitializeParams `json:"initializeParams"`
}

type DiskInitializeParams struct {
DiskSizeGb int64 `json:"diskSizeGb"`
DiskType string `json:"diskType"`
}
95 changes: 60 additions & 35 deletions cloud/google/machineactuator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ import (

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"

"regexp"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/cluster-api/cloud/google/clients"
gceconfigv1 "sigs.k8s.io/cluster-api/cloud/google/gceproviderconfig/v1alpha1"
Expand Down Expand Up @@ -81,6 +81,7 @@ type GCEClientComputeService interface {

type GCEClient struct {
computeService GCEClientComputeService
gceProviderConfigCodec *gceconfigv1.GCEProviderConfigCodec
scheme *runtime.Scheme
codecFactory *serializer.CodecFactory
kubeadmToken string
Expand All @@ -89,25 +90,29 @@ type GCEClient struct {
machineSetupConfigGetter GCEClientMachineSetupConfigGetter
}

type MachineActuatorParams struct {
ComputeService GCEClientComputeService
KubeadmToken string
MachineClient client.MachineInterface
MachineSetupConfigGetter GCEClientMachineSetupConfigGetter
}

const (
gceTimeout = time.Minute * 10
gceWaitSleep = time.Second * 5
)

func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface, machineSetupConfigGetter GCEClientMachineSetupConfigGetter) (*GCEClient, error) {
// The default GCP client expects the environment variable
// GOOGLE_APPLICATION_CREDENTIALS to point to a file with service credentials.
client, err := google.DefaultClient(context.TODO(), compute.ComputeScope)
func NewMachineActuator(params MachineActuatorParams) (*GCEClient, error) {
computeService, err := getOrNewComputeService(params)
if err != nil {
return nil, err
}

computeService, err := clients.NewComputeService(client)
scheme, err := gceconfigv1.NewScheme()
if err != nil {
return nil, err
}

scheme, codecFactory, err := gceconfigv1.NewSchemeAndCodecs()
codec, err := gceconfigv1.NewCodec()
if err != nil {
return nil, err
}
Expand All @@ -126,16 +131,16 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa
}

return &GCEClient{
computeService: computeService,
scheme: scheme,
codecFactory: codecFactory,
kubeadmToken: kubeadmToken,
computeService: computeService,
scheme: scheme,
gceProviderConfigCodec: codec,
kubeadmToken: params.KubeadmToken,
sshCreds: SshCreds{
privateKeyPath: privateKeyPath,
user: user,
},
machineClient: machineClient,
machineSetupConfigGetter: machineSetupConfigGetter,
machineClient: params.MachineClient,
machineSetupConfigGetter: params.MachineSetupConfigGetter,
}, nil
}

Expand Down Expand Up @@ -257,7 +262,6 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
name := machine.ObjectMeta.Name
project := config.Project
zone := config.Zone
diskSize := int64(30)

if instance == nil {
labels := map[string]string{}
Expand Down Expand Up @@ -292,16 +296,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
},
},
},
Disks: []*compute.AttachedDisk{
{
AutoDelete: true,
Boot: true,
InitializeParams: &compute.AttachedDiskInitializeParams{
SourceImage: imagePath,
DiskSizeGb: diskSize,
},
},
},
Disks: newDisks(config, zone, imagePath, int64(30)),
Metadata: &compute.Metadata{
Items: metadataItems,
},
Expand Down Expand Up @@ -569,16 +564,7 @@ func (gce *GCEClient) instanceIfExists(machine *clusterv1.Machine) (*compute.Ins
}

func (gce *GCEClient) providerconfig(providerConfig clusterv1.ProviderConfig) (*gceconfigv1.GCEProviderConfig, error) {
obj, gvk, err := gce.codecFactory.UniversalDecoder(gceconfigv1.SchemeGroupVersion).Decode(providerConfig.Value.Raw, nil, nil)
if err != nil {
return nil, fmt.Errorf("decoding failure: %v", err)
}
config, ok := obj.(*gceconfigv1.GCEProviderConfig)
if !ok {
return nil, fmt.Errorf("failure to cast to gce; type: %v", gvk)
}

return config, nil
return gce.gceProviderConfigCodec.DecodeFromProviderConfig(providerConfig)
}

func (gce *GCEClient) waitForOperation(c *gceconfigv1.GCEProviderConfig, op *compute.Operation) error {
Expand Down Expand Up @@ -719,6 +705,28 @@ func (gce *GCEClient) getImagePath(img string) (imagePath string) {
return defaultImg
}

func newDisks(config *gceconfigv1.GCEProviderConfig, zone string, imagePath string, minDiskSizeGb int64) []*compute.AttachedDisk {
var disks []*compute.AttachedDisk
for idx, disk := range config.Disks {
diskSizeGb := disk.InitializeParams.DiskSizeGb
if diskSizeGb < minDiskSizeGb {
glog.Info("increasing disk size to %$v gb, the supplied disk size of %v gb is below the minimum", minDiskSizeGb, diskSizeGb)
diskSizeGb = minDiskSizeGb
}
d := compute.AttachedDisk{
AutoDelete: true,
Boot: idx == 0,
InitializeParams: &compute.AttachedDiskInitializeParams{
SourceImage: imagePath,
DiskSizeGb: diskSizeGb,
DiskType: fmt.Sprintf("zones/%s/diskTypes/%s", zone, disk.InitializeParams.DiskType),
},
}
disks = append(disks, &d)
}
return disks
}

// Just a temporary hack to grab a single range from the config.
func getSubnet(netRange clusterv1.NetworkRanges) string {
if len(netRange.CIDRBlocks) == 0 {
Expand All @@ -727,6 +735,23 @@ func getSubnet(netRange clusterv1.NetworkRanges) string {
return netRange.CIDRBlocks[0]
}

func getOrNewComputeService(params MachineActuatorParams) (GCEClientComputeService, error) {
if params.ComputeService != nil {
return params.ComputeService, nil
}
// The default GCP client expects the environment variable
// GOOGLE_APPLICATION_CREDENTIALS to point to a file with service credentials.
client, err := google.DefaultClient(context.TODO(), compute.ComputeScope)
if err != nil {
return nil, err
}
computeService, err := clients.NewComputeService(client)
if err != nil {
return nil, err
}
return computeService, nil
}

// TODO: We need to change this when we create dedicated service account for apiserver/controller
// pod.
//
Expand Down
Loading

0 comments on commit aae9762

Please sign in to comment.