Skip to content

Commit

Permalink
Merge pull request #77 from spew/kube-deploy-638
Browse files Browse the repository at this point in the history 
Add the ability to configure disk size and type for GCP clusters (Kube Deploy 638)
  • Loading branch information
@spew
spew authored May 17, 2018
2 parents c0ffba9 + 30eb6d1 commit a81a73f
Show file tree
Hide file tree
Showing 9 changed files with 471 additions and 47 deletions.
8 changes: 6 additions & 2 deletions cloud/google/cmd/gce-machine-controller/app/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,12 @@ func StartMachineController(server *options.MachineControllerServer, shutdown <-
if err != nil {
glog.Fatalf("Could not create config watch: %v", err)
}

actuator, err := google.NewMachineActuator(server.KubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault), configWatch)
params := google.MachineActuatorParams{
KubeadmToken: server.KubeadmToken,
MachineClient: client.ClusterV1alpha1().Machines(corev1.NamespaceDefault),
MachineSetupConfigGetter: configWatch,
}
actuator, err := google.NewMachineActuator(params)
if err != nil {
glog.Fatalf("Could not create Google machine actuator: %v", err)
}
Expand Down
12 changes: 11 additions & 1 deletion cloud/google/gceproviderconfig/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,15 @@ type GCEProviderConfig struct {
MachineType string `json:"machineType"`

// The name of the OS to be installed on the machine.
OS string `json:"os"`
OS string `json:"os"`
Disks []Disk `json:"disks"`
}

type Disk struct {
InitializeParams DiskInitializeParams `json:"initializeParams"`
}

type DiskInitializeParams struct {
DiskSizeGb int64 `json:"diskSizeGb"`
DiskType string `json:"diskType"`
}
66 changes: 61 additions & 5 deletions cloud/google/gceproviderconfig/v1alpha1/register.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,20 @@ limitations under the License.
package v1alpha1

import (
"bytes"
"fmt"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"sigs.k8s.io/cluster-api/cloud/google/gceproviderconfig"
clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
)

type GCEProviderConfigCodec struct {
encoder runtime.Encoder
decoder runtime.Decoder
}

const GroupName = "gceproviderconfig"

var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"}
Expand All @@ -44,14 +52,62 @@ func addKnownTypes(scheme *runtime.Scheme) error {
return nil
}

func NewSchemeAndCodecs() (*runtime.Scheme, *serializer.CodecFactory, error) {
func NewScheme() (*runtime.Scheme, error) {
scheme := runtime.NewScheme()
if err := AddToScheme(scheme); err != nil {
return nil, nil, err
return nil, err
}
if err := gceproviderconfig.AddToScheme(scheme); err != nil {
return nil, nil, err
return nil, err
}
return scheme, nil
}

func NewCodec() (*GCEProviderConfigCodec, error) {
scheme, err := NewScheme()
if err != nil {
return nil, err
}
codecFactory := serializer.NewCodecFactory(scheme)
encoder, err := newEncoder(&codecFactory)
if err != nil {
return nil, err
}
codec := GCEProviderConfigCodec{
encoder: encoder,
decoder: codecFactory.UniversalDecoder(SchemeGroupVersion),
}
return &codec, nil
}

func (codec *GCEProviderConfigCodec) DecodeFromProviderConfig(providerConfig clusterv1.ProviderConfig) (*GCEProviderConfig, error) {
obj, gvk, err := codec.decoder.Decode(providerConfig.Value.Raw, nil, nil)
if err != nil {
return nil, fmt.Errorf("decoding failure: %v", err)
}
config, ok := obj.(*GCEProviderConfig)
if !ok {
return nil, fmt.Errorf("failure to cast to gce; type: %v", gvk)
}
return config, nil
}

func (codec *GCEProviderConfigCodec) EncodeToProviderConfig(gceProviderConfig *GCEProviderConfig) (*clusterv1.ProviderConfig, error) {
var buf bytes.Buffer
if err := codec.encoder.Encode(gceProviderConfig, &buf); err != nil {
return nil, fmt.Errorf("encoding failed: %v", err)
}
providerConfig := clusterv1.ProviderConfig{
Value: &runtime.RawExtension{Raw: buf.Bytes()},
}
return &providerConfig, nil
}

func newEncoder(codecFactory *serializer.CodecFactory) (runtime.Encoder, error) {
serializerInfos := codecFactory.SupportedMediaTypes()
if len(serializerInfos) == 0 {
return nil, fmt.Errorf("unable to find any serlializers")
}
codecs := serializer.NewCodecFactory(scheme)
return scheme, &codecs, nil
encoder := codecFactory.EncoderForVersion(serializerInfos[0].Serializer, SchemeGroupVersion)
return encoder, nil
}
12 changes: 11 additions & 1 deletion cloud/google/gceproviderconfig/v1alpha1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,15 @@ type GCEProviderConfig struct {
MachineType string `json:"machineType"`

// The name of the OS to be installed on the machine.
OS string `json:"os"`
OS string `json:"os"`
Disks []Disk `json:"disks"`
}

type Disk struct {
InitializeParams DiskInitializeParams `json:"initializeParams"`
}

type DiskInitializeParams struct {
DiskSizeGb int64 `json:"diskSizeGb"`
DiskType string `json:"diskType"`
}
97 changes: 62 additions & 35 deletions cloud/google/machineactuator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ import (

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"

"regexp"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/cluster-api/cloud/google/clients"
gceconfigv1 "sigs.k8s.io/cluster-api/cloud/google/gceproviderconfig/v1alpha1"
Expand Down Expand Up @@ -81,6 +81,7 @@ type GCEClientComputeService interface {

type GCEClient struct {
computeService GCEClientComputeService
gceProviderConfigCodec *gceconfigv1.GCEProviderConfigCodec
scheme *runtime.Scheme
codecFactory *serializer.CodecFactory
kubeadmToken string
Expand All @@ -89,25 +90,29 @@ type GCEClient struct {
machineSetupConfigGetter GCEClientMachineSetupConfigGetter
}

type MachineActuatorParams struct {
ComputeService GCEClientComputeService
KubeadmToken string
MachineClient client.MachineInterface
MachineSetupConfigGetter GCEClientMachineSetupConfigGetter
}

const (
gceTimeout = time.Minute * 10
gceWaitSleep = time.Second * 5
)

func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface, machineSetupConfigGetter GCEClientMachineSetupConfigGetter) (*GCEClient, error) {
// The default GCP client expects the environment variable
// GOOGLE_APPLICATION_CREDENTIALS to point to a file with service credentials.
client, err := google.DefaultClient(context.TODO(), compute.ComputeScope)
func NewMachineActuator(params MachineActuatorParams) (*GCEClient, error) {
computeService, err := getOrNewComputeService(params)
if err != nil {
return nil, err
}

computeService, err := clients.NewComputeService(client)
scheme, err := gceconfigv1.NewScheme()
if err != nil {
return nil, err
}

scheme, codecFactory, err := gceconfigv1.NewSchemeAndCodecs()
codec, err := gceconfigv1.NewCodec()
if err != nil {
return nil, err
}
Expand All @@ -126,16 +131,16 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa
}

return &GCEClient{
computeService: computeService,
scheme: scheme,
codecFactory: codecFactory,
kubeadmToken: kubeadmToken,
computeService: computeService,
scheme: scheme,
gceProviderConfigCodec: codec,
kubeadmToken: params.KubeadmToken,
sshCreds: SshCreds{
privateKeyPath: privateKeyPath,
user: user,
},
machineClient: machineClient,
machineSetupConfigGetter: machineSetupConfigGetter,
machineClient: params.MachineClient,
machineSetupConfigGetter: params.MachineSetupConfigGetter,
}, nil
}

Expand Down Expand Up @@ -266,7 +271,6 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
name := machine.ObjectMeta.Name
project := config.Project
zone := config.Zone
diskSize := int64(30)

if instance == nil {
labels := map[string]string{}
Expand All @@ -289,16 +293,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
},
},
},
Disks: []*compute.AttachedDisk{
{
AutoDelete: true,
Boot: true,
InitializeParams: &compute.AttachedDiskInitializeParams{
SourceImage: imagePath,
DiskSizeGb: diskSize,
},
},
},
Disks: newDisks(config, zone, imagePath, int64(30)),
Metadata: &compute.Metadata{
Items: metadataItems,
},
Expand Down Expand Up @@ -606,16 +601,7 @@ func (gce *GCEClient) instanceIfExists(machine *clusterv1.Machine) (*compute.Ins
}

func (gce *GCEClient) providerconfig(providerConfig clusterv1.ProviderConfig) (*gceconfigv1.GCEProviderConfig, error) {
obj, gvk, err := gce.codecFactory.UniversalDecoder(gceconfigv1.SchemeGroupVersion).Decode(providerConfig.Value.Raw, nil, nil)
if err != nil {
return nil, fmt.Errorf("decoding failure: %v", err)
}
config, ok := obj.(*gceconfigv1.GCEProviderConfig)
if !ok {
return nil, fmt.Errorf("failure to cast to gce; type: %v", gvk)
}

return config, nil
return gce.gceProviderConfigCodec.DecodeFromProviderConfig(providerConfig)
}

func (gce *GCEClient) waitForOperation(c *gceconfigv1.GCEProviderConfig, op *compute.Operation) error {
Expand Down Expand Up @@ -756,6 +742,30 @@ func (gce *GCEClient) getImagePath(img string) (imagePath string) {
return defaultImg
}

func newDisks(config *gceconfigv1.GCEProviderConfig, zone string, imagePath string, minDiskSizeGb int64) []*compute.AttachedDisk {
var disks []*compute.AttachedDisk
for idx, disk := range config.Disks {
diskSizeGb := disk.InitializeParams.DiskSizeGb
d := compute.AttachedDisk{
AutoDelete: true,
InitializeParams: &compute.AttachedDiskInitializeParams{
DiskSizeGb: diskSizeGb,
DiskType: fmt.Sprintf("zones/%s/diskTypes/%s", zone, disk.InitializeParams.DiskType),
},
}
if idx == 0 {
d.InitializeParams.SourceImage = imagePath
d.Boot = true
if diskSizeGb < minDiskSizeGb {
glog.Info("increasing disk size to %v gb, the supplied disk size of %v gb is below the minimum", minDiskSizeGb, diskSizeGb)
d.InitializeParams.DiskSizeGb = minDiskSizeGb
}
}
disks = append(disks, &d)
}
return disks
}

// Just a temporary hack to grab a single range from the config.
func getSubnet(netRange clusterv1.NetworkRanges) string {
if len(netRange.CIDRBlocks) == 0 {
Expand All @@ -764,6 +774,23 @@ func getSubnet(netRange clusterv1.NetworkRanges) string {
return netRange.CIDRBlocks[0]
}

func getOrNewComputeService(params MachineActuatorParams) (GCEClientComputeService, error) {
if params.ComputeService != nil {
return params.ComputeService, nil
}
// The default GCP client expects the environment variable
// GOOGLE_APPLICATION_CREDENTIALS to point to a file with service credentials.
client, err := google.DefaultClient(context.TODO(), compute.ComputeScope)
if err != nil {
return nil, err
}
computeService, err := clients.NewComputeService(client)
if err != nil {
return nil, err
}
return computeService, nil
}

// TODO: We need to change this when we create dedicated service account for apiserver/controller
// pod.
//
Expand Down
Loading

0 comments on commit a81a73f

Please sign in to comment.