From d4cc9e2b4cb346a5d1ad365962a03d1ebf835d69 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Wed, 18 Apr 2018 14:15:31 -0700 Subject: [PATCH 1/3] Replace hardcoded startup scripts with machine setup configs - Rename GCEProviderConfig.Image field to OS - Remove call to preloaded script in generate-image - Create a machine-setup ConfigMap to mount the yaml file --- .../google/cmd/gce-machine-controller/main.go | 5 +- cloud/google/cmd/generate-image/main.go | 32 +- cloud/google/config/configtemplate.go | 6 + cloud/google/gceproviderconfig/types.go | 4 +- .../gceproviderconfig/v1alpha1/types.go | 4 +- cloud/google/machineactuator.go | 154 ++++--- cloud/google/machinesetup/config_types.go | 155 +++++++ .../google/machinesetup/config_types_test.go | 420 ++++++++++++++++++ cloud/google/metadata.go | 126 ++++++ cloud/google/templates.go | 321 ------------- gcp-deployer/cmd/add.go | 2 +- gcp-deployer/cmd/create.go | 13 +- gcp-deployer/cmd/delete.go | 2 +- gcp-deployer/deploy/deploy.go | 28 +- gcp-deployer/deploy/deploy_helper.go | 14 +- gcp-deployer/deploy/machinedeployer.go | 3 +- gcp-deployer/machine_setup_configs.yaml | 320 +++++++++++++ gcp-deployer/machines.yaml.template | 10 +- 18 files changed, 1187 insertions(+), 432 deletions(-) create mode 100644 cloud/google/machinesetup/config_types.go create mode 100644 cloud/google/machinesetup/config_types_test.go create mode 100644 cloud/google/metadata.go delete mode 100644 cloud/google/templates.go create mode 100644 gcp-deployer/machine_setup_configs.yaml diff --git a/cloud/google/cmd/gce-machine-controller/main.go b/cloud/google/cmd/gce-machine-controller/main.go index f05038253fc1..64330ef43244 100644 --- a/cloud/google/cmd/gce-machine-controller/main.go +++ b/cloud/google/cmd/gce-machine-controller/main.go @@ -31,7 +31,8 @@ import ( ) var ( - kubeadmToken = pflag.String("token", "", "Kubeadm token to use to join new machines") + kubeadmToken = pflag.String("token", "", "Kubeadm token to use to join new machines") + machineSetupConfigsPath = pflag.String("machinesetup", "", "path to machine setup configs file") ) func init() { @@ -54,7 +55,7 @@ func main() { glog.Fatalf("Could not create client for talking to the apiserver: %v", err) } - actuator, err := google.NewMachineActuator(*kubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault)) + actuator, err := google.NewMachineActuator(*kubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault), *machineSetupConfigsPath) if err != nil { glog.Fatalf("Could not create Google machine actuator: %v", err) } diff --git a/cloud/google/cmd/generate-image/main.go b/cloud/google/cmd/generate-image/main.go index 423207afcd0e..e3b1468239bf 100644 --- a/cloud/google/cmd/generate-image/main.go +++ b/cloud/google/cmd/generate-image/main.go @@ -19,15 +19,15 @@ package main import ( "fmt" + "io/ioutil" + "os" + "github.com/golang/glog" "github.com/spf13/cobra" - "sigs.k8s.io/cluster-api/cloud/google" ) type options struct { - version string - role string - dockerImages []string + script string } var opts options @@ -36,6 +36,12 @@ var generateCmd = &cobra.Command{ Use: "generate_image", Short: "Outputs a script to generate a preloaded image", Run: func(cmd *cobra.Command, args []string) { + if opts.script == "" { + glog.Error("Please provide a startup script.") + cmd.Help() + os.Exit(1) + } + if err := runGenerate(opts); err != nil { glog.Exit(err) } @@ -43,30 +49,18 @@ var generateCmd = &cobra.Command{ } func init() { - generateCmd.Flags().StringVar(&opts.version, "version", "1.7.3", "The version of kubernetes to install") - generateCmd.Flags().StringVar(&opts.role, "role", "master", "The role of the machine (master or node)") - generateCmd.Flags().StringArrayVar(&opts.dockerImages, "extra-docker-images", []string{}, "extra docker images to preload") + generateCmd.Flags().StringVar(&opts.script, "script", "", "The path to the machine's startup script") } func runGenerate(o options) error { - var script string - var err error - switch o.role { - case "master": - script, err = google.PreloadMasterScript(o.version, o.dockerImages) - case "node": - script, err = google.PreloadMasterScript(o.version, o.dockerImages) - default: - return fmt.Errorf("unrecognized role: %q", o.role) - } - + bytes, err := ioutil.ReadFile(o.script) if err != nil { return err } // just print the script for now // TODO actually start a VM, let it run the script, stop the VM, then create the image - fmt.Println(script) + fmt.Println(string(bytes)) return nil } diff --git a/cloud/google/config/configtemplate.go b/cloud/google/config/configtemplate.go index 82b412d41356..81c600706ccc 100644 --- a/cloud/google/config/configtemplate.go +++ b/cloud/google/config/configtemplate.go @@ -139,6 +139,8 @@ spec: mountPath: /etc/credentials - name: sshkeys mountPath: /etc/sshkeys + - name: machine-setup + mountPath: /etc/machinesetup env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/credentials/service-account.json @@ -147,6 +149,7 @@ spec: args: - --kubeconfig=/etc/kubernetes/admin.conf - --token={{ .Token }} + - --machinesetup=/etc/machinesetup/machine_setup_configs.yaml resources: requests: cpu: 100m @@ -171,6 +174,9 @@ spec: - name: credentials secret: secretName: machine-controller-credential + - name: machine-setup + configMap: + name: machine-setup --- apiVersion: apps/v1beta1 kind: StatefulSet diff --git a/cloud/google/gceproviderconfig/types.go b/cloud/google/gceproviderconfig/types.go index f6951f6f4641..1ca0f76f6304 100644 --- a/cloud/google/gceproviderconfig/types.go +++ b/cloud/google/gceproviderconfig/types.go @@ -27,5 +27,7 @@ type GCEProviderConfig struct { Project string `json:"project"` Zone string `json:"zone"` MachineType string `json:"machineType"` - Image string `json:"image"` + + // The name of the OS to be installed on the machine. + OS string `json:"os"` } diff --git a/cloud/google/gceproviderconfig/v1alpha1/types.go b/cloud/google/gceproviderconfig/v1alpha1/types.go index 9ad6940f9f87..b81885cf49e4 100644 --- a/cloud/google/gceproviderconfig/v1alpha1/types.go +++ b/cloud/google/gceproviderconfig/v1alpha1/types.go @@ -27,5 +27,7 @@ type GCEProviderConfig struct { Project string `json:"project"` Zone string `json:"zone"` MachineType string `json:"machineType"` - Image string `json:"image"` + + // The name of the OS to be installed on the machine. + OS string `json:"os"` } diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go index 7b6b7beb62b8..55875d2bf308 100644 --- a/cloud/google/machineactuator.go +++ b/cloud/google/machineactuator.go @@ -32,13 +32,17 @@ import ( "golang.org/x/oauth2/google" compute "google.golang.org/api/compute/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "regexp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" gceconfig "sigs.k8s.io/cluster-api/cloud/google/gceproviderconfig" gceconfigv1 "sigs.k8s.io/cluster-api/cloud/google/gceproviderconfig/v1alpha1" + "sigs.k8s.io/cluster-api/cloud/google/machinesetup" apierrors "sigs.k8s.io/cluster-api/errors" clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" client "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1" @@ -50,7 +54,12 @@ const ( ZoneAnnotationKey = "gcp-zone" NameAnnotationKey = "gcp-name" + UIDLabelKey = "machine-crd-uid" BootstrapLabelKey = "boostrap" + + // This file is a yaml that will be used to create the machine-setup configmap on the machine controller. + // It contains the supported machine configurations along with the startup scripts and OS image paths that correspond to each supported configuration. + MachineSetupConfigsFilename = "machine_setup_configs.yaml" ) type SshCreds struct { @@ -65,6 +74,7 @@ type GCEClient struct { kubeadmToken string sshCreds SshCreds machineClient client.MachineInterface + configWatch *machinesetup.ConfigWatch } const ( @@ -72,7 +82,7 @@ const ( gceWaitSleep = time.Second * 5 ) -func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface) (*GCEClient, error) { +func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface, configListPath string) (*GCEClient, error) { // The default GCP client expects the environment variable // GOOGLE_APPLICATION_CREDENTIALS to point to a file with service credentials. client, err := google.DefaultClient(context.TODO(), compute.ComputeScope) @@ -103,6 +113,15 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa } } + // TODO: get rid of empty string check when we switch to the new bootstrapping method. + var configWatch *machinesetup.ConfigWatch + if configListPath != "" { + configWatch, err = machinesetup.NewConfigWatch(configListPath) + if err != nil { + glog.Errorf("Error creating config watch: %v", err) + } + } + return &GCEClient{ service: service, scheme: scheme, @@ -113,10 +132,11 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa user: user, }, machineClient: machineClient, + configWatch: configWatch, }, nil } -func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { +func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine, clientSet kubernetes.Clientset) error { if err := gce.CreateMachineControllerServiceAccount(cluster, initialMachines); err != nil { return err } @@ -130,6 +150,27 @@ func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initia return err } + // Create the configmap so the machine setup configs can be mounted into the node. + // TODO: create the configmap during bootstrapping instead of being buried in the machine actuator code. + machineSetupConfigs, err := gce.configWatch.ValidConfigs() + if err != nil { + return err + } + yaml, err := machineSetupConfigs.GetYaml() + if err != nil { + return err + } + configMap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "machine-setup"}, + Data: map[string]string{ + MachineSetupConfigsFilename: yaml, + }, + } + configMaps := clientSet.CoreV1().ConfigMaps(corev1.NamespaceDefault) + if _, err := configMaps.Create(&configMap); err != nil { + return err + } + if err := CreateApiServerAndController(gce.kubeadmToken); err != nil { return err } @@ -152,22 +193,32 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach return errors.New("invalid master configuration: missing Machine.Spec.Versions.Kubelet") } - image, preloaded := gce.getImage(machine, config) + machineSetupConfigs, err := gce.configWatch.ValidConfigs() + if err != nil { + return err + } + configParams := &machinesetup.ConfigParams{ + OS: config.OS, + Roles: machine.Spec.Roles, + Versions: machine.Spec.Versions, + } + image, err := machineSetupConfigs.GetImage(configParams) + if err != nil { + return err + } + imagePath := gce.getImagePath(image) + machineSetupMetadata, err := machineSetupConfigs.GetMetadata(configParams) + if err != nil { + return err + } if util.IsMaster(machine) { if machine.Spec.Versions.ControlPlane == "" { return gce.handleMachineError(machine, apierrors.InvalidMachineConfiguration( "invalid master configuration: missing Machine.Spec.Versions.ControlPlane")) } var err error - metadata, err = masterMetadata( - templateParams{ - Token: gce.kubeadmToken, - Cluster: cluster, - Machine: machine, - Preloaded: preloaded, - }, - ) + metadata, err = masterMetadata(gce.kubeadmToken, cluster, machine, config.Project, &machineSetupMetadata) if err != nil { return err } @@ -176,14 +227,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach return errors.New("invalid cluster state: cannot create a Kubernetes node without an API endpoint") } var err error - metadata, err = nodeMetadata( - templateParams{ - Token: gce.kubeadmToken, - Cluster: cluster, - Machine: machine, - Preloaded: preloaded, - }, - ) + metadata, err = nodeMetadata(gce.kubeadmToken, cluster, machine, &machineSetupMetadata) if err != nil { return err } @@ -206,16 +250,12 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach name := machine.ObjectMeta.Name project := config.Project zone := config.Zone - diskSize := int64(10) - - // Our preloaded image already has a lot stored on it, so increase the - // disk size to have more free working space. - if preloaded { - diskSize = 30 - } + diskSize := int64(30) if instance == nil { - labels := map[string]string{} + labels := map[string]string{ + UIDLabelKey: fmt.Sprintf("%v", machine.ObjectMeta.UID), + } if gce.machineClient == nil { labels[BootstrapLabelKey] = "true" } @@ -239,7 +279,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach AutoDelete: true, Boot: true, InitializeParams: &compute.AttachedDiskInitializeParams{ - SourceImage: image, + SourceImage: imagePath, DiskSizeGb: diskSize, }, }, @@ -471,7 +511,8 @@ func (gce *GCEClient) requiresUpdate(a *clusterv1.Machine, b *clusterv1.Machine) !reflect.DeepEqual(a.Spec.ProviderConfig, b.Spec.ProviderConfig) || !reflect.DeepEqual(a.Spec.Roles, b.Spec.Roles) || !reflect.DeepEqual(a.Spec.Versions, b.Spec.Versions) || - a.ObjectMeta.Name != b.ObjectMeta.Name + a.ObjectMeta.Name != b.ObjectMeta.Name || + a.ObjectMeta.UID != b.ObjectMeta.UID } // Gets the instance represented by the given machine @@ -504,6 +545,18 @@ func (gce *GCEClient) instanceIfExists(machine *clusterv1.Machine) (*compute.Ins return nil, err } + uid := instance.Labels[UIDLabelKey] + if uid == "" { + if instance.Labels[BootstrapLabelKey] != "" { + glog.Infof("Skipping uid check since instance %v %v %v is missing uid label due to being provisioned as part of bootstrap.", config.Project, config.Zone, identifyingMachine.ObjectMeta.Name) + } else { + return nil, fmt.Errorf("Instance %v %v %v is missing uid label.", config.Project, config.Zone, identifyingMachine.ObjectMeta.Name) + } + } else if uid != fmt.Sprintf("%v", machine.ObjectMeta.UID) { + glog.Infof("Instance %v exists but it has a different UID. Object UID: %v . Instance UID: %v", machine.ObjectMeta.Name, machine.ObjectMeta.UID, uid) + return nil, nil + } + return instance, nil } @@ -637,42 +690,29 @@ func (gce *GCEClient) handleMachineError(machine *clusterv1.Machine, err *apierr return err } -func (gce *GCEClient) getImage(machine *clusterv1.Machine, config *gceconfig.GCEProviderConfig) (image string, isPreloaded bool) { +func (gce *GCEClient) getImagePath(img string) (imagePath string) { defaultImg := "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710" - project := config.Project - img := config.Image - // A full image path must match the regex format. If it doesn't, we'll assume it's just the image name and try to get it. - // If that doesn't work, we will fall back to a default base image. + // A full image path must match the regex format. If it doesn't, we will fall back to a default base image. matches := regexp.MustCompile("projects/(.+)/global/images/(family/)*(.+)").FindStringSubmatch(img) - if matches == nil { - // Only the image name was specified in config, so check if it is preloaded in the project specified in config. - fullPath := fmt.Sprintf("projects/%s/global/images/%s", project, img) - if _, err := gce.service.Images.Get(project, img).Do(); err == nil { - return fullPath, false + if matches != nil { + // Check to see if the image exists in the given path. The presence of "family" in the path dictates which API call we need to make. + project, family, name := matches[1], matches[2], matches[3] + var err error + if family == "" { + _, err = gce.service.Images.Get(project, name).Do() + } else { + _, err = gce.service.Images.GetFromFamily(project, name).Do() } - // Otherwise, fall back to the non-preloaded base image. - glog.Infof("Could not find image at %s. Defaulting to %s.", fullPath, defaultImg) - return defaultImg, false - } - - // Check to see if the image exists in the given path. The presence of "family" in the path dictates which API call we need to make. - project, family, name := matches[1], matches[2], matches[3] - var err error - if family == "" { - _, err = gce.service.Images.Get(project, name).Do() - } else { - _, err = gce.service.Images.GetFromFamily(project, name).Do() - } - - if err == nil { - return img, false + if err == nil { + return img + } } - // Otherwise, fall back to the non-preloaded base image. + // Otherwise, fall back to the base image. glog.Infof("Could not find image at %s. Defaulting to %s.", img, defaultImg) - return defaultImg, false + return defaultImg } // Just a temporary hack to grab a single range from the config. diff --git a/cloud/google/machinesetup/config_types.go b/cloud/google/machinesetup/config_types.go new file mode 100644 index 000000000000..97dd16318c4f --- /dev/null +++ b/cloud/google/machinesetup/config_types.go @@ -0,0 +1,155 @@ +/* +Copyright 2018 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package machinesetup + +import ( + "fmt" + "io" + "io/ioutil" + "os" + + "reflect" + + "github.com/ghodss/yaml" + clustercommon "sigs.k8s.io/cluster-api/pkg/apis/cluster/common" + clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" +) + +// Config Watch holds the path to the machine setup configs yaml file. +// This works directly with a yaml file is used instead of a ConfigMap object so that we don't take a dependency on the API Server. +type ConfigWatch struct { + path string +} + +// The valid machine setup configs parsed out of the machine setup configs yaml file held in ConfigWatch. +type ValidConfigs struct { + configList *configList +} + +type configList struct { + Items []config `json:"items"` +} + +// A single valid machine setup config that maps a machine's params to the corresponding image and metadata. +type config struct { + // A list of the valid combinations of ConfigParams that will + // map to the given Image and Metadata. + Params []ConfigParams `json:"machineParams"` + + // The fully specified image path. e.g. + // projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts + // projects/ubuntu-os-cloud/global/images/ubuntu-1604-xenial-v20180405 + Image string `json:"image"` + Metadata Metadata `json:"metadata"` +} + +type Metadata struct { + StartupScript string `json:"startupScript"` +} + +type ConfigParams struct { + OS string + Roles []clustercommon.MachineRole + Versions clusterv1.MachineVersionInfo +} + +func NewConfigWatch(path string) (*ConfigWatch, error) { + if _, err := os.Stat(path); err != nil { + return nil, err + } + return &ConfigWatch{path: path}, nil +} + +func (cw *ConfigWatch) ValidConfigs() (*ValidConfigs, error) { + file, err := os.Open(cw.path) + if err != nil { + return nil, err + } + return parseMachineSetupYaml(file) +} + +func parseMachineSetupYaml(reader io.Reader) (*ValidConfigs, error) { + bytes, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + configList := &configList{} + err = yaml.Unmarshal(bytes, configList) + if err != nil { + return nil, err + } + + return &ValidConfigs{configList}, nil +} + +func (vc *ValidConfigs) GetYaml() (string, error) { + bytes, err := yaml.Marshal(vc.configList) + if err != nil { + return "", err + } + return string(bytes), nil +} + +func (vc *ValidConfigs) GetImage(params *ConfigParams) (string, error) { + machineSetupConfig, err := vc.matchMachineSetupConfig(params) + if err != nil { + return "", err + } + return machineSetupConfig.Image, nil +} + +func (vc *ValidConfigs) GetMetadata(params *ConfigParams) (Metadata, error) { + machineSetupConfig, err := vc.matchMachineSetupConfig(params) + if err != nil { + return Metadata{}, err + } + return machineSetupConfig.Metadata, nil +} + +func (vc *ValidConfigs) matchMachineSetupConfig(params *ConfigParams) (*config, error) { + matchingConfigs := make([]config, 0) + for _, conf := range vc.configList.Items { + for _, validParams := range conf.Params { + if params.OS != validParams.OS { + continue + } + validRoles := rolesToMap(validParams.Roles) + paramRoles := rolesToMap(params.Roles) + if !reflect.DeepEqual(paramRoles, validRoles) { + continue + } + if params.Versions != validParams.Versions { + continue + } + matchingConfigs = append(matchingConfigs, conf) + } + } + + if len(matchingConfigs) == 1 { + return &matchingConfigs[0], nil + } else if len(matchingConfigs) == 0 { + return nil, fmt.Errorf("could not find a matching machine setup config for params %+v", params) + } else { + return nil, fmt.Errorf("found multiple matching machine setup configs for params %+v", params) + } +} + +func rolesToMap(roles []clustercommon.MachineRole) map[clustercommon.MachineRole]int { + rolesMap := map[clustercommon.MachineRole]int{} + for _, role := range roles { + rolesMap[role] = rolesMap[role] + 1 + } + return rolesMap +} diff --git a/cloud/google/machinesetup/config_types_test.go b/cloud/google/machinesetup/config_types_test.go new file mode 100644 index 000000000000..5870bb29e568 --- /dev/null +++ b/cloud/google/machinesetup/config_types_test.go @@ -0,0 +1,420 @@ +/* +Copyright 2018 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package machinesetup + +import ( + "io" + "reflect" + "strings" + "testing" + + clustercommon "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" + clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" +) + +func TestParseMachineSetupYaml(t *testing.T) { + testTables := []struct { + reader io.Reader + expectedErr bool + }{ + { + reader: strings.NewReader(`items: +- machineParams: + - os: ubuntu-1710 + roles: + - Master + versions: + kubelet: 1.9.3 + controlPlane: 1.9.3 + containerRuntime: + name: docker + version: 1.12.0 + - os: ubuntu-1710 + roles: + - Master + versions: + kubelet: 1.9.4 + controlPlane: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710 + metadata: + startupScript: | + #!/bin/bash +- machineParams: + - os: ubuntu-1710 + roles: + - Node + versions: + kubelet: 1.9.3 + containerRuntime: + name: docker + version: 1.12.0 + - os: ubuntu-1710 + roles: + - Node + versions: + kubelet: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710 + metadata: + startupScript: | + #!/bin/bash + echo this is the node config.`), + expectedErr: false, + }, + { + reader: strings.NewReader("Not valid yaml"), + expectedErr: true, + }, + } + + for _, table := range testTables { + validConfigs, err := parseMachineSetupYaml(table.reader) + if table.expectedErr { + if err == nil { + t.Errorf("An error was not received as expected.") + } + if validConfigs != nil { + t.Errorf("ValidConfigs should be nil, got %v", validConfigs) + } + } + if !table.expectedErr { + if err != nil { + t.Errorf("Got unexpected error: %s", err) + } + if validConfigs == nil { + t.Errorf("ValidConfigs should have been parsed, but was nil") + } + } + } +} + +func TestGetYaml(t *testing.T) { + testTables := []struct { + validConfigs ValidConfigs + expectedStrings []string + expectedErr bool + }{ + { + validConfigs: ValidConfigs{ + configList: &configList{ + Items: []config{ + { + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Master startup script", + }, + }, + { + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Node startup script", + }, + }, + }, + }, + }, + expectedStrings: []string{"startupScript: Master startup script", "startupScript: Node startup script"}, + expectedErr: false, + }, + } + + for _, table := range testTables { + yaml, err := table.validConfigs.GetYaml() + if err == nil && table.expectedErr { + t.Errorf("An error was not received as expected.") + } + if err != nil && !table.expectedErr { + t.Errorf("Got unexpected error: %s", err) + } + for _, expectedString := range table.expectedStrings { + if !strings.Contains(yaml, expectedString) { + t.Errorf("Yaml did not contain expected string, got:\n%s\nwant:\n%s", yaml, expectedString) + } + } + } +} + +func validConfigs(configs ...config) ValidConfigs { + return ValidConfigs{ + configList: &configList{ + Items: configs, + }, + } +} + +func TestMatchMachineSetupConfig(t *testing.T) { + dockerRuntimeInfo := clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + } + + masterMachineSetupConfig := config{ + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.3", + ControlPlane: "1.9.3", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Master startup script", + }, + } + nodeMachineSetupConfig := config{ + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.3", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Node startup script", + }, + } + multiRoleSetupConfig := config{ + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Multi-role startup script", + }, + } + duplicateMasterMachineSetupConfig := config{ + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.3", + ControlPlane: "1.9.3", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Duplicate master startup script", + }, + } + + testTables := []struct { + validConfigs ValidConfigs + params ConfigParams + expectedMatch *config + expectedErr bool + }{ + { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + expectedMatch: &masterMachineSetupConfig, + expectedErr: false, + }, + { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + expectedMatch: &nodeMachineSetupConfig, + expectedErr: false, + }, + { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + expectedMatch: &multiRoleSetupConfig, + expectedErr: false, + }, + { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + expectedMatch: nil, + expectedErr: true, + }, + { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.13.0", + }, + }, + }, + expectedMatch: nil, + expectedErr: true, + }, + { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.3", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + expectedMatch: nil, + expectedErr: true, + }, + { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + expectedMatch: nil, + expectedErr: true, + }, + { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig, duplicateMasterMachineSetupConfig), + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + expectedMatch: nil, + expectedErr: true, + }, + } + + for _, table := range testTables { + matched, err := table.validConfigs.matchMachineSetupConfig(&table.params) + if !reflect.DeepEqual(matched, table.expectedMatch) { + t.Errorf("Matched machine setup config was incorrect, got: %+v,\n want %+v.", matched, table.expectedMatch) + } + if err == nil && table.expectedErr { + t.Errorf("An error was not received as expected.") + } + if err != nil && !table.expectedErr { + t.Errorf("Got unexpected error: %s", err) + } + } +} diff --git a/cloud/google/metadata.go b/cloud/google/metadata.go new file mode 100644 index 000000000000..2469b0552ed3 --- /dev/null +++ b/cloud/google/metadata.go @@ -0,0 +1,126 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package google + +import ( + "bytes" + "fmt" + "text/template" + + "sigs.k8s.io/cluster-api/cloud/google/machinesetup" + clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" +) + +type metadataParams struct { + Token string + Cluster *clusterv1.Cluster + Machine *clusterv1.Machine + DockerImages []string + Project string + Metadata *machinesetup.Metadata + + // These fields are set when executing the template if they are necessary. + PodCIDR string + ServiceCIDR string + MasterEndpoint string +} + +func nodeMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.Machine, metadata *machinesetup.Metadata) (map[string]string, error) { + params := metadataParams{ + Token: token, + Cluster: cluster, + Machine: machine, + Metadata: metadata, + PodCIDR: getSubnet(cluster.Spec.ClusterNetwork.Pods), + ServiceCIDR: getSubnet(cluster.Spec.ClusterNetwork.Services), + MasterEndpoint: getEndpoint(cluster.Status.APIEndpoints[0]), + } + + nodeMetadata := map[string]string{} + var buf bytes.Buffer + if err := nodeEnvironmentVarsTemplate.Execute(&buf, params); err != nil { + return nil, err + } + buf.WriteString(params.Metadata.StartupScript) + nodeMetadata["startup-script"] = buf.String() + return nodeMetadata, nil +} + +func masterMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.Machine, project string, metadata *machinesetup.Metadata) (map[string]string, error) { + params := metadataParams{ + Token: token, + Cluster: cluster, + Machine: machine, + Project: project, + Metadata: metadata, + PodCIDR: getSubnet(cluster.Spec.ClusterNetwork.Pods), + ServiceCIDR: getSubnet(cluster.Spec.ClusterNetwork.Services), + } + + masterMetadata := map[string]string{} + var buf bytes.Buffer + if err := masterEnvironmentVarsTemplate.Execute(&buf, params); err != nil { + return nil, err + } + buf.WriteString(params.Metadata.StartupScript) + masterMetadata["startup-script"] = buf.String() + return masterMetadata, nil +} + +func getEndpoint(apiEndpoint clusterv1.APIEndpoint) string { + return fmt.Sprintf("%s:%d", apiEndpoint.Host, apiEndpoint.Port) +} + +var ( + masterEnvironmentVarsTemplate *template.Template + nodeEnvironmentVarsTemplate *template.Template +) + +func init() { + masterEnvironmentVarsTemplate = template.Must(template.New("masterEnvironmentVars").Parse(masterEnvironmentVars)) + nodeEnvironmentVarsTemplate = template.Must(template.New("nodeEnvironmentVars").Parse(nodeEnvironmentVars)) +} + +// TODO(kcoronado): replace with actual network and node tag args when they are added into provider config. +const masterEnvironmentVars = ` +#!/bin/bash +KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} +VERSION=v${KUBELET_VERSION} +TOKEN={{ .Token }} +PORT=443 +MACHINE={{ .Machine.ObjectMeta.Name }} +CONTROL_PLANE_VERSION={{ .Machine.Spec.Versions.ControlPlane }} +CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }} +POD_CIDR={{ .PodCIDR }} +SERVICE_CIDR={{ .ServiceCIDR }} +# Environment variables for GCE cloud config +PROJECT={{ .Project }} +NETWORK=default +SUBNETWORK=kubernetes +NODE_TAG=worker +` + +const nodeEnvironmentVars = ` +#!/bin/bash +KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} +TOKEN={{ .Token }} +MASTER={{ .MasterEndpoint }} +MACHINE={{ .Machine.ObjectMeta.Name }} +CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }} +POD_CIDR={{ .PodCIDR }} +SERVICE_CIDER={{ .ServiceCIDR }} +` diff --git a/cloud/google/templates.go b/cloud/google/templates.go deleted file mode 100644 index d77605138d90..000000000000 --- a/cloud/google/templates.go +++ /dev/null @@ -1,321 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package google - -import ( - "bytes" - "fmt" - "text/template" - - clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" -) - -type templateParams struct { - Token string - Cluster *clusterv1.Cluster - Machine *clusterv1.Machine - DockerImages []string - Preloaded bool -} - -func nodeMetadata(params templateParams) (map[string]string, error) { - metadata := map[string]string{} - var buf bytes.Buffer - tName := "fullScript" - if isPreloaded(params) { - tName = "preloadedScript" - } - - if err := nodeStartupScriptTemplate.ExecuteTemplate(&buf, tName, params); err != nil { - return nil, err - } - metadata["startup-script"] = buf.String() - - return metadata, nil -} - -func masterMetadata(params templateParams) (map[string]string, error) { - metadata := map[string]string{} - var buf bytes.Buffer - tName := "fullScript" - if isPreloaded(params) { - tName = "preloadedScript" - } - - if err := masterStartupScriptTemplate.ExecuteTemplate(&buf, tName, params); err != nil { - return nil, err - } - metadata["startup-script"] = buf.String() - return metadata, nil -} - -func isPreloaded(params templateParams) bool { - return params.Preloaded -} - -// PreloadMasterScript returns a script that can be used to preload a master. -func PreloadMasterScript(version string, dockerImages []string) (string, error) { - return preloadScript(masterStartupScriptTemplate, version, dockerImages) -} - -// PreloadNodeScript returns a script that can be used to preload a master. -func PreloadNodeScript(version string, dockerImages []string) (string, error) { - return preloadScript(nodeStartupScriptTemplate, version, dockerImages) -} - -func preloadScript(t *template.Template, version string, dockerImages []string) (string, error) { - var buf bytes.Buffer - params := templateParams{ - Machine: &clusterv1.Machine{}, - DockerImages: dockerImages, - } - params.Machine.Spec.Versions.Kubelet = version - err := t.ExecuteTemplate(&buf, "generatePreloadedImage", params) - return buf.String(), err -} - -var ( - nodeStartupScriptTemplate *template.Template - masterStartupScriptTemplate *template.Template -) - -func init() { - endpoint := func(apiEndpoint *clusterv1.APIEndpoint) string { - return fmt.Sprintf("%s:%d", apiEndpoint.Host, apiEndpoint.Port) - } - // Force a compliation error if getSubnet changes. This is the - // signature the templates expect, so changes need to be - // reflected in templates below. - var _ func(clusterv1.NetworkRanges) string = getSubnet - funcMap := map[string]interface{}{ - "endpoint": endpoint, - "getSubnet": getSubnet, - } - nodeStartupScriptTemplate = template.Must(template.New("nodeStartupScript").Funcs(funcMap).Parse(nodeStartupScript)) - nodeStartupScriptTemplate = template.Must(nodeStartupScriptTemplate.Parse(genericTemplates)) - masterStartupScriptTemplate = template.Must(template.New("masterStartupScript").Funcs(funcMap).Parse(masterStartupScript)) - masterStartupScriptTemplate = template.Must(masterStartupScriptTemplate.Parse(genericTemplates)) -} - -const genericTemplates = ` -{{ define "fullScript" -}} - {{ template "startScript" . }} - {{ template "install" . }} - {{ template "configure" . }} - {{ template "endScript" . }} -{{- end }} - -{{ define "preloadedScript" -}} - {{ template "startScript" . }} - {{ template "configure" . }} - {{ template "endScript" . }} -{{- end }} - -{{ define "generatePreloadedImage" -}} - {{ template "startScript" . }} - {{ template "install" . }} - -systemctl enable docker || true -systemctl start docker || true - - {{ range .DockerImages }} -docker pull {{ . }} - {{ end }} - - {{ template "endScript" . }} -{{- end }} - -{{ define "startScript" -}} -#!/bin/bash - -set -e -set -x - -( -{{- end }} - -{{define "endScript" -}} - -echo done. -) 2>&1 | tee /var/log/startup.log - -{{- end }} -` - -const nodeStartupScript = ` -{{ define "install" -}} -apt-get update -apt-get install -y apt-transport-https prips -apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys F76221572C52609D - -cat < /etc/apt/sources.list.d/k8s.list -deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main -EOF - -apt-get update -apt-get install -y docker-engine=1.12.0-0~xenial - -curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -cat < /etc/apt/sources.list.d/kubernetes.list -deb http://apt.kubernetes.io/ kubernetes-xenial main -EOF -apt-get update - -{{- end }} {{/* end install */}} - -{{ define "configure" -}} -KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} -TOKEN={{ .Token }} -MASTER={{ index .Cluster.Status.APIEndpoints 0 | endpoint }} -MACHINE={{ .Machine.ObjectMeta.Name }} -CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }} -SERVICE_CIDR={{ getSubnet .Cluster.Spec.ClusterNetwork.Services }} - -# Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix -# search based on our SemVer to find the right (newest) package version. -function getversion() { - name=$1 - prefix=$2 - version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) - if [[ -z "$version" ]]; then - echo Can\'t find package $name with prefix $prefix - exit 1 - fi - echo $version -} - -KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) -KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) -KUBECTL=$(getversion kubectl ${KUBELET_VERSION}-) -# Explicit cni version is a temporary workaround till the right version can be automatically detected correctly -apt-get install -y kubelet=${KUBELET} kubeadm=${KUBEADM} kubectl=${KUBECTL} kubernetes-cni=0.5.1-00 - -systemctl enable docker || true -systemctl start docker || true - -sysctl net.bridge.bridge-nf-call-iptables=1 - -# kubeadm uses 10th IP as DNS server -CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) - -sed -i "s/KUBELET_DNS_ARGS=[^\"]*/KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -systemctl daemon-reload -systemctl restart kubelet.service - -kubeadm join --token "${TOKEN}" "${MASTER}" --skip-preflight-checks - -for tries in $(seq 1 60); do - kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break - sleep 1 -done -{{- end }} {{/* end configure */}} -` - -// TODO: actually init the cluster, templatize token, etc. -const masterStartupScript = ` -{{ define "install" -}} - -KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} - -curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - -touch /etc/apt/sources.list.d/kubernetes.list -sh -c 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list' - -apt-get update -y - -apt-get install -y \ - socat \ - ebtables \ - docker.io \ - apt-transport-https \ - cloud-utils \ - prips - -export VERSION=v${KUBELET_VERSION} -export ARCH=amd64 -curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl -chmod a+rx /usr/bin/kubeadm.dl -{{- end }} {{/* end install */}} - - -{{ define "configure" -}} -KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} -TOKEN={{ .Token }} -PORT=443 -MACHINE={{ .Machine.ObjectMeta.Name }} -CONTROL_PLANE_VERSION={{ .Machine.Spec.Versions.ControlPlane }} -CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }} -POD_CIDR={{ getSubnet .Cluster.Spec.ClusterNetwork.Pods }} -SERVICE_CIDR={{ getSubnet .Cluster.Spec.ClusterNetwork.Services }} - -# kubeadm uses 10th IP as DNS server -CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) - -# Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix -# search based on our SemVer to find the right (newest) package version. -function getversion() { - name=$1 - prefix=$2 - version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) - if [[ -z "$version" ]]; then - echo Can\'t find package $name with prefix $prefix - exit 1 - fi - echo $version -} - -KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) -KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) - -# Explicit cni version is a temporary workaround till the right version can be automatically detected correctly -apt-get install -y \ - kubelet=${KUBELET} \ - kubeadm=${KUBEADM} \ - kubernetes-cni=0.5.1-00 - -mv /usr/bin/kubeadm.dl /usr/bin/kubeadm -chmod a+rx /usr/bin/kubeadm - -systemctl enable docker -systemctl start docker -sed -i "s/KUBELET_DNS_ARGS=[^\"]*/KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -systemctl daemon-reload -systemctl restart kubelet.service -` + - "PRIVATEIP=`curl --retry 5 -sfH \"Metadata-Flavor: Google\" \"http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip\"`" + ` -echo $PRIVATEIP > /tmp/.ip -` + - "PUBLICIP=`curl --retry 5 -sfH \"Metadata-Flavor: Google\" \"http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip\"`" + ` - - -kubeadm init --apiserver-bind-port ${PORT} --token ${TOKEN} --kubernetes-version v${CONTROL_PLANE_VERSION} \ - --apiserver-advertise-address ${PUBLICIP} --apiserver-cert-extra-sans ${PUBLICIP} ${PRIVATEIP} \ - --service-cidr ${SERVICE_CIDR} - -# install weavenet -sysctl net.bridge.bridge-nf-call-iptables=1 -export kubever=$(kubectl version --kubeconfig /etc/kubernetes/admin.conf | base64 | tr -d '\n') -kubectl apply --kubeconfig /etc/kubernetes/admin.conf -f "https://cloud.weave.works/k8s/net?k8s-version=$kubever" - -for tries in $(seq 1 60); do - kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break - sleep 1 -done - -{{- end }} {{/* end configure */}} -` diff --git a/gcp-deployer/cmd/add.go b/gcp-deployer/cmd/add.go index dbc366a90aea..4051735a64a2 100644 --- a/gcp-deployer/cmd/add.go +++ b/gcp-deployer/cmd/add.go @@ -52,7 +52,7 @@ func RunAdd(ao *AddOptions) error { return err } - d := deploy.NewDeployer(provider, kubeConfig) + d := deploy.NewDeployer(provider, kubeConfig, "") return d.AddNodes(machines) } diff --git a/gcp-deployer/cmd/create.go b/gcp-deployer/cmd/create.go index 793c7197ef04..2a2be289c4f3 100644 --- a/gcp-deployer/cmd/create.go +++ b/gcp-deployer/cmd/create.go @@ -25,8 +25,9 @@ import ( ) type CreateOptions struct { - Cluster string - Machine string + Cluster string + Machine string + MachineSetup string } var co = &CreateOptions{} @@ -46,6 +47,11 @@ var createCmd = &cobra.Command{ cmd.Help() os.Exit(1) } + if co.MachineSetup == "" { + glog.Error("Please provide yaml file for machine setup configs.") + cmd.Help() + os.Exit(1) + } if err := RunCreate(co); err != nil { glog.Exit(err) } @@ -63,13 +69,14 @@ func RunCreate(co *CreateOptions) error { return err } - d := deploy.NewDeployer(provider, kubeConfig) + d := deploy.NewDeployer(provider, kubeConfig, co.MachineSetup) return d.CreateCluster(cluster, machines) } func init() { createCmd.Flags().StringVarP(&co.Cluster, "cluster", "c", "", "cluster yaml file") createCmd.Flags().StringVarP(&co.Machine, "machines", "m", "", "machine yaml file") + createCmd.Flags().StringVarP(&co.MachineSetup, "machinesetup", "s", "machine_setup_configs.yaml", "machine setup configs yaml file") RootCmd.AddCommand(createCmd) } diff --git a/gcp-deployer/cmd/delete.go b/gcp-deployer/cmd/delete.go index 0c9ed6c94055..e8fa29d81e29 100644 --- a/gcp-deployer/cmd/delete.go +++ b/gcp-deployer/cmd/delete.go @@ -34,7 +34,7 @@ var deleteCmd = &cobra.Command{ } func RunDelete() error { - d := deploy.NewDeployer(provider, kubeConfig) + d := deploy.NewDeployer(provider, kubeConfig, "") return d.DeleteCluster() } diff --git a/gcp-deployer/deploy/deploy.go b/gcp-deployer/deploy/deploy.go index 407784214d05..ae87f20ff2d8 100644 --- a/gcp-deployer/deploy/deploy.go +++ b/gcp-deployer/deploy/deploy.go @@ -22,6 +22,7 @@ import ( "github.com/golang/glog" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/cluster-api/cloud/google" clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset" @@ -31,37 +32,38 @@ import ( ) type deployer struct { - token string - configPath string - machineDeployer machineDeployer - client v1alpha1.ClusterV1alpha1Interface - clientSet clientset.Interface + token string + configPath string + machineDeployer machineDeployer + client v1alpha1.ClusterV1alpha1Interface + clientSet clientset.Interface + kubernetesClientSet kubernetes.Clientset } // NewDeployer returns a cloud provider specific deployer and // sets kubeconfig path for the cluster to be deployed -func NewDeployer(provider string, configPath string) *deployer { +func NewDeployer(provider string, kubeConfigPath string, machineSetupConfigPath string) *deployer { token := util.RandomToken() - if configPath == "" { - configPath = os.Getenv("KUBECONFIG") - if configPath == "" { - configPath = apiutil.GetDefaultKubeConfigPath() + if kubeConfigPath == "" { + kubeConfigPath = os.Getenv("KUBECONFIG") + if kubeConfigPath == "" { + kubeConfigPath = apiutil.GetDefaultKubeConfigPath() } } else { // This is needed for kubectl commands run later to create secret in function // CreateMachineControllerServiceAccount - if err := os.Setenv("KUBECONFIG", configPath); err != nil { + if err := os.Setenv("KUBECONFIG", kubeConfigPath); err != nil { glog.Exit(fmt.Sprintf("Failed to set Kubeconfig path err %v\n", err)) } } - ma, err := google.NewMachineActuator(token, nil) + ma, err := google.NewMachineActuator(token, nil, machineSetupConfigPath) if err != nil { glog.Exit(err) } return &deployer{ token: token, machineDeployer: ma, - configPath: configPath, + configPath: kubeConfigPath, } } diff --git a/gcp-deployer/deploy/deploy_helper.go b/gcp-deployer/deploy/deploy_helper.go index 75921ab58c4a..337a1e366380 100644 --- a/gcp-deployer/deploy/deploy_helper.go +++ b/gcp-deployer/deploy/deploy_helper.go @@ -88,7 +88,7 @@ func (d *deployer) createCluster(c *clusterv1.Cluster, machines []*clusterv1.Mac } glog.Info("Deploying the addon apiserver and controller manager...") - if err := d.machineDeployer.CreateMachineController(c, machines); err != nil { + if err := d.machineDeployer.CreateMachineController(c, machines, d.kubernetesClientSet); err != nil { return fmt.Errorf("can't create machine controller: %v", err) } @@ -232,8 +232,13 @@ func (d *deployer) initApiClient() error { if err != nil { return err } + kubernetesClientSet, err := util.NewKubernetesClient(d.configPath) + if err != nil { + return err + } d.clientSet = c d.client = c.ClusterV1alpha1() + d.kubernetesClientSet = *kubernetesClientSet return nil } @@ -278,14 +283,9 @@ func (d *deployer) waitForApiserver(master string) error { // Make sure the default service account in kube-system namespace exists. func (d *deployer) waitForServiceAccount() error { - client, err := util.NewKubernetesClient(d.configPath) - if err != nil { - return err - } - waitErr := util.Retry(func() (bool, error) { glog.Info("Waiting for the service account to exist...") - _, err = client.CoreV1().ServiceAccounts(ServiceAccountNs).Get(ServiceAccountName, metav1.GetOptions{}) + _, err := d.kubernetesClientSet.CoreV1().ServiceAccounts(ServiceAccountNs).Get(ServiceAccountName, metav1.GetOptions{}) return (err == nil), nil }, 5) diff --git a/gcp-deployer/deploy/machinedeployer.go b/gcp-deployer/deploy/machinedeployer.go index 4e0561882f96..449494a3ea8d 100644 --- a/gcp-deployer/deploy/machinedeployer.go +++ b/gcp-deployer/deploy/machinedeployer.go @@ -1,6 +1,7 @@ package deploy import ( + "k8s.io/client-go/kubernetes" clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" "sigs.k8s.io/cluster-api/pkg/controller/machine" ) @@ -16,6 +17,6 @@ type machineDeployer interface { // are provided in case the function wants to refer to them (and their // ProviderConfigs) to know how to configure the machine controller. // Not idempotent. - CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error + CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine, clientSet kubernetes.Clientset) error PostDelete(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error } diff --git a/gcp-deployer/machine_setup_configs.yaml b/gcp-deployer/machine_setup_configs.yaml new file mode 100644 index 000000000000..cbdcd98540b1 --- /dev/null +++ b/gcp-deployer/machine_setup_configs.yaml @@ -0,0 +1,320 @@ +items: +- machineParams: + - os: ubuntu-1710-weave + roles: + - Node + versions: + kubelet: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710 + metadata: + startupScript: | + set -e + set -x + ( + apt-get update + apt-get install -y apt-transport-https prips + apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys F76221572C52609D + cat < /etc/apt/sources.list.d/k8s.list + deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main + EOF + apt-get update + apt-get install -y docker-engine=1.12.0-0~xenial + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + cat < /etc/apt/sources.list.d/kubernetes.list + deb http://apt.kubernetes.io/ kubernetes-xenial main + EOF + apt-get update + # Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix + # search based on our SemVer to find the right (newest) package version. + function getversion() { + name=$1 + prefix=$2 + version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) + if [[ -z "$version" ]]; then + echo Can\'t find package $name with prefix $prefix + exit 1 + fi + echo $version + } + KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) + KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) + KUBECTL=$(getversion kubectl ${KUBELET_VERSION}-) + apt-get install -y kubelet=${KUBELET} kubeadm=${KUBEADM} kubectl=${KUBECTL} + systemctl enable docker || true + systemctl start docker || true + # kubeadm uses 10th IP as DNS server + CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) + # Override Kubelet DNS args. + cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <&1 | tee /var/log/startup.log +- machineParams: + - os: ubuntu-1710-weave + roles: + - Master + versions: + kubelet: 1.9.4 + controlPlane: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710 + metadata: + startupScript: | + set -e + set -x + ( + ARCH=amd64 + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - + touch /etc/apt/sources.list.d/kubernetes.list + sh -c 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list' + apt-get update -y + apt-get install -y \ + socat \ + ebtables \ + docker.io \ + apt-transport-https \ + cloud-utils \ + prips + curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl + chmod a+rx /usr/bin/kubeadm.dl + # kubeadm uses 10th IP as DNS server + CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) + # Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix + # search based on our SemVer to find the right (newest) package version. + function getversion() { + name=$1 + prefix=$2 + version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) + if [[ -z "$version" ]]; then + echo Can\'t find package $name with prefix $prefix + exit 1 + fi + echo $version + } + KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) + KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) + apt-get install -y \ + kubelet=${KUBELET} \ + kubeadm=${KUBEADM} + mv /usr/bin/kubeadm.dl /usr/bin/kubeadm + chmod a+rx /usr/bin/kubeadm + systemctl enable docker + systemctl start docker + # Override Kubelet DNS args. + cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf < /tmp/.ip + PUBLICIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"` + # Set up kubeadm config file to pass parameters to kubeadm init. + cat > /etc/kubernetes/kubeadm_config.yaml <&1 | tee /var/log/startup.log +# These configs currently don't work - they need service accounts. +- machineParams: + - os: ubuntu-1604-lts + roles: + - Master + versions: + kubelet: 1.9.4 + controlPlane: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts + metadata: + startupScript: | + set -e + set -x + ( + ARCH=amd64 + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - + touch /etc/apt/sources.list.d/kubernetes.list + sh -c 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list' + apt-get update -y + apt-get install -y \ + socat \ + ebtables \ + docker.io \ + apt-transport-https \ + cloud-utils \ + prips + curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl + chmod a+rx /usr/bin/kubeadm.dl + # kubeadm uses 10th IP as DNS server + CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) + # Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix + # search based on our SemVer to find the right (newest) package version. + function getversion() { + name=$1 + prefix=$2 + version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) + if [[ -z "$version" ]]; then + echo Can\'t find package $name with prefix $prefix + exit 1 + fi + echo $version + } + KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) + KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) + apt-get install -y \ + kubelet=${KUBELET} \ + kubeadm=${KUBEADM} + mv /usr/bin/kubeadm.dl /usr/bin/kubeadm + chmod a+rx /usr/bin/kubeadm + systemctl enable docker + systemctl start docker + # Override network args to use kubenet instead of cni, and override Kubelet DNS args. + cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf < /tmp/.ip + PUBLICIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"` + # Set up the GCE cloud config, which gets picked up by kubeadm init since cloudProvider is set to GCE. + cat > /etc/kubernetes/cloud-config < /etc/kubernetes/kubeadm_config.yaml <&1 | tee /var/log/startup.log +- machineParams: + - os: ubuntu-1604-lts + roles: + - Node + versions: + kubelet: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts + metadata: + startupScript: | + set -e + set -x + ( + apt-get update + apt-get install -y apt-transport-https prips + apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys F76221572C52609D + cat < /etc/apt/sources.list.d/k8s.list + deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main + EOF + apt-get update + apt-get install -y docker-engine=1.12.0-0~xenial + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + cat < /etc/apt/sources.list.d/kubernetes.list + deb http://apt.kubernetes.io/ kubernetes-xenial main + EOF + apt-get update + # Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix + # search based on our SemVer to find the right (newest) package version. + function getversion() { + name=$1 + prefix=$2 + version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) + if [[ -z "$version" ]]; then + echo Can\'t find package $name with prefix $prefix + exit 1 + fi + echo $version + } + KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) + KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) + KUBECTL=$(getversion kubectl ${KUBELET_VERSION}-) + apt-get install -y kubelet=${KUBELET} kubeadm=${KUBEADM} kubectl=${KUBECTL} + systemctl enable docker || true + systemctl start docker || true + # kubeadm uses 10th IP as DNS server + CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) + # Override network args to use kubenet instead of cni, and override Kubelet DNS args. + cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <&1 | tee /var/log/startup.log diff --git a/gcp-deployer/machines.yaml.template b/gcp-deployer/machines.yaml.template index c0850a3e5943..b95699df7b01 100644 --- a/gcp-deployer/machines.yaml.template +++ b/gcp-deployer/machines.yaml.template @@ -13,10 +13,10 @@ items: project: "$GCLOUD_PROJECT" zone: "us-central1-f" machineType: "n1-standard-2" - image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts" + os: "ubuntu-1604-lts" versions: - kubelet: 1.8.3 - controlPlane: 1.8.3 + kubelet: 1.9.4 + controlPlane: 1.9.4 containerRuntime: name: docker version: 1.12.0 @@ -36,9 +36,9 @@ items: project: "$GCLOUD_PROJECT" zone: "us-central1-f" machineType: "n1-standard-1" - image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts" + os: "ubuntu-1604-lts" versions: - kubelet: 1.8.3 + kubelet: 1.9.4 containerRuntime: name: docker version: 1.12.0 From 46490acf312b2043630521d2779b30e6a994cc40 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 19 Apr 2018 13:53:58 -0700 Subject: [PATCH 2/3] Remove UID logic --- cloud/google/machineactuator.go | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go index 55875d2bf308..7166e17110b8 100644 --- a/cloud/google/machineactuator.go +++ b/cloud/google/machineactuator.go @@ -54,7 +54,6 @@ const ( ZoneAnnotationKey = "gcp-zone" NameAnnotationKey = "gcp-name" - UIDLabelKey = "machine-crd-uid" BootstrapLabelKey = "boostrap" // This file is a yaml that will be used to create the machine-setup configmap on the machine controller. @@ -253,9 +252,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach diskSize := int64(30) if instance == nil { - labels := map[string]string{ - UIDLabelKey: fmt.Sprintf("%v", machine.ObjectMeta.UID), - } + labels := map[string]string{} if gce.machineClient == nil { labels[BootstrapLabelKey] = "true" } @@ -511,8 +508,7 @@ func (gce *GCEClient) requiresUpdate(a *clusterv1.Machine, b *clusterv1.Machine) !reflect.DeepEqual(a.Spec.ProviderConfig, b.Spec.ProviderConfig) || !reflect.DeepEqual(a.Spec.Roles, b.Spec.Roles) || !reflect.DeepEqual(a.Spec.Versions, b.Spec.Versions) || - a.ObjectMeta.Name != b.ObjectMeta.Name || - a.ObjectMeta.UID != b.ObjectMeta.UID + a.ObjectMeta.Name != b.ObjectMeta.Name } // Gets the instance represented by the given machine @@ -545,18 +541,6 @@ func (gce *GCEClient) instanceIfExists(machine *clusterv1.Machine) (*compute.Ins return nil, err } - uid := instance.Labels[UIDLabelKey] - if uid == "" { - if instance.Labels[BootstrapLabelKey] != "" { - glog.Infof("Skipping uid check since instance %v %v %v is missing uid label due to being provisioned as part of bootstrap.", config.Project, config.Zone, identifyingMachine.ObjectMeta.Name) - } else { - return nil, fmt.Errorf("Instance %v %v %v is missing uid label.", config.Project, config.Zone, identifyingMachine.ObjectMeta.Name) - } - } else if uid != fmt.Sprintf("%v", machine.ObjectMeta.UID) { - glog.Infof("Instance %v exists but it has a different UID. Object UID: %v . Instance UID: %v", machine.ObjectMeta.Name, machine.ObjectMeta.UID, uid) - return nil, nil - } - return instance, nil } From 231f71b536b4c0a4bfd64123042e608663a85c5b Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Fri, 20 Apr 2018 14:44:19 -0700 Subject: [PATCH 3/3] Fix import path in tests --- cloud/google/machinesetup/config_types_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud/google/machinesetup/config_types_test.go b/cloud/google/machinesetup/config_types_test.go index 5870bb29e568..259beb3dad40 100644 --- a/cloud/google/machinesetup/config_types_test.go +++ b/cloud/google/machinesetup/config_types_test.go @@ -19,8 +19,8 @@ import ( "strings" "testing" - clustercommon "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" - clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" + clustercommon "sigs.k8s.io/cluster-api/pkg/apis/cluster/common" + clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" ) func TestParseMachineSetupYaml(t *testing.T) {