From 5cd6e74575678f151096584de670d7bc4a0b13ac Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Mon, 2 Apr 2018 15:10:44 -0700 Subject: [PATCH 01/21] Rename GCEProviderConfig.Image field to OS --- cluster-api/cloud/google/gceproviderconfig/types.go | 2 +- cluster-api/cloud/google/gceproviderconfig/v1alpha1/types.go | 2 +- cluster-api/gcp-deployer/machines.yaml.template | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-api/cloud/google/gceproviderconfig/types.go b/cluster-api/cloud/google/gceproviderconfig/types.go index f6951f6f4..376fc4da0 100644 --- a/cluster-api/cloud/google/gceproviderconfig/types.go +++ b/cluster-api/cloud/google/gceproviderconfig/types.go @@ -27,5 +27,5 @@ type GCEProviderConfig struct { Project string `json:"project"` Zone string `json:"zone"` MachineType string `json:"machineType"` - Image string `json:"image"` + OS string `json:"os"` } diff --git a/cluster-api/cloud/google/gceproviderconfig/v1alpha1/types.go b/cluster-api/cloud/google/gceproviderconfig/v1alpha1/types.go index 9ad6940f9..f0175404a 100644 --- a/cluster-api/cloud/google/gceproviderconfig/v1alpha1/types.go +++ b/cluster-api/cloud/google/gceproviderconfig/v1alpha1/types.go @@ -27,5 +27,5 @@ type GCEProviderConfig struct { Project string `json:"project"` Zone string `json:"zone"` MachineType string `json:"machineType"` - Image string `json:"image"` + OS string `json:"os"` } diff --git a/cluster-api/gcp-deployer/machines.yaml.template b/cluster-api/gcp-deployer/machines.yaml.template index c0850a3e5..40ffe914b 100644 --- a/cluster-api/gcp-deployer/machines.yaml.template +++ b/cluster-api/gcp-deployer/machines.yaml.template @@ -13,7 +13,7 @@ items: project: "$GCLOUD_PROJECT" zone: "us-central1-f" machineType: "n1-standard-2" - image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts" + os: "ubuntu-1604-lts" versions: kubelet: 1.8.3 controlPlane: 1.8.3 @@ -36,7 +36,7 @@ items: project: "$GCLOUD_PROJECT" zone: "us-central1-f" machineType: "n1-standard-1" - image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts" + os: "ubuntu-1604-lts" versions: kubelet: 1.8.3 containerRuntime: From 6bc01dda3a5ddf0354795c519e0ca4579e09785a Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 22 Mar 2018 10:53:38 -0700 Subject: [PATCH 02/21] Replace hardcoded startup script with machine setup configs. (work in progress) --- .../google/cmd/gce-machine-controller/main.go | 5 +- .../cloud/google/cmd/generate-image/main.go | 1 + cluster-api/cloud/google/machineactuator.go | 84 ++-- .../cloud/google/machinesetup/config_types.go | 118 ++++++ .../google/machinesetup/config_types_test.go | 243 +++++++++++ cluster-api/cloud/google/metadata.go | 117 ++++++ cluster-api/cloud/google/templates.go | 321 --------------- .../gcp-deployer/machine_setup_configs.yaml | 377 ++++++++++++++++++ 8 files changed, 913 insertions(+), 353 deletions(-) create mode 100644 cluster-api/cloud/google/machinesetup/config_types.go create mode 100644 cluster-api/cloud/google/machinesetup/config_types_test.go create mode 100644 cluster-api/cloud/google/metadata.go delete mode 100644 cluster-api/cloud/google/templates.go create mode 100644 cluster-api/gcp-deployer/machine_setup_configs.yaml diff --git a/cluster-api/cloud/google/cmd/gce-machine-controller/main.go b/cluster-api/cloud/google/cmd/gce-machine-controller/main.go index 6f0af853f..ecf10b4c9 100644 --- a/cluster-api/cloud/google/cmd/gce-machine-controller/main.go +++ b/cluster-api/cloud/google/cmd/gce-machine-controller/main.go @@ -31,7 +31,8 @@ import ( ) var ( - kubeadmToken = pflag.String("token", "", "Kubeadm token to use to join new machines") + kubeadmToken = pflag.String("token", "", "Kubeadm token to use to join new machines") + machineSetupConfigsPath = pflag.String("config", "", "path to machine setup configs file") ) func init() { @@ -54,7 +55,7 @@ func main() { glog.Fatalf("Could not create client for talking to the apiserver: %v", err) } - actuator, err := google.NewMachineActuator(*kubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault)) + actuator, err := google.NewMachineActuator(*kubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault), *machineSetupConfigsPath) if err != nil { glog.Fatalf("Could not create Google machine actuator: %v", err) } diff --git a/cluster-api/cloud/google/cmd/generate-image/main.go b/cluster-api/cloud/google/cmd/generate-image/main.go index 55b172ea9..18c69a5b9 100644 --- a/cluster-api/cloud/google/cmd/generate-image/main.go +++ b/cluster-api/cloud/google/cmd/generate-image/main.go @@ -42,6 +42,7 @@ var generateCmd = &cobra.Command{ }, } +// TODO(kcoronado): take a script file as a parameter instead of generating the preloaded script from the template. func init() { generateCmd.Flags().StringVar(&opts.version, "version", "1.7.3", "The version of kubernetes to install") generateCmd.Flags().StringVar(&opts.role, "role", "master", "The role of the machine (master or node)") diff --git a/cluster-api/cloud/google/machineactuator.go b/cluster-api/cloud/google/machineactuator.go index bd4754002..b18bd02eb 100644 --- a/cluster-api/cloud/google/machineactuator.go +++ b/cluster-api/cloud/google/machineactuator.go @@ -39,6 +39,7 @@ import ( gceconfig "k8s.io/kube-deploy/cluster-api/cloud/google/gceproviderconfig" gceconfigv1 "k8s.io/kube-deploy/cluster-api/cloud/google/gceproviderconfig/v1alpha1" + "k8s.io/kube-deploy/cluster-api/cloud/google/machinesetup" apierrors "k8s.io/kube-deploy/cluster-api/errors" clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" client "k8s.io/kube-deploy/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1" @@ -66,6 +67,7 @@ type GCEClient struct { kubeadmToken string sshCreds SshCreds machineClient client.MachineInterface + configWatch *machinesetup.ConfigWatch } const ( @@ -73,7 +75,7 @@ const ( gceWaitSleep = time.Second * 5 ) -func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface) (*GCEClient, error) { +func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface, configListPath string) (*GCEClient, error) { // The default GCP client expects the environment variable // GOOGLE_APPLICATION_CREDENTIALS to point to a file with service credentials. client, err := google.DefaultClient(context.TODO(), compute.ComputeScope) @@ -104,6 +106,14 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa } } + var configWatch *machinesetup.ConfigWatch + if configListPath != "" { + configWatch, err = machinesetup.NewConfigWatch(configListPath) + if err != nil { + glog.Errorf("Error creating config watch: %v", err) + } + } + return &GCEClient{ service: service, scheme: scheme, @@ -114,6 +124,7 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa user: user, }, machineClient: machineClient, + configWatch: configWatch, }, nil } @@ -153,8 +164,28 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach return errors.New("invalid master configuration: missing Machine.Spec.Versions.Kubelet") } - image, preloaded := gce.getImage(machine, config) + if gce.configWatch == nil { + return errors.New("invalid machine setup configuration: missing GCEClient.ConfigWatch") + } + machineSetupConfigs, err := gce.configWatch.ValidConfigs() + if err != nil { + return err + } + configParams := &machinesetup.ConfigParams{ + OS: config.OS, + Roles: machine.Spec.Roles, + Versions: machine.Spec.Versions, + } + image, err := machineSetupConfigs.GetImage(configParams) + if err != nil { + return err + } + imagePath := gce.getImagePath(image, config.Project) + machineSetupMetadata, err := machineSetupConfigs.GetMetadata(configParams) + if err != nil { + return err + } if util.IsMaster(machine) { if machine.Spec.Versions.ControlPlane == "" { return gce.handleMachineError(machine, apierrors.InvalidMachineConfiguration( @@ -162,11 +193,12 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach } var err error metadata, err = masterMetadata( - templateParams{ - Token: gce.kubeadmToken, - Cluster: cluster, - Machine: machine, - Preloaded: preloaded, + metadataParams{ + Token: gce.kubeadmToken, + Cluster: cluster, + Machine: machine, + Project: config.Project, + Metadata: &machineSetupMetadata, }, ) if err != nil { @@ -178,11 +210,11 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach } var err error metadata, err = nodeMetadata( - templateParams{ - Token: gce.kubeadmToken, - Cluster: cluster, - Machine: machine, - Preloaded: preloaded, + metadataParams{ + Token: gce.kubeadmToken, + Cluster: cluster, + Machine: machine, + Metadata: &machineSetupMetadata, }, ) if err != nil { @@ -207,13 +239,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach name := machine.ObjectMeta.Name project := config.Project zone := config.Zone - diskSize := int64(10) - - // Our preloaded image already has a lot stored on it, so increase the - // disk size to have more free working space. - if preloaded { - diskSize = 30 - } + diskSize := int64(30) if instance == nil { labels := map[string]string{ @@ -242,7 +268,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach AutoDelete: true, Boot: true, InitializeParams: &compute.AttachedDiskInitializeParams{ - SourceImage: image, + SourceImage: imagePath, DiskSizeGb: diskSize, }, }, @@ -653,24 +679,22 @@ func (gce *GCEClient) handleMachineError(machine *clusterv1.Machine, err *apierr return err } -func (gce *GCEClient) getImage(machine *clusterv1.Machine, config *gceconfig.GCEProviderConfig) (image string, isPreloaded bool) { +func (gce *GCEClient) getImagePath(img string, project string) (imagePath string) { defaultImg := "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710" - project := config.Project - img := config.Image // A full image path must match the regex format. If it doesn't, we'll assume it's just the image name and try to get it. // If that doesn't work, we will fall back to a default base image. matches := regexp.MustCompile("projects/(.+)/global/images/(family/)*(.+)").FindStringSubmatch(img) if matches == nil { - // Only the image name was specified in config, so check if it is preloaded in the project specified in config. + // Only the image name was specified in config, so check if it exists in the project specified in config. fullPath := fmt.Sprintf("projects/%s/global/images/%s", project, img) if _, err := gce.service.Images.Get(project, img).Do(); err == nil { - return fullPath, false + return fullPath } - // Otherwise, fall back to the non-preloaded base image. + // Otherwise, fall back to the base image. glog.Infof("Could not find image at %s. Defaulting to %s.", fullPath, defaultImg) - return defaultImg, false + return defaultImg } // Check to see if the image exists in the given path. The presence of "family" in the path dictates which API call we need to make. @@ -683,12 +707,12 @@ func (gce *GCEClient) getImage(machine *clusterv1.Machine, config *gceconfig.GCE } if err == nil { - return img, false + return img } - // Otherwise, fall back to the non-preloaded base image. + // Otherwise, fall back to the base image. glog.Infof("Could not find image at %s. Defaulting to %s.", img, defaultImg) - return defaultImg, false + return defaultImg } // Just a temporary hack to grab a single range from the config. diff --git a/cluster-api/cloud/google/machinesetup/config_types.go b/cluster-api/cloud/google/machinesetup/config_types.go new file mode 100644 index 000000000..422c5362e --- /dev/null +++ b/cluster-api/cloud/google/machinesetup/config_types.go @@ -0,0 +1,118 @@ +package machinesetup + +import ( + "fmt" + "github.com/ghodss/yaml" + "io" + "io/ioutil" + clustercommon "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" + clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" + "k8s.io/kube-deploy/cluster-api/util" + "os" +) + +type ConfigWatch struct { + Path string +} + +type ValidConfigs struct { + configList *configList +} + +type configList struct { + Items []config `json:"items"` +} + +type config struct { + // A list of the valid combinations of ConfigParams that will + // map to the given Image and Metadata. + Params []ConfigParams `json:"machineParams"` + + // This can either be a full projects path to an image/family, + // or just the image name which is in the project. + // If it's an image in the project, this field may be + // identical to the OS field (see GCEClient.getImagePath()). + Image string `json:"image"` + Metadata Metadata `json:"metadata"` +} + +type Metadata struct { + StartupScript string `json:"startupScript"` +} + +type ConfigParams struct { + OS string + Roles []clustercommon.MachineRole + Versions clusterv1.MachineVersionInfo +} + +func NewConfigWatch(path string) (*ConfigWatch, error) { + if _, err := os.Stat(path); err != nil { + return nil, err + } + return &ConfigWatch{Path: path}, nil +} + +func (cw *ConfigWatch) ValidConfigs() (*ValidConfigs, error) { + file, err := os.Open(cw.Path) + if err != nil { + return nil, err + } + return parseMachineSetupYaml(file) +} + +func parseMachineSetupYaml(reader io.Reader) (*ValidConfigs, error) { + bytes, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + configList := &configList{} + err = yaml.Unmarshal(bytes, configList) + if err != nil { + return nil, err + } + + return &ValidConfigs{configList}, nil +} + +func (vc *ValidConfigs) GetImage(params *ConfigParams) (string, error) { + machineSetupConfig, err := vc.matchMachineSetupConfig(params) + if err != nil { + return "", err + } + return machineSetupConfig.Image, nil +} + +func (vc *ValidConfigs) GetMetadata(params *ConfigParams) (Metadata, error) { + machineSetupConfig, err := vc.matchMachineSetupConfig(params) + if err != nil { + return Metadata{}, err + } + return machineSetupConfig.Metadata, nil +} + +func (vc *ValidConfigs) matchMachineSetupConfig(params *ConfigParams) (*config, error) { + for _, conf := range vc.configList.Items { + for _, validParams := range conf.Params { + if params.OS != validParams.OS { + continue + } + foundRoles := true + for _, role := range params.Roles { + if !util.RoleContains(role, validParams.Roles) { + foundRoles = false + break + } + } + if !foundRoles { + continue + } + if params.Versions != validParams.Versions { + continue + } + return &conf, nil + } + } + return nil, fmt.Errorf("could not find a matching machine setup config for params %+v", params) +} diff --git a/cluster-api/cloud/google/machinesetup/config_types_test.go b/cluster-api/cloud/google/machinesetup/config_types_test.go new file mode 100644 index 000000000..46fcacda7 --- /dev/null +++ b/cluster-api/cloud/google/machinesetup/config_types_test.go @@ -0,0 +1,243 @@ +package machinesetup + +import ( + "io" + "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" + "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" + "reflect" + "strings" + "testing" +) + +func TestParseMachineSetupYaml(t *testing.T) { + testTables := []struct { + reader io.Reader + expectedErr bool + }{ + { + reader: strings.NewReader(`items: +- machineParams: + - os: ubuntu-1710 + roles: + - Master + versions: + kubelet: 1.9.3 + controlPlane: 1.9.3 + containerRuntime: + name: docker + version: 1.12.0 + - os: ubuntu-1710 + roles: + - Master + versions: + kubelet: 1.9.4 + controlPlane: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710 + metadata: + startupScript: | + #!/bin/bash +- machineParams: + - os: ubuntu-1710 + roles: + - Node + versions: + kubelet: 1.9.3 + containerRuntime: + name: docker + version: 1.12.0 + - os: ubuntu-1710 + roles: + - Node + versions: + kubelet: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710 + metadata: + startupScript: | + #!/bin/bash + echo this is the node config.`), + expectedErr: false, + }, + { + reader: strings.NewReader("Not valid yaml"), + expectedErr: true, + }, + } + + for _, table := range testTables { + validConfigs, err := parseMachineSetupYaml(table.reader) + if table.expectedErr { + if err == nil { + t.Errorf("An error was not received as expected.") + } + if validConfigs != nil { + t.Errorf("ValidConfigs should be nil, got %v", validConfigs) + } + } + if !table.expectedErr { + if err != nil { + t.Errorf("Got unexpected error: %s", err) + } + if validConfigs == nil { + t.Errorf("ValidConfigs should have been parsed, but was nil") + } + } + } +} + +func TestMatchMachineSetupConfig(t *testing.T) { + masterMachineSetupConfig := config{ + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []common.MachineRole{common.MasterRole}, + Versions: v1alpha1.MachineVersionInfo{ + Kubelet: "1.9.3", + ControlPlane: "1.9.3", + ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + { + OS: "ubuntu-1710", + Roles: []common.MachineRole{common.MasterRole}, + Versions: v1alpha1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Master startup script", + }, + } + nodeMachineSetupConfig := config{ + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []common.MachineRole{common.NodeRole}, + Versions: v1alpha1.MachineVersionInfo{ + Kubelet: "1.9.3", + ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + { + OS: "ubuntu-1710", + Roles: []common.MachineRole{common.NodeRole}, + Versions: v1alpha1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Node startup script", + }, + } + + validConfigs := ValidConfigs{ + configList: &configList{ + Items: []config{masterMachineSetupConfig, nodeMachineSetupConfig}, + }, + } + + testTables := []struct { + params ConfigParams + expectedMatch *config + expectedErr bool + }{ + { + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []common.MachineRole{common.MasterRole}, + Versions: v1alpha1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + expectedMatch: &masterMachineSetupConfig, + expectedErr: false, + }, + { + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []common.MachineRole{common.NodeRole}, + Versions: v1alpha1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + expectedMatch: &nodeMachineSetupConfig, + expectedErr: false, + }, + { + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []common.MachineRole{common.NodeRole}, + Versions: v1alpha1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.13.0", + }, + }, + }, + expectedMatch: nil, + expectedErr: true, + }, + { + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []common.MachineRole{common.MasterRole, common.NodeRole}, + Versions: v1alpha1.MachineVersionInfo{ + Kubelet: "1.9.3", + ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + expectedMatch: nil, + expectedErr: true, + }, + } + + for _, table := range testTables { + matched, err := validConfigs.matchMachineSetupConfig(&table.params) + if !reflect.DeepEqual(matched, table.expectedMatch) { + t.Errorf("Matched machine setup config was incorrect, got: %+v,\n want %+v.", matched, table.expectedMatch) + } + if table.expectedErr && err == nil { + t.Errorf("An error was not received as expected.") + } + if !table.expectedErr && err != nil { + t.Errorf("Got unexpected error: %s", err) + } + } +} diff --git a/cluster-api/cloud/google/metadata.go b/cluster-api/cloud/google/metadata.go new file mode 100644 index 000000000..42e23f631 --- /dev/null +++ b/cluster-api/cloud/google/metadata.go @@ -0,0 +1,117 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package google + +import ( + "bytes" + "fmt" + "text/template" + + "k8s.io/kube-deploy/cluster-api/cloud/google/machinesetup" + clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" +) + +type metadataParams struct { + Token string + Cluster *clusterv1.Cluster + Machine *clusterv1.Machine + DockerImages []string + Project string + Metadata *machinesetup.Metadata + + // These fields are set when executing the template if they are necessary. + PodCIDR string + ServiceCIDR string + MasterEndpoint string +} + +func nodeMetadata(params metadataParams) (map[string]string, error) { + params.PodCIDR = getSubnet(params.Cluster.Spec.ClusterNetwork.Pods) + params.ServiceCIDR = getSubnet(params.Cluster.Spec.ClusterNetwork.Services) + params.MasterEndpoint = getEndpoint(¶ms.Cluster.Status.APIEndpoints[0]) + + metadata := map[string]string{} + var buf bytes.Buffer + if err := nodeEnvironmentVarsTemplate.Execute(&buf, params); err != nil { + return nil, err + } + buf.WriteString(params.Metadata.StartupScript) + metadata["startup-script"] = buf.String() + return metadata, nil +} + +func masterMetadata(params metadataParams) (map[string]string, error) { + params.PodCIDR = getSubnet(params.Cluster.Spec.ClusterNetwork.Pods) + params.ServiceCIDR = getSubnet(params.Cluster.Spec.ClusterNetwork.Services) + + metadata := map[string]string{} + var buf bytes.Buffer + if err := masterEnvironmentVarsTemplate.Execute(&buf, params); err != nil { + return nil, err + } + buf.WriteString(params.Metadata.StartupScript) + metadata["startup-script"] = buf.String() + return metadata, nil +} + +func getEndpoint(apiEndpoint *clusterv1.APIEndpoint) string { + return fmt.Sprintf("%s:%d", apiEndpoint.Host, apiEndpoint.Port) +} + +var ( + masterEnvironmentVarsTemplate *template.Template + nodeEnvironmentVarsTemplate *template.Template +) + +func init() { + masterEnvironmentVarsTemplate = template.Must(template.New("masterEnvironmentVars").Parse(masterEnvironmentVars)) + nodeEnvironmentVarsTemplate = template.Must(template.New("nodeEnvironmentVars").Parse(nodeEnvironmentVars)) +} + +// TODO(kcoronado): replace with actual network and node tag args when they are added into provider config. +const masterEnvironmentVars = ` +#!/bin/bash + +KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} +export VERSION=v${KUBELET_VERSION} +export ARCH=amd64 +TOKEN={{ .Token }} +PORT=443 +MACHINE={{ .Machine.ObjectMeta.Name }} +CONTROL_PLANE_VERSION={{ .Machine.Spec.Versions.ControlPlane }} +CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.DNSDomain }} +POD_CIDR={{ .PodCIDR }} +SERVICE_CIDER={{ .ServiceCIDR }} + +# Environment variables for GCE cloud config +PROJECT={{ .Project }} +NETWORK=default +SUBNETWORK=kubernetes +NODE_TAG=worker +` + +const nodeEnvironmentVars = ` +#!/bin/bash + +KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} +TOKEN={{ .Token }} +MASTER={{ .MasterEndpoint }} +MACHINE={{ .Machine.ObjectMeta.Name }} +CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.DNSDomain }} +POD_CIDR={{ .PodCIDR }} +SERVICE_CIDER={{ .ServiceCIDR }} +` diff --git a/cluster-api/cloud/google/templates.go b/cluster-api/cloud/google/templates.go deleted file mode 100644 index eb22c00ce..000000000 --- a/cluster-api/cloud/google/templates.go +++ /dev/null @@ -1,321 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package google - -import ( - "bytes" - "fmt" - "text/template" - - clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" -) - -type templateParams struct { - Token string - Cluster *clusterv1.Cluster - Machine *clusterv1.Machine - DockerImages []string - Preloaded bool -} - -func nodeMetadata(params templateParams) (map[string]string, error) { - metadata := map[string]string{} - var buf bytes.Buffer - tName := "fullScript" - if isPreloaded(params) { - tName = "preloadedScript" - } - - if err := nodeStartupScriptTemplate.ExecuteTemplate(&buf, tName, params); err != nil { - return nil, err - } - metadata["startup-script"] = buf.String() - - return metadata, nil -} - -func masterMetadata(params templateParams) (map[string]string, error) { - metadata := map[string]string{} - var buf bytes.Buffer - tName := "fullScript" - if isPreloaded(params) { - tName = "preloadedScript" - } - - if err := masterStartupScriptTemplate.ExecuteTemplate(&buf, tName, params); err != nil { - return nil, err - } - metadata["startup-script"] = buf.String() - return metadata, nil -} - -func isPreloaded(params templateParams) bool { - return params.Preloaded -} - -// PreloadMasterScript returns a script that can be used to preload a master. -func PreloadMasterScript(version string, dockerImages []string) (string, error) { - return preloadScript(masterStartupScriptTemplate, version, dockerImages) -} - -// PreloadNodeScript returns a script that can be used to preload a master. -func PreloadNodeScript(version string, dockerImages []string) (string, error) { - return preloadScript(nodeStartupScriptTemplate, version, dockerImages) -} - -func preloadScript(t *template.Template, version string, dockerImages []string) (string, error) { - var buf bytes.Buffer - params := templateParams{ - Machine: &clusterv1.Machine{}, - DockerImages: dockerImages, - } - params.Machine.Spec.Versions.Kubelet = version - err := t.ExecuteTemplate(&buf, "generatePreloadedImage", params) - return buf.String(), err -} - -var ( - nodeStartupScriptTemplate *template.Template - masterStartupScriptTemplate *template.Template -) - -func init() { - endpoint := func(apiEndpoint *clusterv1.APIEndpoint) string { - return fmt.Sprintf("%s:%d", apiEndpoint.Host, apiEndpoint.Port) - } - // Force a compliation error if getSubnet changes. This is the - // signature the templates expect, so changes need to be - // reflected in templates below. - var _ func(clusterv1.NetworkRanges) string = getSubnet - funcMap := map[string]interface{}{ - "endpoint": endpoint, - "getSubnet": getSubnet, - } - nodeStartupScriptTemplate = template.Must(template.New("nodeStartupScript").Funcs(funcMap).Parse(nodeStartupScript)) - nodeStartupScriptTemplate = template.Must(nodeStartupScriptTemplate.Parse(genericTemplates)) - masterStartupScriptTemplate = template.Must(template.New("masterStartupScript").Funcs(funcMap).Parse(masterStartupScript)) - masterStartupScriptTemplate = template.Must(masterStartupScriptTemplate.Parse(genericTemplates)) -} - -const genericTemplates = ` -{{ define "fullScript" -}} - {{ template "startScript" . }} - {{ template "install" . }} - {{ template "configure" . }} - {{ template "endScript" . }} -{{- end }} - -{{ define "preloadedScript" -}} - {{ template "startScript" . }} - {{ template "configure" . }} - {{ template "endScript" . }} -{{- end }} - -{{ define "generatePreloadedImage" -}} - {{ template "startScript" . }} - {{ template "install" . }} - -systemctl enable docker || true -systemctl start docker || true - - {{ range .DockerImages }} -docker pull {{ . }} - {{ end }} - - {{ template "endScript" . }} -{{- end }} - -{{ define "startScript" -}} -#!/bin/bash - -set -e -set -x - -( -{{- end }} - -{{define "endScript" -}} - -echo done. -) 2>&1 | tee /var/log/startup.log - -{{- end }} -` - -const nodeStartupScript = ` -{{ define "install" -}} -apt-get update -apt-get install -y apt-transport-https prips -apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys F76221572C52609D - -cat < /etc/apt/sources.list.d/k8s.list -deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main -EOF - -apt-get update -apt-get install -y docker-engine=1.12.0-0~xenial - -curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -cat < /etc/apt/sources.list.d/kubernetes.list -deb http://apt.kubernetes.io/ kubernetes-xenial main -EOF -apt-get update - -{{- end }} {{/* end install */}} - -{{ define "configure" -}} -KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} -TOKEN={{ .Token }} -MASTER={{ index .Cluster.Status.APIEndpoints 0 | endpoint }} -MACHINE={{ .Machine.ObjectMeta.Name }} -CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }} -SERVICE_CIDR={{ getSubnet .Cluster.Spec.ClusterNetwork.Services }} - -# Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix -# search based on our SemVer to find the right (newest) package version. -function getversion() { - name=$1 - prefix=$2 - version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) - if [[ -z "$version" ]]; then - echo Can\'t find package $name with prefix $prefix - exit 1 - fi - echo $version -} - -KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) -KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) -KUBECTL=$(getversion kubectl ${KUBELET_VERSION}-) -# Explicit cni version is a temporary workaround till the right version can be automatically detected correctly -apt-get install -y kubelet=${KUBELET} kubeadm=${KUBEADM} kubectl=${KUBECTL} kubernetes-cni=0.5.1-00 - -systemctl enable docker || true -systemctl start docker || true - -sysctl net.bridge.bridge-nf-call-iptables=1 - -# kubeadm uses 10th IP as DNS server -CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) - -sed -i "s/KUBELET_DNS_ARGS=[^\"]*/KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -systemctl daemon-reload -systemctl restart kubelet.service - -kubeadm join --token "${TOKEN}" "${MASTER}" --skip-preflight-checks - -for tries in $(seq 1 60); do - kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break - sleep 1 -done -{{- end }} {{/* end configure */}} -` - -// TODO: actually init the cluster, templatize token, etc. -const masterStartupScript = ` -{{ define "install" -}} - -KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} - -curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - -touch /etc/apt/sources.list.d/kubernetes.list -sh -c 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list' - -apt-get update -y - -apt-get install -y \ - socat \ - ebtables \ - docker.io \ - apt-transport-https \ - cloud-utils \ - prips - -export VERSION=v${KUBELET_VERSION} -export ARCH=amd64 -curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl -chmod a+rx /usr/bin/kubeadm.dl -{{- end }} {{/* end install */}} - - -{{ define "configure" -}} -KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} -TOKEN={{ .Token }} -PORT=443 -MACHINE={{ .Machine.ObjectMeta.Name }} -CONTROL_PLANE_VERSION={{ .Machine.Spec.Versions.ControlPlane }} -CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }} -POD_CIDR={{ getSubnet .Cluster.Spec.ClusterNetwork.Pods }} -SERVICE_CIDR={{ getSubnet .Cluster.Spec.ClusterNetwork.Services }} - -# kubeadm uses 10th IP as DNS server -CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) - -# Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix -# search based on our SemVer to find the right (newest) package version. -function getversion() { - name=$1 - prefix=$2 - version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) - if [[ -z "$version" ]]; then - echo Can\'t find package $name with prefix $prefix - exit 1 - fi - echo $version -} - -KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) -KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) - -# Explicit cni version is a temporary workaround till the right version can be automatically detected correctly -apt-get install -y \ - kubelet=${KUBELET} \ - kubeadm=${KUBEADM} \ - kubernetes-cni=0.5.1-00 - -mv /usr/bin/kubeadm.dl /usr/bin/kubeadm -chmod a+rx /usr/bin/kubeadm - -systemctl enable docker -systemctl start docker -sed -i "s/KUBELET_DNS_ARGS=[^\"]*/KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -systemctl daemon-reload -systemctl restart kubelet.service -` + - "PRIVATEIP=`curl --retry 5 -sfH \"Metadata-Flavor: Google\" \"http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip\"`" + ` -echo $PRIVATEIP > /tmp/.ip -` + - "PUBLICIP=`curl --retry 5 -sfH \"Metadata-Flavor: Google\" \"http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip\"`" + ` - - -kubeadm init --apiserver-bind-port ${PORT} --token ${TOKEN} --kubernetes-version v${CONTROL_PLANE_VERSION} \ - --apiserver-advertise-address ${PUBLICIP} --apiserver-cert-extra-sans ${PUBLICIP} ${PRIVATEIP} \ - --service-cidr ${SERVICE_CIDR} - -# install weavenet -sysctl net.bridge.bridge-nf-call-iptables=1 -export kubever=$(kubectl version --kubeconfig /etc/kubernetes/admin.conf | base64 | tr -d '\n') -kubectl apply --kubeconfig /etc/kubernetes/admin.conf -f "https://cloud.weave.works/k8s/net?k8s-version=$kubever" - -for tries in $(seq 1 60); do - kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break - sleep 1 -done - -{{- end }} {{/* end configure */}} -` diff --git a/cluster-api/gcp-deployer/machine_setup_configs.yaml b/cluster-api/gcp-deployer/machine_setup_configs.yaml new file mode 100644 index 000000000..1543c6ae0 --- /dev/null +++ b/cluster-api/gcp-deployer/machine_setup_configs.yaml @@ -0,0 +1,377 @@ +items: +- machineParams: + - os: ubuntu-1710-weave + roles: + - Node + versions: + kubelet: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710 + metadata: + startupScript: | + set -e + set -x + + ( + apt-get update + apt-get install -y apt-transport-https prips + apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys F76221572C52609D + + cat < /etc/apt/sources.list.d/k8s.list + deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main + EOF + + apt-get update + apt-get install -y docker-engine=1.12.0-0~xenial + + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + + cat < /etc/apt/sources.list.d/kubernetes.list + deb http://apt.kubernetes.io/ kubernetes-xenial main + EOF + apt-get update + + # Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix + # search based on our SemVer to find the right (newest) package version. + function getversion() { + name=$1 + prefix=$2 + version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) + if [[ -z "$version" ]]; then + echo Can\'t find package $name with prefix $prefix + exit 1 + fi + echo $version + } + + KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) + KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) + KUBECTL=$(getversion kubectl ${KUBELET_VERSION}-) + apt-get install -y kubelet=${KUBELET} kubeadm=${KUBEADM} kubectl=${KUBECTL} + + systemctl enable docker || true + systemctl start docker || true + + # kubeadm uses 10th IP as DNS server + CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) + + # Override Kubelet DNS args. + cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <&1 | tee /var/log/startup.log +- machineParams: + - os: ubuntu-1710-weave + roles: + - Master + versions: + kubelet: 1.9.4 + controlPlane: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710 + metadata: + startupScript: | + set -e + set -x + + ( + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - + touch /etc/apt/sources.list.d/kubernetes.list + sh -c 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list' + + apt-get update -y + + apt-get install -y \ + socat \ + ebtables \ + docker.io \ + apt-transport-https \ + cloud-utils \ + prips + + curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl + chmod a+rx /usr/bin/kubeadm.dl + + # kubeadm uses 10th IP as DNS server + CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) + + # Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix + # search based on our SemVer to find the right (newest) package version. + function getversion() { + name=$1 + prefix=$2 + version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) + if [[ -z "$version" ]]; then + echo Can\'t find package $name with prefix $prefix + exit 1 + fi + echo $version + } + + KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) + KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) + + apt-get install -y \ + kubelet=${KUBELET} \ + kubeadm=${KUBEADM} + + mv /usr/bin/kubeadm.dl /usr/bin/kubeadm + chmod a+rx /usr/bin/kubeadm + + systemctl enable docker + systemctl start docker + + # Override Kubelet DNS args. + cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf < /tmp/.ip + PUBLICIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"` + + # Set up kubeadm config file to pass parameters to kubeadm init. + cat > /etc/kubernetes/kubeadm_config.yaml <&1 | tee /var/log/startup.log + +# These configs currently don't work - they need service accounts. +- machineParams: + - os: ubuntu-1604-lts + roles: + - Master + versions: + kubelet: 1.9.4 + controlPlane: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts + metadata: + startupScript: | + set -e + set -x + + ( + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - + touch /etc/apt/sources.list.d/kubernetes.list + sh -c 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list' + + apt-get update -y + + apt-get install -y \ + socat \ + ebtables \ + docker.io \ + apt-transport-https \ + cloud-utils \ + prips + + curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl + chmod a+rx /usr/bin/kubeadm.dl + + # kubeadm uses 10th IP as DNS server + CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) + + # Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix + # search based on our SemVer to find the right (newest) package version. + function getversion() { + name=$1 + prefix=$2 + version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) + if [[ -z "$version" ]]; then + echo Can\'t find package $name with prefix $prefix + exit 1 + fi + echo $version + } + + KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) + KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) + + apt-get install -y \ + kubelet=${KUBELET} \ + kubeadm=${KUBEADM} + + mv /usr/bin/kubeadm.dl /usr/bin/kubeadm + chmod a+rx /usr/bin/kubeadm + + systemctl enable docker + systemctl start docker + + # Override network args to use kubenet instead of cni, and override Kubelet DNS args. + cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf < /tmp/.ip + PUBLICIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"` + + # Set up the GCE cloud config, which gets picked up by kubeadm init since cloudProvider is set to GCE. + cat > /etc/kubernetes/cloud-config < /etc/kubernetes/kubeadm_config.yaml <&1 | tee /var/log/startup.log +- machineParams: + - os: ubuntu-1604-lts + roles: + - Node + versions: + kubelet: 1.9.4 + containerRuntime: + name: docker + version: 1.12.0 + image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts + metadata: + startupScript: | + set -e + set -x + + ( + apt-get update + apt-get install -y apt-transport-https prips + apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys F76221572C52609D + + cat < /etc/apt/sources.list.d/k8s.list + deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main + EOF + + apt-get update + apt-get install -y docker-engine=1.12.0-0~xenial + + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + + cat < /etc/apt/sources.list.d/kubernetes.list + deb http://apt.kubernetes.io/ kubernetes-xenial main + EOF + apt-get update + + # Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix + # search based on our SemVer to find the right (newest) package version. + function getversion() { + name=$1 + prefix=$2 + version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1) + if [[ -z "$version" ]]; then + echo Can\'t find package $name with prefix $prefix + exit 1 + fi + echo $version + } + + KUBELET=$(getversion kubelet ${KUBELET_VERSION}-) + KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-) + KUBECTL=$(getversion kubectl ${KUBELET_VERSION}-) + apt-get install -y kubelet=${KUBELET} kubeadm=${KUBEADM} kubectl=${KUBECTL} + + systemctl enable docker || true + systemctl start docker || true + + # kubeadm uses 10th IP as DNS server + CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1) + + # Override network args to use kubenet instead of cni, and override Kubelet DNS args. + cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <&1 | tee /var/log/startup.log From 242d1935079320c3834beaa00e131e3254146744 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Tue, 27 Mar 2018 16:44:44 -0700 Subject: [PATCH 03/21] Remove call to preloaded script in generate-image --- .../cloud/google/cmd/generate-image/main.go | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/cluster-api/cloud/google/cmd/generate-image/main.go b/cluster-api/cloud/google/cmd/generate-image/main.go index 18c69a5b9..398deb954 100644 --- a/cluster-api/cloud/google/cmd/generate-image/main.go +++ b/cluster-api/cloud/google/cmd/generate-image/main.go @@ -21,13 +21,12 @@ import ( "github.com/golang/glog" "github.com/spf13/cobra" - "k8s.io/kube-deploy/cluster-api/cloud/google" + "io/ioutil" + "os" ) type options struct { - version string - role string - dockerImages []string + script string } var opts options @@ -36,38 +35,31 @@ var generateCmd = &cobra.Command{ Use: "generate_image", Short: "Outputs a script to generate a preloaded image", Run: func(cmd *cobra.Command, args []string) { + if opts.script == "" { + glog.Error("Please provide a startup script.") + cmd.Help() + os.Exit(1) + } + if err := runGenerate(opts); err != nil { glog.Exit(err) } }, } -// TODO(kcoronado): take a script file as a parameter instead of generating the preloaded script from the template. func init() { - generateCmd.Flags().StringVar(&opts.version, "version", "1.7.3", "The version of kubernetes to install") - generateCmd.Flags().StringVar(&opts.role, "role", "master", "The role of the machine (master or node)") - generateCmd.Flags().StringArrayVar(&opts.dockerImages, "extra-docker-images", []string{}, "extra docker images to preload") + generateCmd.Flags().StringVar(&opts.script, "script", "", "The path to the machine's startup script") } func runGenerate(o options) error { - var script string - var err error - switch o.role { - case "master": - script, err = google.PreloadMasterScript(o.version, o.dockerImages) - case "node": - script, err = google.PreloadMasterScript(o.version, o.dockerImages) - default: - return fmt.Errorf("unrecognized role: %q", o.role) - } - + bytes, err := ioutil.ReadFile(o.script) if err != nil { return err } // just print the script for now // TODO actually start a VM, let it run the script, stop the VM, then create the image - fmt.Println(script) + fmt.Println(string(bytes)) return nil } From 9b9a8b95ac9de51d4839161f85571ff8d03fb534 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Wed, 28 Mar 2018 16:42:07 -0700 Subject: [PATCH 04/21] Create/mount configmap for machine setup configs --- .../cloud/google/config/configtemplate.go | 6 + cluster-api/cloud/google/machineactuator.go | 27 +++- .../cloud/google/machinesetup/config_types.go | 8 ++ .../google/machinesetup/config_types_test.go | 127 ++++++++++++++---- cluster-api/cloud/google/metadata.go | 6 +- cluster-api/gcp-deployer/cmd/add.go | 2 +- cluster-api/gcp-deployer/cmd/create.go | 13 +- cluster-api/gcp-deployer/cmd/delete.go | 2 +- cluster-api/gcp-deployer/deploy/deploy.go | 28 ++-- .../gcp-deployer/deploy/deploy_helper.go | 14 +- .../gcp-deployer/deploy/machinedeployer.go | 3 +- .../gcp-deployer/machines.yaml.template | 6 +- 12 files changed, 182 insertions(+), 60 deletions(-) diff --git a/cluster-api/cloud/google/config/configtemplate.go b/cluster-api/cloud/google/config/configtemplate.go index 82b412d41..6c024dfaf 100644 --- a/cluster-api/cloud/google/config/configtemplate.go +++ b/cluster-api/cloud/google/config/configtemplate.go @@ -139,6 +139,8 @@ spec: mountPath: /etc/credentials - name: sshkeys mountPath: /etc/sshkeys + - name: machine-setup + mountPath: /etc/machinesetup env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/credentials/service-account.json @@ -147,6 +149,7 @@ spec: args: - --kubeconfig=/etc/kubernetes/admin.conf - --token={{ .Token }} + - --config=/etc/machinesetup/machine_setup_configs.yaml resources: requests: cpu: 100m @@ -171,6 +174,9 @@ spec: - name: credentials secret: secretName: machine-controller-credential + - name: machine-setup + configMap: + name: machine-setup --- apiVersion: apps/v1beta1 kind: StatefulSet diff --git a/cluster-api/cloud/google/machineactuator.go b/cluster-api/cloud/google/machineactuator.go index b18bd02eb..9031d5088 100644 --- a/cluster-api/cloud/google/machineactuator.go +++ b/cluster-api/cloud/google/machineactuator.go @@ -37,6 +37,9 @@ import ( "regexp" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" gceconfig "k8s.io/kube-deploy/cluster-api/cloud/google/gceproviderconfig" gceconfigv1 "k8s.io/kube-deploy/cluster-api/cloud/google/gceproviderconfig/v1alpha1" "k8s.io/kube-deploy/cluster-api/cloud/google/machinesetup" @@ -53,6 +56,8 @@ const ( UIDLabelKey = "machine-crd-uid" BootstrapLabelKey = "boostrap" + + MachineSetupConfigsFilename = "machine_setup_configs.yaml" ) type SshCreds struct { @@ -128,7 +133,7 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa }, nil } -func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { +func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine, clientSet kubernetes.Clientset) error { if err := gce.CreateMachineControllerServiceAccount(cluster, initialMachines); err != nil { return err } @@ -142,6 +147,26 @@ func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initia return err } + // Create the configmap so the machine setup configs can be mounted into the node. + machineSetupConfigs, err := gce.configWatch.ValidConfigs() + if err != nil { + return err + } + yaml, err := machineSetupConfigs.GetYaml() + if err != nil { + return err + } + configMap := corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{Name: "machine-setup"}, + Data: map[string]string{ + MachineSetupConfigsFilename: yaml, + }, + } + configMaps := clientSet.CoreV1().ConfigMaps(corev1.NamespaceDefault) + if _, err := configMaps.Create(&configMap); err != nil { + return err + } + if err := CreateApiServerAndController(gce.kubeadmToken); err != nil { return err } diff --git a/cluster-api/cloud/google/machinesetup/config_types.go b/cluster-api/cloud/google/machinesetup/config_types.go index 422c5362e..4482a3524 100644 --- a/cluster-api/cloud/google/machinesetup/config_types.go +++ b/cluster-api/cloud/google/machinesetup/config_types.go @@ -76,6 +76,14 @@ func parseMachineSetupYaml(reader io.Reader) (*ValidConfigs, error) { return &ValidConfigs{configList}, nil } +func (vc *ValidConfigs) GetYaml() (string, error) { + bytes, err := yaml.Marshal(vc.configList) + if err != nil { + return "", err + } + return string(bytes), nil +} + func (vc *ValidConfigs) GetImage(params *ConfigParams) (string, error) { machineSetupConfig, err := vc.matchMachineSetupConfig(params) if err != nil { diff --git a/cluster-api/cloud/google/machinesetup/config_types_test.go b/cluster-api/cloud/google/machinesetup/config_types_test.go index 46fcacda7..0b6c56305 100644 --- a/cluster-api/cloud/google/machinesetup/config_types_test.go +++ b/cluster-api/cloud/google/machinesetup/config_types_test.go @@ -2,8 +2,8 @@ package machinesetup import ( "io" - "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" - "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" + clustercommon "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" + clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" "reflect" "strings" "testing" @@ -90,16 +90,89 @@ func TestParseMachineSetupYaml(t *testing.T) { } } +func TestGetYaml(t *testing.T) { + testTables := []struct { + validConfigs ValidConfigs + expectedStrings []string + expectedErr bool + }{ + { + validConfigs: ValidConfigs{ + configList: &configList{ + Items: []config{ + { + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Master startup script", + }, + }, + { + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Node startup script", + }, + }, + }, + }, + }, + expectedStrings: []string{"startupScript: Master startup script", "startupScript: Node startup script"}, + expectedErr: false, + }, + } + + for _, table := range testTables { + yaml, err := table.validConfigs.GetYaml() + if table.expectedErr && err == nil { + t.Errorf("An error was not received as expected.") + } + if !table.expectedErr && err != nil { + t.Errorf("Got unexpected error: %s", err) + } + for _, expectedString := range table.expectedStrings { + if !strings.Contains(yaml, expectedString) { + t.Errorf("Yaml did not contain expected string, got:\n%s\nwant:\n%s", yaml, expectedString) + } + } + } +} + func TestMatchMachineSetupConfig(t *testing.T) { masterMachineSetupConfig := config{ Params: []ConfigParams{ { OS: "ubuntu-1710", - Roles: []common.MachineRole{common.MasterRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.3", ControlPlane: "1.9.3", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -107,11 +180,11 @@ func TestMatchMachineSetupConfig(t *testing.T) { }, { OS: "ubuntu-1710", - Roles: []common.MachineRole{common.MasterRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.4", ControlPlane: "1.9.4", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -127,10 +200,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { Params: []ConfigParams{ { OS: "ubuntu-1710", - Roles: []common.MachineRole{common.NodeRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.3", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -138,10 +211,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { }, { OS: "ubuntu-1710", - Roles: []common.MachineRole{common.NodeRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.4", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -168,11 +241,11 @@ func TestMatchMachineSetupConfig(t *testing.T) { { params: ConfigParams{ OS: "ubuntu-1710", - Roles: []common.MachineRole{common.MasterRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.4", ControlPlane: "1.9.4", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -184,10 +257,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { { params: ConfigParams{ OS: "ubuntu-1710", - Roles: []common.MachineRole{common.NodeRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.4", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -199,10 +272,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { { params: ConfigParams{ OS: "ubuntu-1710", - Roles: []common.MachineRole{common.NodeRole}, - Versions: v1alpha1.MachineVersionInfo{ - Kubelet: "1.9.4", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.13.0", }, @@ -214,10 +287,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { { params: ConfigParams{ OS: "ubuntu-1710", - Roles: []common.MachineRole{common.MasterRole, common.NodeRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.3", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, diff --git a/cluster-api/cloud/google/metadata.go b/cluster-api/cloud/google/metadata.go index 42e23f631..972bcf32a 100644 --- a/cluster-api/cloud/google/metadata.go +++ b/cluster-api/cloud/google/metadata.go @@ -87,15 +87,15 @@ const masterEnvironmentVars = ` #!/bin/bash KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} -export VERSION=v${KUBELET_VERSION} -export ARCH=amd64 +VERSION=v${KUBELET_VERSION} +ARCH=amd64 TOKEN={{ .Token }} PORT=443 MACHINE={{ .Machine.ObjectMeta.Name }} CONTROL_PLANE_VERSION={{ .Machine.Spec.Versions.ControlPlane }} CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.DNSDomain }} POD_CIDR={{ .PodCIDR }} -SERVICE_CIDER={{ .ServiceCIDR }} +SERVICE_CIDR={{ .ServiceCIDR }} # Environment variables for GCE cloud config PROJECT={{ .Project }} diff --git a/cluster-api/gcp-deployer/cmd/add.go b/cluster-api/gcp-deployer/cmd/add.go index df45341e7..a7a1ef2eb 100644 --- a/cluster-api/gcp-deployer/cmd/add.go +++ b/cluster-api/gcp-deployer/cmd/add.go @@ -52,7 +52,7 @@ func RunAdd(ao *AddOptions) error { return err } - d := deploy.NewDeployer(provider, kubeConfig) + d := deploy.NewDeployer(provider, kubeConfig, "") return d.AddNodes(machines) } diff --git a/cluster-api/gcp-deployer/cmd/create.go b/cluster-api/gcp-deployer/cmd/create.go index 63bbe9c05..d05e2e7cd 100644 --- a/cluster-api/gcp-deployer/cmd/create.go +++ b/cluster-api/gcp-deployer/cmd/create.go @@ -25,8 +25,9 @@ import ( ) type CreateOptions struct { - Cluster string - Machine string + Cluster string + Machine string + MachineSetup string } var co = &CreateOptions{} @@ -46,6 +47,11 @@ var createCmd = &cobra.Command{ cmd.Help() os.Exit(1) } + if co.MachineSetup == "" { + glog.Error("Please provide yaml file for machine setup configs.") + cmd.Help() + os.Exit(1) + } if err := RunCreate(co); err != nil { glog.Exit(err) } @@ -63,13 +69,14 @@ func RunCreate(co *CreateOptions) error { return err } - d := deploy.NewDeployer(provider, kubeConfig) + d := deploy.NewDeployer(provider, kubeConfig, co.MachineSetup) return d.CreateCluster(cluster, machines) } func init() { createCmd.Flags().StringVarP(&co.Cluster, "cluster", "c", "", "cluster yaml file") createCmd.Flags().StringVarP(&co.Machine, "machines", "m", "", "machine yaml file") + createCmd.Flags().StringVarP(&co.MachineSetup, "setup", "s", "", "machine setup configs yaml file") RootCmd.AddCommand(createCmd) } diff --git a/cluster-api/gcp-deployer/cmd/delete.go b/cluster-api/gcp-deployer/cmd/delete.go index ec991ea5a..5d375ff8b 100644 --- a/cluster-api/gcp-deployer/cmd/delete.go +++ b/cluster-api/gcp-deployer/cmd/delete.go @@ -34,7 +34,7 @@ var deleteCmd = &cobra.Command{ } func RunDelete() error { - d := deploy.NewDeployer(provider, kubeConfig) + d := deploy.NewDeployer(provider, kubeConfig, "") return d.DeleteCluster() } diff --git a/cluster-api/gcp-deployer/deploy/deploy.go b/cluster-api/gcp-deployer/deploy/deploy.go index adb2ab331..efd2c90ca 100644 --- a/cluster-api/gcp-deployer/deploy/deploy.go +++ b/cluster-api/gcp-deployer/deploy/deploy.go @@ -22,6 +22,7 @@ import ( "github.com/golang/glog" + "k8s.io/client-go/kubernetes" "k8s.io/kube-deploy/cluster-api/cloud/google" clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" "k8s.io/kube-deploy/cluster-api/pkg/client/clientset_generated/clientset" @@ -31,37 +32,38 @@ import ( ) type deployer struct { - token string - configPath string - machineDeployer machineDeployer - client v1alpha1.ClusterV1alpha1Interface - clientSet clientset.Interface + token string + configPath string + machineDeployer machineDeployer + client v1alpha1.ClusterV1alpha1Interface + clientSet clientset.Interface + kubernetesClientSet kubernetes.Clientset } // NewDeployer returns a cloud provider specific deployer and // sets kubeconfig path for the cluster to be deployed -func NewDeployer(provider string, configPath string) *deployer { +func NewDeployer(provider string, kubeConfigPath string, machineSetupConfigPath string) *deployer { token := util.RandomToken() - if configPath == "" { - configPath = os.Getenv("KUBECONFIG") - if configPath == "" { - configPath = apiutil.GetDefaultKubeConfigPath() + if kubeConfigPath == "" { + kubeConfigPath = os.Getenv("KUBECONFIG") + if kubeConfigPath == "" { + kubeConfigPath = apiutil.GetDefaultKubeConfigPath() } } else { // This is needed for kubectl commands run later to create secret in function // CreateMachineControllerServiceAccount - if err := os.Setenv("KUBECONFIG", configPath); err != nil { + if err := os.Setenv("KUBECONFIG", kubeConfigPath); err != nil { glog.Exit(fmt.Sprintf("Failed to set Kubeconfig path err %v\n", err)) } } - ma, err := google.NewMachineActuator(token, nil) + ma, err := google.NewMachineActuator(token, nil, machineSetupConfigPath) if err != nil { glog.Exit(err) } return &deployer{ token: token, machineDeployer: ma, - configPath: configPath, + configPath: kubeConfigPath, } } diff --git a/cluster-api/gcp-deployer/deploy/deploy_helper.go b/cluster-api/gcp-deployer/deploy/deploy_helper.go index 4366d1245..8adcd457e 100644 --- a/cluster-api/gcp-deployer/deploy/deploy_helper.go +++ b/cluster-api/gcp-deployer/deploy/deploy_helper.go @@ -88,7 +88,7 @@ func (d *deployer) createCluster(c *clusterv1.Cluster, machines []*clusterv1.Mac } glog.Info("Deploying the addon apiserver and controller manager...") - if err := d.machineDeployer.CreateMachineController(c, machines); err != nil { + if err := d.machineDeployer.CreateMachineController(c, machines, d.kubernetesClientSet); err != nil { return fmt.Errorf("can't create machine controller: %v", err) } @@ -232,8 +232,13 @@ func (d *deployer) initApiClient() error { if err != nil { return err } + kubernetesClientSet, err := util.NewKubernetesClient(d.configPath) + if err != nil { + return err + } d.clientSet = c d.client = c.ClusterV1alpha1() + d.kubernetesClientSet = *kubernetesClientSet return nil } @@ -278,14 +283,9 @@ func (d *deployer) waitForApiserver(master string) error { // Make sure the default service account in kube-system namespace exists. func (d *deployer) waitForServiceAccount() error { - client, err := util.NewKubernetesClient(d.configPath) - if err != nil { - return err - } - waitErr := util.Retry(func() (bool, error) { glog.Info("Waiting for the service account to exist...") - _, err = client.CoreV1().ServiceAccounts(ServiceAccountNs).Get(ServiceAccountName, metav1.GetOptions{}) + _, err := d.kubernetesClientSet.CoreV1().ServiceAccounts(ServiceAccountNs).Get(ServiceAccountName, metav1.GetOptions{}) return (err == nil), nil }, 5) diff --git a/cluster-api/gcp-deployer/deploy/machinedeployer.go b/cluster-api/gcp-deployer/deploy/machinedeployer.go index 5e32f3f4a..649c4522f 100644 --- a/cluster-api/gcp-deployer/deploy/machinedeployer.go +++ b/cluster-api/gcp-deployer/deploy/machinedeployer.go @@ -1,6 +1,7 @@ package deploy import ( + "k8s.io/client-go/kubernetes" clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" "k8s.io/kube-deploy/cluster-api/pkg/controller/machine" ) @@ -16,6 +17,6 @@ type machineDeployer interface { // are provided in case the function wants to refer to them (and their // ProviderConfigs) to know how to configure the machine controller. // Not idempotent. - CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error + CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine, clientSet kubernetes.Clientset) error PostDelete(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error } diff --git a/cluster-api/gcp-deployer/machines.yaml.template b/cluster-api/gcp-deployer/machines.yaml.template index 40ffe914b..b95699df7 100644 --- a/cluster-api/gcp-deployer/machines.yaml.template +++ b/cluster-api/gcp-deployer/machines.yaml.template @@ -15,8 +15,8 @@ items: machineType: "n1-standard-2" os: "ubuntu-1604-lts" versions: - kubelet: 1.8.3 - controlPlane: 1.8.3 + kubelet: 1.9.4 + controlPlane: 1.9.4 containerRuntime: name: docker version: 1.12.0 @@ -38,7 +38,7 @@ items: machineType: "n1-standard-1" os: "ubuntu-1604-lts" versions: - kubelet: 1.8.3 + kubelet: 1.9.4 containerRuntime: name: docker version: 1.12.0 From bd5d0ec93c0881088caf3603f50ba2e6326395ec Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Wed, 4 Apr 2018 16:43:17 -0700 Subject: [PATCH 05/21] fixup 6bc01dd Remove nil check for ConfigWatch --- cluster-api/cloud/google/machineactuator.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cluster-api/cloud/google/machineactuator.go b/cluster-api/cloud/google/machineactuator.go index 9031d5088..504c29438 100644 --- a/cluster-api/cloud/google/machineactuator.go +++ b/cluster-api/cloud/google/machineactuator.go @@ -189,9 +189,6 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach return errors.New("invalid master configuration: missing Machine.Spec.Versions.Kubelet") } - if gce.configWatch == nil { - return errors.New("invalid machine setup configuration: missing GCEClient.ConfigWatch") - } machineSetupConfigs, err := gce.configWatch.ValidConfigs() if err != nil { return err From a2782a14a7c6c37995300030c6ac3ea1bf253729 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Wed, 4 Apr 2018 17:03:55 -0700 Subject: [PATCH 06/21] fixup 6bc01dd fix metadata method parameters --- cluster-api/cloud/google/machineactuator.go | 19 +-------- cluster-api/cloud/google/metadata.go | 43 ++++++++++++++------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/cluster-api/cloud/google/machineactuator.go b/cluster-api/cloud/google/machineactuator.go index 504c29438..33d83d41a 100644 --- a/cluster-api/cloud/google/machineactuator.go +++ b/cluster-api/cloud/google/machineactuator.go @@ -214,15 +214,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach "invalid master configuration: missing Machine.Spec.Versions.ControlPlane")) } var err error - metadata, err = masterMetadata( - metadataParams{ - Token: gce.kubeadmToken, - Cluster: cluster, - Machine: machine, - Project: config.Project, - Metadata: &machineSetupMetadata, - }, - ) + metadata, err = masterMetadata(gce.kubeadmToken, cluster, machine, config.Project, &machineSetupMetadata) if err != nil { return err } @@ -231,14 +223,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach return errors.New("invalid cluster state: cannot create a Kubernetes node without an API endpoint") } var err error - metadata, err = nodeMetadata( - metadataParams{ - Token: gce.kubeadmToken, - Cluster: cluster, - Machine: machine, - Metadata: &machineSetupMetadata, - }, - ) + metadata, err = nodeMetadata(gce.kubeadmToken, cluster, machine, &machineSetupMetadata) if err != nil { return err } diff --git a/cluster-api/cloud/google/metadata.go b/cluster-api/cloud/google/metadata.go index 972bcf32a..7cc9f938d 100644 --- a/cluster-api/cloud/google/metadata.go +++ b/cluster-api/cloud/google/metadata.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -39,36 +39,49 @@ type metadataParams struct { MasterEndpoint string } -func nodeMetadata(params metadataParams) (map[string]string, error) { - params.PodCIDR = getSubnet(params.Cluster.Spec.ClusterNetwork.Pods) - params.ServiceCIDR = getSubnet(params.Cluster.Spec.ClusterNetwork.Services) - params.MasterEndpoint = getEndpoint(¶ms.Cluster.Status.APIEndpoints[0]) +func nodeMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.Machine, metadata *machinesetup.Metadata) (map[string]string, error) { + params := metadataParams{ + Token: token, + Cluster: cluster, + Machine: machine, + Metadata: metadata, + PodCIDR: getSubnet(cluster.Spec.ClusterNetwork.Pods), + ServiceCIDR: getSubnet(cluster.Spec.ClusterNetwork.Services), + MasterEndpoint: getEndpoint(cluster.Status.APIEndpoints[0]), + } - metadata := map[string]string{} + nodeMetadata := map[string]string{} var buf bytes.Buffer if err := nodeEnvironmentVarsTemplate.Execute(&buf, params); err != nil { return nil, err } buf.WriteString(params.Metadata.StartupScript) - metadata["startup-script"] = buf.String() - return metadata, nil + nodeMetadata["startup-script"] = buf.String() + return nodeMetadata, nil } -func masterMetadata(params metadataParams) (map[string]string, error) { - params.PodCIDR = getSubnet(params.Cluster.Spec.ClusterNetwork.Pods) - params.ServiceCIDR = getSubnet(params.Cluster.Spec.ClusterNetwork.Services) +func masterMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.Machine, project string, metadata *machinesetup.Metadata) (map[string]string, error) { + params := metadataParams{ + Token: token, + Cluster: cluster, + Machine: machine, + Project: project, + Metadata: metadata, + PodCIDR: getSubnet(cluster.Spec.ClusterNetwork.Pods), + ServiceCIDR: getSubnet(cluster.Spec.ClusterNetwork.Services), + } - metadata := map[string]string{} + masterMetadata := map[string]string{} var buf bytes.Buffer if err := masterEnvironmentVarsTemplate.Execute(&buf, params); err != nil { return nil, err } buf.WriteString(params.Metadata.StartupScript) - metadata["startup-script"] = buf.String() - return metadata, nil + masterMetadata["startup-script"] = buf.String() + return masterMetadata, nil } -func getEndpoint(apiEndpoint *clusterv1.APIEndpoint) string { +func getEndpoint(apiEndpoint clusterv1.APIEndpoint) string { return fmt.Sprintf("%s:%d", apiEndpoint.Host, apiEndpoint.Port) } From dfc5f96c94d88d764d7e78829da5c53ad279c991 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 5 Apr 2018 09:45:52 -0700 Subject: [PATCH 07/21] fixup 6bc01dd must use full image path in configs --- cluster-api/cloud/google/machineactuator.go | 39 +++++++------------ .../cloud/google/machinesetup/config_types.go | 5 +-- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/cluster-api/cloud/google/machineactuator.go b/cluster-api/cloud/google/machineactuator.go index 33d83d41a..27c774859 100644 --- a/cluster-api/cloud/google/machineactuator.go +++ b/cluster-api/cloud/google/machineactuator.go @@ -202,7 +202,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach if err != nil { return err } - imagePath := gce.getImagePath(image, config.Project) + imagePath := gce.getImagePath(image) machineSetupMetadata, err := machineSetupConfigs.GetMetadata(configParams) if err != nil { @@ -686,35 +686,24 @@ func (gce *GCEClient) handleMachineError(machine *clusterv1.Machine, err *apierr return err } -func (gce *GCEClient) getImagePath(img string, project string) (imagePath string) { +func (gce *GCEClient) getImagePath(img string) (imagePath string) { defaultImg := "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710" - // A full image path must match the regex format. If it doesn't, we'll assume it's just the image name and try to get it. - // If that doesn't work, we will fall back to a default base image. + // A full image path must match the regex format. If it doesn't, we will fall back to a default base image. matches := regexp.MustCompile("projects/(.+)/global/images/(family/)*(.+)").FindStringSubmatch(img) - if matches == nil { - // Only the image name was specified in config, so check if it exists in the project specified in config. - fullPath := fmt.Sprintf("projects/%s/global/images/%s", project, img) - if _, err := gce.service.Images.Get(project, img).Do(); err == nil { - return fullPath + if matches != nil { + // Check to see if the image exists in the given path. The presence of "family" in the path dictates which API call we need to make. + project, family, name := matches[1], matches[2], matches[3] + var err error + if family == "" { + _, err = gce.service.Images.Get(project, name).Do() + } else { + _, err = gce.service.Images.GetFromFamily(project, name).Do() } - // Otherwise, fall back to the base image. - glog.Infof("Could not find image at %s. Defaulting to %s.", fullPath, defaultImg) - return defaultImg - } - - // Check to see if the image exists in the given path. The presence of "family" in the path dictates which API call we need to make. - project, family, name := matches[1], matches[2], matches[3] - var err error - if family == "" { - _, err = gce.service.Images.Get(project, name).Do() - } else { - _, err = gce.service.Images.GetFromFamily(project, name).Do() - } - - if err == nil { - return img + if err == nil { + return img + } } // Otherwise, fall back to the base image. diff --git a/cluster-api/cloud/google/machinesetup/config_types.go b/cluster-api/cloud/google/machinesetup/config_types.go index 4482a3524..a89b0089c 100644 --- a/cluster-api/cloud/google/machinesetup/config_types.go +++ b/cluster-api/cloud/google/machinesetup/config_types.go @@ -28,10 +28,7 @@ type config struct { // map to the given Image and Metadata. Params []ConfigParams `json:"machineParams"` - // This can either be a full projects path to an image/family, - // or just the image name which is in the project. - // If it's an image in the project, this field may be - // identical to the OS field (see GCEClient.getImagePath()). + // The fully specified image path. Image string `json:"image"` Metadata Metadata `json:"metadata"` } From 9c5614b6c7580a1faf2e8c1fb1cce1bbab56fd50 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 5 Apr 2018 10:09:49 -0700 Subject: [PATCH 08/21] fixup 6bc01dd fix config_types.go --- .../cloud/google/machinesetup/config_types.go | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/cluster-api/cloud/google/machinesetup/config_types.go b/cluster-api/cloud/google/machinesetup/config_types.go index a89b0089c..353f38e99 100644 --- a/cluster-api/cloud/google/machinesetup/config_types.go +++ b/cluster-api/cloud/google/machinesetup/config_types.go @@ -1,18 +1,35 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package machinesetup import ( "fmt" - "github.com/ghodss/yaml" "io" "io/ioutil" + "os" + + "github.com/ghodss/yaml" clustercommon "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" "k8s.io/kube-deploy/cluster-api/util" - "os" ) type ConfigWatch struct { - Path string + path string } type ValidConfigs struct { @@ -47,11 +64,11 @@ func NewConfigWatch(path string) (*ConfigWatch, error) { if _, err := os.Stat(path); err != nil { return nil, err } - return &ConfigWatch{Path: path}, nil + return &ConfigWatch{path: path}, nil } func (cw *ConfigWatch) ValidConfigs() (*ValidConfigs, error) { - file, err := os.Open(cw.Path) + file, err := os.Open(cw.path) if err != nil { return nil, err } From 88ceb5dbce6758b66a75b09dd622ea22d833043c Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 5 Apr 2018 10:27:55 -0700 Subject: [PATCH 09/21] fixup 6bc01dd fix config_types_test.go --- .../google/machinesetup/config_types_test.go | 65 +++++++++++++++++-- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/cluster-api/cloud/google/machinesetup/config_types_test.go b/cluster-api/cloud/google/machinesetup/config_types_test.go index 0b6c56305..43ef86dfe 100644 --- a/cluster-api/cloud/google/machinesetup/config_types_test.go +++ b/cluster-api/cloud/google/machinesetup/config_types_test.go @@ -1,12 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package machinesetup import ( "io" - clustercommon "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" - clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" "reflect" "strings" "testing" + + clustercommon "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" + clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" ) func TestParseMachineSetupYaml(t *testing.T) { @@ -226,10 +243,30 @@ func TestMatchMachineSetupConfig(t *testing.T) { StartupScript: "Node startup script", }, } + multiRoleSetupConfig := config{ + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Multi-role startup script", + }, + } validConfigs := ValidConfigs{ configList: &configList{ - Items: []config{masterMachineSetupConfig, nodeMachineSetupConfig}, + Items: []config{masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig}, }, } @@ -269,12 +306,28 @@ func TestMatchMachineSetupConfig(t *testing.T) { expectedMatch: &nodeMachineSetupConfig, expectedErr: false, }, + { + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + expectedMatch: &multiRoleSetupConfig, + expectedErr: false, + }, { params: ConfigParams{ OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.4", + Kubelet: "1.9.4", ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.13.0", @@ -306,10 +359,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { if !reflect.DeepEqual(matched, table.expectedMatch) { t.Errorf("Matched machine setup config was incorrect, got: %+v,\n want %+v.", matched, table.expectedMatch) } - if table.expectedErr && err == nil { + if err == nil && table.expectedErr { t.Errorf("An error was not received as expected.") } - if !table.expectedErr && err != nil { + if err != nil && !table.expectedErr { t.Errorf("Got unexpected error: %s", err) } } From c79b5a0697d6f6abbcdf7f5445a3217c35047ef1 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 5 Apr 2018 10:29:07 -0700 Subject: [PATCH 10/21] fixup 9b9a8b9 fix config_types_test.go --- cluster-api/cloud/google/machinesetup/config_types_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-api/cloud/google/machinesetup/config_types_test.go b/cluster-api/cloud/google/machinesetup/config_types_test.go index 43ef86dfe..df64034c5 100644 --- a/cluster-api/cloud/google/machinesetup/config_types_test.go +++ b/cluster-api/cloud/google/machinesetup/config_types_test.go @@ -166,10 +166,10 @@ func TestGetYaml(t *testing.T) { for _, table := range testTables { yaml, err := table.validConfigs.GetYaml() - if table.expectedErr && err == nil { + if err == nil && table.expectedErr { t.Errorf("An error was not received as expected.") } - if !table.expectedErr && err != nil { + if err != nil && !table.expectedErr { t.Errorf("Got unexpected error: %s", err) } for _, expectedString := range table.expectedStrings { From 7e00e5b001c2c5df11012d35802bef18401bdd7a Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 5 Apr 2018 10:41:48 -0700 Subject: [PATCH 11/21] fixup 6bc01dd fix create flag and default value --- cluster-api/gcp-deployer/cmd/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-api/gcp-deployer/cmd/create.go b/cluster-api/gcp-deployer/cmd/create.go index d05e2e7cd..df5f63e5c 100644 --- a/cluster-api/gcp-deployer/cmd/create.go +++ b/cluster-api/gcp-deployer/cmd/create.go @@ -76,7 +76,7 @@ func RunCreate(co *CreateOptions) error { func init() { createCmd.Flags().StringVarP(&co.Cluster, "cluster", "c", "", "cluster yaml file") createCmd.Flags().StringVarP(&co.Machine, "machines", "m", "", "machine yaml file") - createCmd.Flags().StringVarP(&co.MachineSetup, "setup", "s", "", "machine setup configs yaml file") + createCmd.Flags().StringVarP(&co.MachineSetup, "machinesetup", "s", "machine_setup_configs.yaml", "machine setup configs yaml file") RootCmd.AddCommand(createCmd) } From 1cc6a4d775b25294b6771db0b686c1acc384f6e9 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 5 Apr 2018 10:43:14 -0700 Subject: [PATCH 12/21] fixup 6bc01dd fix spacing in configs file --- cluster-api/gcp-deployer/machine_setup_configs.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-api/gcp-deployer/machine_setup_configs.yaml b/cluster-api/gcp-deployer/machine_setup_configs.yaml index 1543c6ae0..7d00c7774 100644 --- a/cluster-api/gcp-deployer/machine_setup_configs.yaml +++ b/cluster-api/gcp-deployer/machine_setup_configs.yaml @@ -15,7 +15,7 @@ items: set -x ( - apt-get update + apt-get update apt-get install -y apt-transport-https prips apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys F76221572C52609D @@ -71,7 +71,7 @@ items: kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break sleep 1 done - echo done. + echo done. ) 2>&1 | tee /var/log/startup.log - machineParams: - os: ubuntu-1710-weave @@ -316,7 +316,7 @@ items: set -x ( - apt-get update + apt-get update apt-get install -y apt-transport-https prips apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys F76221572C52609D From 078e7e83ab0d71df73dce7fb0ba8db97a035ebf6 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 5 Apr 2018 10:45:07 -0700 Subject: [PATCH 13/21] fixup 6bc01dd add TODO --- cluster-api/cloud/google/machineactuator.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-api/cloud/google/machineactuator.go b/cluster-api/cloud/google/machineactuator.go index 27c774859..63f5d71af 100644 --- a/cluster-api/cloud/google/machineactuator.go +++ b/cluster-api/cloud/google/machineactuator.go @@ -111,6 +111,7 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa } } + // TODO: get rid of empty string check when we switch to the new bootstrapping method. var configWatch *machinesetup.ConfigWatch if configListPath != "" { configWatch, err = machinesetup.NewConfigWatch(configListPath) From 5c450a0d0a4c13591fb46f07d4c955e4c9502405 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 5 Apr 2018 14:35:34 -0700 Subject: [PATCH 14/21] fixup 6bc01dd fix env vars template bug --- cluster-api/cloud/google/metadata.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cluster-api/cloud/google/metadata.go b/cluster-api/cloud/google/metadata.go index 7cc9f938d..f9015ad1a 100644 --- a/cluster-api/cloud/google/metadata.go +++ b/cluster-api/cloud/google/metadata.go @@ -41,12 +41,12 @@ type metadataParams struct { func nodeMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.Machine, metadata *machinesetup.Metadata) (map[string]string, error) { params := metadataParams{ - Token: token, - Cluster: cluster, - Machine: machine, - Metadata: metadata, - PodCIDR: getSubnet(cluster.Spec.ClusterNetwork.Pods), - ServiceCIDR: getSubnet(cluster.Spec.ClusterNetwork.Services), + Token: token, + Cluster: cluster, + Machine: machine, + Metadata: metadata, + PodCIDR: getSubnet(cluster.Spec.ClusterNetwork.Pods), + ServiceCIDR: getSubnet(cluster.Spec.ClusterNetwork.Services), MasterEndpoint: getEndpoint(cluster.Status.APIEndpoints[0]), } @@ -62,12 +62,12 @@ func nodeMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.M func masterMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.Machine, project string, metadata *machinesetup.Metadata) (map[string]string, error) { params := metadataParams{ - Token: token, - Cluster: cluster, - Machine: machine, - Project: project, - Metadata: metadata, - PodCIDR: getSubnet(cluster.Spec.ClusterNetwork.Pods), + Token: token, + Cluster: cluster, + Machine: machine, + Project: project, + Metadata: metadata, + PodCIDR: getSubnet(cluster.Spec.ClusterNetwork.Pods), ServiceCIDR: getSubnet(cluster.Spec.ClusterNetwork.Services), } @@ -106,7 +106,7 @@ TOKEN={{ .Token }} PORT=443 MACHINE={{ .Machine.ObjectMeta.Name }} CONTROL_PLANE_VERSION={{ .Machine.Spec.Versions.ControlPlane }} -CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.DNSDomain }} +CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }} POD_CIDR={{ .PodCIDR }} SERVICE_CIDR={{ .ServiceCIDR }} @@ -124,7 +124,7 @@ KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} TOKEN={{ .Token }} MASTER={{ .MasterEndpoint }} MACHINE={{ .Machine.ObjectMeta.Name }} -CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.DNSDomain }} +CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }} POD_CIDR={{ .PodCIDR }} SERVICE_CIDER={{ .ServiceCIDR }} ` From e1d2b289770304cd920f7bdad99384dd5e735442 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Tue, 10 Apr 2018 11:09:02 -0700 Subject: [PATCH 15/21] fixup 6bc01dd roles must be exact match --- cluster-api/cloud/google/machinesetup/config_types.go | 3 +++ cluster-api/cloud/google/machinesetup/config_types_test.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cluster-api/cloud/google/machinesetup/config_types.go b/cluster-api/cloud/google/machinesetup/config_types.go index 353f38e99..3adb6d8ff 100644 --- a/cluster-api/cloud/google/machinesetup/config_types.go +++ b/cluster-api/cloud/google/machinesetup/config_types.go @@ -120,6 +120,9 @@ func (vc *ValidConfigs) matchMachineSetupConfig(params *ConfigParams) (*config, if params.OS != validParams.OS { continue } + if len(params.Roles) != len(validParams.Roles) { + continue + } foundRoles := true for _, role := range params.Roles { if !util.RoleContains(role, validParams.Roles) { diff --git a/cluster-api/cloud/google/machinesetup/config_types_test.go b/cluster-api/cloud/google/machinesetup/config_types_test.go index df64034c5..977d78d5d 100644 --- a/cluster-api/cloud/google/machinesetup/config_types_test.go +++ b/cluster-api/cloud/google/machinesetup/config_types_test.go @@ -319,8 +319,8 @@ func TestMatchMachineSetupConfig(t *testing.T) { }, }, }, - expectedMatch: &multiRoleSetupConfig, - expectedErr: false, + expectedMatch: nil, + expectedErr: true, }, { params: ConfigParams{ From 6eea28eb8e3ae9b506477194ea2c1d7e6d99970f Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Tue, 10 Apr 2018 11:14:22 -0700 Subject: [PATCH 16/21] fixup 6bc01dd move ARCH env var to scripts --- cluster-api/cloud/google/metadata.go | 1 - cluster-api/gcp-deployer/machine_setup_configs.yaml | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster-api/cloud/google/metadata.go b/cluster-api/cloud/google/metadata.go index f9015ad1a..0462e5cf3 100644 --- a/cluster-api/cloud/google/metadata.go +++ b/cluster-api/cloud/google/metadata.go @@ -101,7 +101,6 @@ const masterEnvironmentVars = ` KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} VERSION=v${KUBELET_VERSION} -ARCH=amd64 TOKEN={{ .Token }} PORT=443 MACHINE={{ .Machine.ObjectMeta.Name }} diff --git a/cluster-api/gcp-deployer/machine_setup_configs.yaml b/cluster-api/gcp-deployer/machine_setup_configs.yaml index 7d00c7774..bd256ba72 100644 --- a/cluster-api/gcp-deployer/machine_setup_configs.yaml +++ b/cluster-api/gcp-deployer/machine_setup_configs.yaml @@ -90,6 +90,7 @@ items: set -x ( + ARCH=amd64 curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - touch /etc/apt/sources.list.d/kubernetes.list sh -c 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list' @@ -201,6 +202,7 @@ items: set -x ( + ARCH=amd64 curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - touch /etc/apt/sources.list.d/kubernetes.list sh -c 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list' From 66a419f97f2cfafa65dc1a013a6b28c61d5813ac Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Tue, 10 Apr 2018 14:52:49 -0700 Subject: [PATCH 17/21] fixup 6bc01dd fix role matching check --- .../cloud/google/machinesetup/config_types.go | 25 ++++++++++--------- .../google/machinesetup/config_types_test.go | 16 ++++++++++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/cluster-api/cloud/google/machinesetup/config_types.go b/cluster-api/cloud/google/machinesetup/config_types.go index 3adb6d8ff..d01d654da 100644 --- a/cluster-api/cloud/google/machinesetup/config_types.go +++ b/cluster-api/cloud/google/machinesetup/config_types.go @@ -22,10 +22,11 @@ import ( "io/ioutil" "os" + "reflect" + "github.com/ghodss/yaml" clustercommon "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" - "k8s.io/kube-deploy/cluster-api/util" ) type ConfigWatch struct { @@ -120,17 +121,9 @@ func (vc *ValidConfigs) matchMachineSetupConfig(params *ConfigParams) (*config, if params.OS != validParams.OS { continue } - if len(params.Roles) != len(validParams.Roles) { - continue - } - foundRoles := true - for _, role := range params.Roles { - if !util.RoleContains(role, validParams.Roles) { - foundRoles = false - break - } - } - if !foundRoles { + validRoles := rolesToMap(validParams.Roles) + paramRoles := rolesToMap(params.Roles) + if !reflect.DeepEqual(paramRoles, validRoles) { continue } if params.Versions != validParams.Versions { @@ -141,3 +134,11 @@ func (vc *ValidConfigs) matchMachineSetupConfig(params *ConfigParams) (*config, } return nil, fmt.Errorf("could not find a matching machine setup config for params %+v", params) } + +func rolesToMap(roles []clustercommon.MachineRole) map[clustercommon.MachineRole]int { + rolesMap := map[clustercommon.MachineRole]int{} + for _, role := range roles { + rolesMap[role] = rolesMap[role] + 1 + } + return rolesMap +} diff --git a/cluster-api/cloud/google/machinesetup/config_types_test.go b/cluster-api/cloud/google/machinesetup/config_types_test.go index 977d78d5d..32cf417e7 100644 --- a/cluster-api/cloud/google/machinesetup/config_types_test.go +++ b/cluster-api/cloud/google/machinesetup/config_types_test.go @@ -352,6 +352,22 @@ func TestMatchMachineSetupConfig(t *testing.T) { expectedMatch: nil, expectedErr: true, }, + { + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + expectedMatch: nil, + expectedErr: true, + }, } for _, table := range testTables { From e829e94a983fc38fac639e7ff9f2682d6065f975 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Thu, 12 Apr 2018 14:49:11 -0700 Subject: [PATCH 18/21] fixup 6bc01dd rename gce-machine-controller flag --- cluster-api/cloud/google/cmd/gce-machine-controller/main.go | 2 +- cluster-api/cloud/google/config/configtemplate.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-api/cloud/google/cmd/gce-machine-controller/main.go b/cluster-api/cloud/google/cmd/gce-machine-controller/main.go index ecf10b4c9..1d3011307 100644 --- a/cluster-api/cloud/google/cmd/gce-machine-controller/main.go +++ b/cluster-api/cloud/google/cmd/gce-machine-controller/main.go @@ -32,7 +32,7 @@ import ( var ( kubeadmToken = pflag.String("token", "", "Kubeadm token to use to join new machines") - machineSetupConfigsPath = pflag.String("config", "", "path to machine setup configs file") + machineSetupConfigsPath = pflag.String("machinesetup", "", "path to machine setup configs file") ) func init() { diff --git a/cluster-api/cloud/google/config/configtemplate.go b/cluster-api/cloud/google/config/configtemplate.go index 6c024dfaf..81c600706 100644 --- a/cluster-api/cloud/google/config/configtemplate.go +++ b/cluster-api/cloud/google/config/configtemplate.go @@ -149,7 +149,7 @@ spec: args: - --kubeconfig=/etc/kubernetes/admin.conf - --token={{ .Token }} - - --config=/etc/machinesetup/machine_setup_configs.yaml + - --machinesetup=/etc/machinesetup/machine_setup_configs.yaml resources: requests: cpu: 100m From c124da105f049c76191def409da97397c50fc1cc Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Fri, 13 Apr 2018 11:29:42 -0700 Subject: [PATCH 19/21] fixup 6bc01dd add comments/documentation --- cluster-api/cloud/google/gceproviderconfig/types.go | 4 +++- .../cloud/google/gceproviderconfig/v1alpha1/types.go | 4 +++- cluster-api/cloud/google/machineactuator.go | 2 ++ cluster-api/cloud/google/machinesetup/config_types.go | 7 ++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cluster-api/cloud/google/gceproviderconfig/types.go b/cluster-api/cloud/google/gceproviderconfig/types.go index 376fc4da0..1ca0f76f6 100644 --- a/cluster-api/cloud/google/gceproviderconfig/types.go +++ b/cluster-api/cloud/google/gceproviderconfig/types.go @@ -27,5 +27,7 @@ type GCEProviderConfig struct { Project string `json:"project"` Zone string `json:"zone"` MachineType string `json:"machineType"` - OS string `json:"os"` + + // The name of the OS to be installed on the machine. + OS string `json:"os"` } diff --git a/cluster-api/cloud/google/gceproviderconfig/v1alpha1/types.go b/cluster-api/cloud/google/gceproviderconfig/v1alpha1/types.go index f0175404a..b81885cf4 100644 --- a/cluster-api/cloud/google/gceproviderconfig/v1alpha1/types.go +++ b/cluster-api/cloud/google/gceproviderconfig/v1alpha1/types.go @@ -27,5 +27,7 @@ type GCEProviderConfig struct { Project string `json:"project"` Zone string `json:"zone"` MachineType string `json:"machineType"` - OS string `json:"os"` + + // The name of the OS to be installed on the machine. + OS string `json:"os"` } diff --git a/cluster-api/cloud/google/machineactuator.go b/cluster-api/cloud/google/machineactuator.go index 63f5d71af..9d1a63976 100644 --- a/cluster-api/cloud/google/machineactuator.go +++ b/cluster-api/cloud/google/machineactuator.go @@ -57,6 +57,8 @@ const ( UIDLabelKey = "machine-crd-uid" BootstrapLabelKey = "boostrap" + // This file is a yaml that will be used to create the machine-setup configmap on the machine controller. + // It contains the supported machine configurations along with the startup scripts and OS image paths that correspond to each supported configuration. MachineSetupConfigsFilename = "machine_setup_configs.yaml" ) diff --git a/cluster-api/cloud/google/machinesetup/config_types.go b/cluster-api/cloud/google/machinesetup/config_types.go index d01d654da..44f79e12f 100644 --- a/cluster-api/cloud/google/machinesetup/config_types.go +++ b/cluster-api/cloud/google/machinesetup/config_types.go @@ -29,10 +29,12 @@ import ( clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" ) +// Config Watch holds the path to the machine setup configs yaml file. type ConfigWatch struct { path string } +// The valid machine setup configs parsed out of the machine setup configs yaml file held in ConfigWatch. type ValidConfigs struct { configList *configList } @@ -41,12 +43,15 @@ type configList struct { Items []config `json:"items"` } +// A single valid machine setup config that maps a machine's params to the corresponding image and metadata. type config struct { // A list of the valid combinations of ConfigParams that will // map to the given Image and Metadata. Params []ConfigParams `json:"machineParams"` - // The fully specified image path. + // The fully specified image path. e.g. + // projects/ubuntu-os-cloud/global/images/family/ubuntu-1604-lts + // projects/ubuntu-os-cloud/global/images/ubuntu-1604-xenial-v20180405 Image string `json:"image"` Metadata Metadata `json:"metadata"` } From c1c88ef59545addd34230a2ae4581657e0e77389 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Fri, 13 Apr 2018 15:42:30 -0700 Subject: [PATCH 20/21] fixup 6bc01dd return error if >1 config match --- .../cloud/google/machinesetup/config_types.go | 12 +- .../google/machinesetup/config_types_test.go | 160 +++++++++++------- 2 files changed, 109 insertions(+), 63 deletions(-) diff --git a/cluster-api/cloud/google/machinesetup/config_types.go b/cluster-api/cloud/google/machinesetup/config_types.go index 44f79e12f..3434d1314 100644 --- a/cluster-api/cloud/google/machinesetup/config_types.go +++ b/cluster-api/cloud/google/machinesetup/config_types.go @@ -121,6 +121,7 @@ func (vc *ValidConfigs) GetMetadata(params *ConfigParams) (Metadata, error) { } func (vc *ValidConfigs) matchMachineSetupConfig(params *ConfigParams) (*config, error) { + matchingConfigs := make([]config, 0) for _, conf := range vc.configList.Items { for _, validParams := range conf.Params { if params.OS != validParams.OS { @@ -134,10 +135,17 @@ func (vc *ValidConfigs) matchMachineSetupConfig(params *ConfigParams) (*config, if params.Versions != validParams.Versions { continue } - return &conf, nil + matchingConfigs = append(matchingConfigs, conf) } } - return nil, fmt.Errorf("could not find a matching machine setup config for params %+v", params) + + if len(matchingConfigs) == 1 { + return &matchingConfigs[0], nil + } else if len(matchingConfigs) == 0 { + return nil, fmt.Errorf("could not find a matching machine setup config for params %+v", params) + } else { + return nil, fmt.Errorf("found multiple matching machine setup configs for params %+v", params) + } } func rolesToMap(roles []clustercommon.MachineRole) map[clustercommon.MachineRole]int { diff --git a/cluster-api/cloud/google/machinesetup/config_types_test.go b/cluster-api/cloud/google/machinesetup/config_types_test.go index 32cf417e7..c1b226fc5 100644 --- a/cluster-api/cloud/google/machinesetup/config_types_test.go +++ b/cluster-api/cloud/google/machinesetup/config_types_test.go @@ -180,31 +180,38 @@ func TestGetYaml(t *testing.T) { } } +func validConfigs(configs ...config) ValidConfigs { + return ValidConfigs{ + configList: &configList{ + Items: configs, + }, + } +} + func TestMatchMachineSetupConfig(t *testing.T) { + dockerRuntimeInfo := clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + } + masterMachineSetupConfig := config{ Params: []ConfigParams{ { OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.3", - ControlPlane: "1.9.3", - ContainerRuntime: clusterv1.ContainerRuntimeInfo{ - Name: "docker", - Version: "1.12.0", - }, + Kubelet: "1.9.3", + ControlPlane: "1.9.3", + ContainerRuntime: dockerRuntimeInfo, }, }, { OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.4", - ControlPlane: "1.9.4", - ContainerRuntime: clusterv1.ContainerRuntimeInfo{ - Name: "docker", - Version: "1.12.0", - }, + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, }, }, }, @@ -219,22 +226,16 @@ func TestMatchMachineSetupConfig(t *testing.T) { OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.3", - ContainerRuntime: clusterv1.ContainerRuntimeInfo{ - Name: "docker", - Version: "1.12.0", - }, + Kubelet: "1.9.3", + ContainerRuntime: dockerRuntimeInfo, }, }, { OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.4", - ContainerRuntime: clusterv1.ContainerRuntimeInfo{ - Name: "docker", - Version: "1.12.0", - }, + Kubelet: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, }, }, }, @@ -249,12 +250,9 @@ func TestMatchMachineSetupConfig(t *testing.T) { OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.NodeRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.5", - ControlPlane: "1.9.5", - ContainerRuntime: clusterv1.ContainerRuntimeInfo{ - Name: "docker", - Version: "1.12.0", - }, + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: dockerRuntimeInfo, }, }, }, @@ -263,66 +261,96 @@ func TestMatchMachineSetupConfig(t *testing.T) { StartupScript: "Multi-role startup script", }, } - - validConfigs := ValidConfigs{ - configList: &configList{ - Items: []config{masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig}, + duplicateMasterMachineSetupConfig := config{ + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.3", + ControlPlane: "1.9.3", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Duplicate master startup script", }, } testTables := []struct { + validConfigs ValidConfigs params ConfigParams expectedMatch *config expectedErr bool }{ { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), params: ConfigParams{ OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.4", - ControlPlane: "1.9.4", - ContainerRuntime: clusterv1.ContainerRuntimeInfo{ - Name: "docker", - Version: "1.12.0", - }, + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, }, }, expectedMatch: &masterMachineSetupConfig, expectedErr: false, }, { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), params: ConfigParams{ OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.4", - ContainerRuntime: clusterv1.ContainerRuntimeInfo{ - Name: "docker", - Version: "1.12.0", - }, + Kubelet: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, }, }, expectedMatch: &nodeMachineSetupConfig, expectedErr: false, }, { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + expectedMatch: &multiRoleSetupConfig, + expectedErr: false, + }, + { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), params: ConfigParams{ OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.5", - ControlPlane: "1.9.5", - ContainerRuntime: clusterv1.ContainerRuntimeInfo{ - Name: "docker", - Version: "1.12.0", - }, + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: dockerRuntimeInfo, }, }, expectedMatch: nil, expectedErr: true, }, { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), params: ConfigParams{ OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, @@ -338,31 +366,41 @@ func TestMatchMachineSetupConfig(t *testing.T) { expectedErr: true, }, { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), params: ConfigParams{ OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.NodeRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.3", - ContainerRuntime: clusterv1.ContainerRuntimeInfo{ - Name: "docker", - Version: "1.12.0", - }, + Kubelet: "1.9.3", + ContainerRuntime: dockerRuntimeInfo, }, }, expectedMatch: nil, expectedErr: true, }, { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig), params: ConfigParams{ OS: "ubuntu-1710", Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.MasterRole}, Versions: clusterv1.MachineVersionInfo{ - Kubelet: "1.9.5", - ControlPlane: "1.9.5", - ContainerRuntime: clusterv1.ContainerRuntimeInfo{ - Name: "docker", - Version: "1.12.0", - }, + Kubelet: "1.9.5", + ControlPlane: "1.9.5", + ContainerRuntime: dockerRuntimeInfo, + }, + }, + expectedMatch: nil, + expectedErr: true, + }, + { + validConfigs: validConfigs(masterMachineSetupConfig, nodeMachineSetupConfig, multiRoleSetupConfig, duplicateMasterMachineSetupConfig), + params: ConfigParams{ + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: dockerRuntimeInfo, }, }, expectedMatch: nil, @@ -371,7 +409,7 @@ func TestMatchMachineSetupConfig(t *testing.T) { } for _, table := range testTables { - matched, err := validConfigs.matchMachineSetupConfig(&table.params) + matched, err := table.validConfigs.matchMachineSetupConfig(&table.params) if !reflect.DeepEqual(matched, table.expectedMatch) { t.Errorf("Matched machine setup config was incorrect, got: %+v,\n want %+v.", matched, table.expectedMatch) } From 1cfe66bd484fcbaf2b18d7242dcc0670ebb643f0 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Wed, 18 Apr 2018 10:00:33 -0700 Subject: [PATCH 21/21] fixup 9b9a8b9 add TODO for improving configmap visibility --- cluster-api/cloud/google/machineactuator.go | 1 + cluster-api/cloud/google/machinesetup/config_types.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cluster-api/cloud/google/machineactuator.go b/cluster-api/cloud/google/machineactuator.go index 9d1a63976..a591fb4d9 100644 --- a/cluster-api/cloud/google/machineactuator.go +++ b/cluster-api/cloud/google/machineactuator.go @@ -151,6 +151,7 @@ func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initia } // Create the configmap so the machine setup configs can be mounted into the node. + // TODO: create the configmap during bootstrapping instead of being buried in the machine actuator code. machineSetupConfigs, err := gce.configWatch.ValidConfigs() if err != nil { return err diff --git a/cluster-api/cloud/google/machinesetup/config_types.go b/cluster-api/cloud/google/machinesetup/config_types.go index 3434d1314..bc504b5cd 100644 --- a/cluster-api/cloud/google/machinesetup/config_types.go +++ b/cluster-api/cloud/google/machinesetup/config_types.go @@ -30,6 +30,7 @@ import ( ) // Config Watch holds the path to the machine setup configs yaml file. +// This works directly with a yaml file is used instead of a ConfigMap object so that we don't take a dependency on the API Server. type ConfigWatch struct { path string }