From 9b9a8b95ac9de51d4839161f85571ff8d03fb534 Mon Sep 17 00:00:00 2001 From: Katie Coronado Date: Wed, 28 Mar 2018 16:42:07 -0700 Subject: [PATCH] Create/mount configmap for machine setup configs --- .../cloud/google/config/configtemplate.go | 6 + cluster-api/cloud/google/machineactuator.go | 27 +++- .../cloud/google/machinesetup/config_types.go | 8 ++ .../google/machinesetup/config_types_test.go | 127 ++++++++++++++---- cluster-api/cloud/google/metadata.go | 6 +- cluster-api/gcp-deployer/cmd/add.go | 2 +- cluster-api/gcp-deployer/cmd/create.go | 13 +- cluster-api/gcp-deployer/cmd/delete.go | 2 +- cluster-api/gcp-deployer/deploy/deploy.go | 28 ++-- .../gcp-deployer/deploy/deploy_helper.go | 14 +- .../gcp-deployer/deploy/machinedeployer.go | 3 +- .../gcp-deployer/machines.yaml.template | 6 +- 12 files changed, 182 insertions(+), 60 deletions(-) diff --git a/cluster-api/cloud/google/config/configtemplate.go b/cluster-api/cloud/google/config/configtemplate.go index 82b412d41..6c024dfaf 100644 --- a/cluster-api/cloud/google/config/configtemplate.go +++ b/cluster-api/cloud/google/config/configtemplate.go @@ -139,6 +139,8 @@ spec: mountPath: /etc/credentials - name: sshkeys mountPath: /etc/sshkeys + - name: machine-setup + mountPath: /etc/machinesetup env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/credentials/service-account.json @@ -147,6 +149,7 @@ spec: args: - --kubeconfig=/etc/kubernetes/admin.conf - --token={{ .Token }} + - --config=/etc/machinesetup/machine_setup_configs.yaml resources: requests: cpu: 100m @@ -171,6 +174,9 @@ spec: - name: credentials secret: secretName: machine-controller-credential + - name: machine-setup + configMap: + name: machine-setup --- apiVersion: apps/v1beta1 kind: StatefulSet diff --git a/cluster-api/cloud/google/machineactuator.go b/cluster-api/cloud/google/machineactuator.go index b18bd02eb..9031d5088 100644 --- a/cluster-api/cloud/google/machineactuator.go +++ b/cluster-api/cloud/google/machineactuator.go @@ -37,6 +37,9 @@ import ( "regexp" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" gceconfig "k8s.io/kube-deploy/cluster-api/cloud/google/gceproviderconfig" gceconfigv1 "k8s.io/kube-deploy/cluster-api/cloud/google/gceproviderconfig/v1alpha1" "k8s.io/kube-deploy/cluster-api/cloud/google/machinesetup" @@ -53,6 +56,8 @@ const ( UIDLabelKey = "machine-crd-uid" BootstrapLabelKey = "boostrap" + + MachineSetupConfigsFilename = "machine_setup_configs.yaml" ) type SshCreds struct { @@ -128,7 +133,7 @@ func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterfa }, nil } -func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { +func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine, clientSet kubernetes.Clientset) error { if err := gce.CreateMachineControllerServiceAccount(cluster, initialMachines); err != nil { return err } @@ -142,6 +147,26 @@ func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initia return err } + // Create the configmap so the machine setup configs can be mounted into the node. + machineSetupConfigs, err := gce.configWatch.ValidConfigs() + if err != nil { + return err + } + yaml, err := machineSetupConfigs.GetYaml() + if err != nil { + return err + } + configMap := corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{Name: "machine-setup"}, + Data: map[string]string{ + MachineSetupConfigsFilename: yaml, + }, + } + configMaps := clientSet.CoreV1().ConfigMaps(corev1.NamespaceDefault) + if _, err := configMaps.Create(&configMap); err != nil { + return err + } + if err := CreateApiServerAndController(gce.kubeadmToken); err != nil { return err } diff --git a/cluster-api/cloud/google/machinesetup/config_types.go b/cluster-api/cloud/google/machinesetup/config_types.go index 422c5362e..4482a3524 100644 --- a/cluster-api/cloud/google/machinesetup/config_types.go +++ b/cluster-api/cloud/google/machinesetup/config_types.go @@ -76,6 +76,14 @@ func parseMachineSetupYaml(reader io.Reader) (*ValidConfigs, error) { return &ValidConfigs{configList}, nil } +func (vc *ValidConfigs) GetYaml() (string, error) { + bytes, err := yaml.Marshal(vc.configList) + if err != nil { + return "", err + } + return string(bytes), nil +} + func (vc *ValidConfigs) GetImage(params *ConfigParams) (string, error) { machineSetupConfig, err := vc.matchMachineSetupConfig(params) if err != nil { diff --git a/cluster-api/cloud/google/machinesetup/config_types_test.go b/cluster-api/cloud/google/machinesetup/config_types_test.go index 46fcacda7..0b6c56305 100644 --- a/cluster-api/cloud/google/machinesetup/config_types_test.go +++ b/cluster-api/cloud/google/machinesetup/config_types_test.go @@ -2,8 +2,8 @@ package machinesetup import ( "io" - "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" - "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" + clustercommon "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/common" + clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" "reflect" "strings" "testing" @@ -90,16 +90,89 @@ func TestParseMachineSetupYaml(t *testing.T) { } } +func TestGetYaml(t *testing.T) { + testTables := []struct { + validConfigs ValidConfigs + expectedStrings []string + expectedErr bool + }{ + { + validConfigs: ValidConfigs{ + configList: &configList{ + Items: []config{ + { + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ControlPlane: "1.9.4", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Master startup script", + }, + }, + { + Params: []ConfigParams{ + { + OS: "ubuntu-1710", + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ + Name: "docker", + Version: "1.12.0", + }, + }, + }, + }, + Image: "projects/ubuntu-os-cloud/global/images/family/ubuntu-1710", + Metadata: Metadata{ + StartupScript: "Node startup script", + }, + }, + }, + }, + }, + expectedStrings: []string{"startupScript: Master startup script", "startupScript: Node startup script"}, + expectedErr: false, + }, + } + + for _, table := range testTables { + yaml, err := table.validConfigs.GetYaml() + if table.expectedErr && err == nil { + t.Errorf("An error was not received as expected.") + } + if !table.expectedErr && err != nil { + t.Errorf("Got unexpected error: %s", err) + } + for _, expectedString := range table.expectedStrings { + if !strings.Contains(yaml, expectedString) { + t.Errorf("Yaml did not contain expected string, got:\n%s\nwant:\n%s", yaml, expectedString) + } + } + } +} + func TestMatchMachineSetupConfig(t *testing.T) { masterMachineSetupConfig := config{ Params: []ConfigParams{ { OS: "ubuntu-1710", - Roles: []common.MachineRole{common.MasterRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.3", ControlPlane: "1.9.3", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -107,11 +180,11 @@ func TestMatchMachineSetupConfig(t *testing.T) { }, { OS: "ubuntu-1710", - Roles: []common.MachineRole{common.MasterRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.4", ControlPlane: "1.9.4", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -127,10 +200,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { Params: []ConfigParams{ { OS: "ubuntu-1710", - Roles: []common.MachineRole{common.NodeRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.3", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -138,10 +211,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { }, { OS: "ubuntu-1710", - Roles: []common.MachineRole{common.NodeRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.4", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -168,11 +241,11 @@ func TestMatchMachineSetupConfig(t *testing.T) { { params: ConfigParams{ OS: "ubuntu-1710", - Roles: []common.MachineRole{common.MasterRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.MasterRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.4", ControlPlane: "1.9.4", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -184,10 +257,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { { params: ConfigParams{ OS: "ubuntu-1710", - Roles: []common.MachineRole{common.NodeRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.4", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, @@ -199,10 +272,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { { params: ConfigParams{ OS: "ubuntu-1710", - Roles: []common.MachineRole{common.NodeRole}, - Versions: v1alpha1.MachineVersionInfo{ - Kubelet: "1.9.4", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + Roles: []clustercommon.MachineRole{clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ + Kubelet: "1.9.4", + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.13.0", }, @@ -214,10 +287,10 @@ func TestMatchMachineSetupConfig(t *testing.T) { { params: ConfigParams{ OS: "ubuntu-1710", - Roles: []common.MachineRole{common.MasterRole, common.NodeRole}, - Versions: v1alpha1.MachineVersionInfo{ + Roles: []clustercommon.MachineRole{clustercommon.MasterRole, clustercommon.NodeRole}, + Versions: clusterv1.MachineVersionInfo{ Kubelet: "1.9.3", - ContainerRuntime: v1alpha1.ContainerRuntimeInfo{ + ContainerRuntime: clusterv1.ContainerRuntimeInfo{ Name: "docker", Version: "1.12.0", }, diff --git a/cluster-api/cloud/google/metadata.go b/cluster-api/cloud/google/metadata.go index 42e23f631..972bcf32a 100644 --- a/cluster-api/cloud/google/metadata.go +++ b/cluster-api/cloud/google/metadata.go @@ -87,15 +87,15 @@ const masterEnvironmentVars = ` #!/bin/bash KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }} -export VERSION=v${KUBELET_VERSION} -export ARCH=amd64 +VERSION=v${KUBELET_VERSION} +ARCH=amd64 TOKEN={{ .Token }} PORT=443 MACHINE={{ .Machine.ObjectMeta.Name }} CONTROL_PLANE_VERSION={{ .Machine.Spec.Versions.ControlPlane }} CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.DNSDomain }} POD_CIDR={{ .PodCIDR }} -SERVICE_CIDER={{ .ServiceCIDR }} +SERVICE_CIDR={{ .ServiceCIDR }} # Environment variables for GCE cloud config PROJECT={{ .Project }} diff --git a/cluster-api/gcp-deployer/cmd/add.go b/cluster-api/gcp-deployer/cmd/add.go index df45341e7..a7a1ef2eb 100644 --- a/cluster-api/gcp-deployer/cmd/add.go +++ b/cluster-api/gcp-deployer/cmd/add.go @@ -52,7 +52,7 @@ func RunAdd(ao *AddOptions) error { return err } - d := deploy.NewDeployer(provider, kubeConfig) + d := deploy.NewDeployer(provider, kubeConfig, "") return d.AddNodes(machines) } diff --git a/cluster-api/gcp-deployer/cmd/create.go b/cluster-api/gcp-deployer/cmd/create.go index 63bbe9c05..d05e2e7cd 100644 --- a/cluster-api/gcp-deployer/cmd/create.go +++ b/cluster-api/gcp-deployer/cmd/create.go @@ -25,8 +25,9 @@ import ( ) type CreateOptions struct { - Cluster string - Machine string + Cluster string + Machine string + MachineSetup string } var co = &CreateOptions{} @@ -46,6 +47,11 @@ var createCmd = &cobra.Command{ cmd.Help() os.Exit(1) } + if co.MachineSetup == "" { + glog.Error("Please provide yaml file for machine setup configs.") + cmd.Help() + os.Exit(1) + } if err := RunCreate(co); err != nil { glog.Exit(err) } @@ -63,13 +69,14 @@ func RunCreate(co *CreateOptions) error { return err } - d := deploy.NewDeployer(provider, kubeConfig) + d := deploy.NewDeployer(provider, kubeConfig, co.MachineSetup) return d.CreateCluster(cluster, machines) } func init() { createCmd.Flags().StringVarP(&co.Cluster, "cluster", "c", "", "cluster yaml file") createCmd.Flags().StringVarP(&co.Machine, "machines", "m", "", "machine yaml file") + createCmd.Flags().StringVarP(&co.MachineSetup, "setup", "s", "", "machine setup configs yaml file") RootCmd.AddCommand(createCmd) } diff --git a/cluster-api/gcp-deployer/cmd/delete.go b/cluster-api/gcp-deployer/cmd/delete.go index ec991ea5a..5d375ff8b 100644 --- a/cluster-api/gcp-deployer/cmd/delete.go +++ b/cluster-api/gcp-deployer/cmd/delete.go @@ -34,7 +34,7 @@ var deleteCmd = &cobra.Command{ } func RunDelete() error { - d := deploy.NewDeployer(provider, kubeConfig) + d := deploy.NewDeployer(provider, kubeConfig, "") return d.DeleteCluster() } diff --git a/cluster-api/gcp-deployer/deploy/deploy.go b/cluster-api/gcp-deployer/deploy/deploy.go index adb2ab331..efd2c90ca 100644 --- a/cluster-api/gcp-deployer/deploy/deploy.go +++ b/cluster-api/gcp-deployer/deploy/deploy.go @@ -22,6 +22,7 @@ import ( "github.com/golang/glog" + "k8s.io/client-go/kubernetes" "k8s.io/kube-deploy/cluster-api/cloud/google" clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" "k8s.io/kube-deploy/cluster-api/pkg/client/clientset_generated/clientset" @@ -31,37 +32,38 @@ import ( ) type deployer struct { - token string - configPath string - machineDeployer machineDeployer - client v1alpha1.ClusterV1alpha1Interface - clientSet clientset.Interface + token string + configPath string + machineDeployer machineDeployer + client v1alpha1.ClusterV1alpha1Interface + clientSet clientset.Interface + kubernetesClientSet kubernetes.Clientset } // NewDeployer returns a cloud provider specific deployer and // sets kubeconfig path for the cluster to be deployed -func NewDeployer(provider string, configPath string) *deployer { +func NewDeployer(provider string, kubeConfigPath string, machineSetupConfigPath string) *deployer { token := util.RandomToken() - if configPath == "" { - configPath = os.Getenv("KUBECONFIG") - if configPath == "" { - configPath = apiutil.GetDefaultKubeConfigPath() + if kubeConfigPath == "" { + kubeConfigPath = os.Getenv("KUBECONFIG") + if kubeConfigPath == "" { + kubeConfigPath = apiutil.GetDefaultKubeConfigPath() } } else { // This is needed for kubectl commands run later to create secret in function // CreateMachineControllerServiceAccount - if err := os.Setenv("KUBECONFIG", configPath); err != nil { + if err := os.Setenv("KUBECONFIG", kubeConfigPath); err != nil { glog.Exit(fmt.Sprintf("Failed to set Kubeconfig path err %v\n", err)) } } - ma, err := google.NewMachineActuator(token, nil) + ma, err := google.NewMachineActuator(token, nil, machineSetupConfigPath) if err != nil { glog.Exit(err) } return &deployer{ token: token, machineDeployer: ma, - configPath: configPath, + configPath: kubeConfigPath, } } diff --git a/cluster-api/gcp-deployer/deploy/deploy_helper.go b/cluster-api/gcp-deployer/deploy/deploy_helper.go index 4366d1245..8adcd457e 100644 --- a/cluster-api/gcp-deployer/deploy/deploy_helper.go +++ b/cluster-api/gcp-deployer/deploy/deploy_helper.go @@ -88,7 +88,7 @@ func (d *deployer) createCluster(c *clusterv1.Cluster, machines []*clusterv1.Mac } glog.Info("Deploying the addon apiserver and controller manager...") - if err := d.machineDeployer.CreateMachineController(c, machines); err != nil { + if err := d.machineDeployer.CreateMachineController(c, machines, d.kubernetesClientSet); err != nil { return fmt.Errorf("can't create machine controller: %v", err) } @@ -232,8 +232,13 @@ func (d *deployer) initApiClient() error { if err != nil { return err } + kubernetesClientSet, err := util.NewKubernetesClient(d.configPath) + if err != nil { + return err + } d.clientSet = c d.client = c.ClusterV1alpha1() + d.kubernetesClientSet = *kubernetesClientSet return nil } @@ -278,14 +283,9 @@ func (d *deployer) waitForApiserver(master string) error { // Make sure the default service account in kube-system namespace exists. func (d *deployer) waitForServiceAccount() error { - client, err := util.NewKubernetesClient(d.configPath) - if err != nil { - return err - } - waitErr := util.Retry(func() (bool, error) { glog.Info("Waiting for the service account to exist...") - _, err = client.CoreV1().ServiceAccounts(ServiceAccountNs).Get(ServiceAccountName, metav1.GetOptions{}) + _, err := d.kubernetesClientSet.CoreV1().ServiceAccounts(ServiceAccountNs).Get(ServiceAccountName, metav1.GetOptions{}) return (err == nil), nil }, 5) diff --git a/cluster-api/gcp-deployer/deploy/machinedeployer.go b/cluster-api/gcp-deployer/deploy/machinedeployer.go index 5e32f3f4a..649c4522f 100644 --- a/cluster-api/gcp-deployer/deploy/machinedeployer.go +++ b/cluster-api/gcp-deployer/deploy/machinedeployer.go @@ -1,6 +1,7 @@ package deploy import ( + "k8s.io/client-go/kubernetes" clusterv1 "k8s.io/kube-deploy/cluster-api/pkg/apis/cluster/v1alpha1" "k8s.io/kube-deploy/cluster-api/pkg/controller/machine" ) @@ -16,6 +17,6 @@ type machineDeployer interface { // are provided in case the function wants to refer to them (and their // ProviderConfigs) to know how to configure the machine controller. // Not idempotent. - CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error + CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine, clientSet kubernetes.Clientset) error PostDelete(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error } diff --git a/cluster-api/gcp-deployer/machines.yaml.template b/cluster-api/gcp-deployer/machines.yaml.template index 40ffe914b..b95699df7 100644 --- a/cluster-api/gcp-deployer/machines.yaml.template +++ b/cluster-api/gcp-deployer/machines.yaml.template @@ -15,8 +15,8 @@ items: machineType: "n1-standard-2" os: "ubuntu-1604-lts" versions: - kubelet: 1.8.3 - controlPlane: 1.8.3 + kubelet: 1.9.4 + controlPlane: 1.9.4 containerRuntime: name: docker version: 1.12.0 @@ -38,7 +38,7 @@ items: machineType: "n1-standard-1" os: "ubuntu-1604-lts" versions: - kubelet: 1.8.3 + kubelet: 1.9.4 containerRuntime: name: docker version: 1.12.0