From 62e051616cd4a4275ce83d2785ef3485adc384d2 Mon Sep 17 00:00:00 2001 From: Meaghan Kjelland Date: Tue, 8 May 2018 09:29:48 -0700 Subject: [PATCH 1/7] Issue #81: - Create service accounts on cluster creation for master/worker nodes - Add default storage class - Start kubelet with gce cloud provider configuration --- cloud/google/machineactuator.go | 50 ++++++--- cloud/google/metadata.go | 9 +- cloud/google/serviceaccount.go | 137 ++++++++++++++++++------ gcp-deployer/README.md | 4 + gcp-deployer/deploy/deploy_helper.go | 6 ++ gcp-deployer/machine_setup_configs.yaml | 26 +++++ pkg/controller/machine/actuator.go | 4 + 7 files changed, 187 insertions(+), 49 deletions(-) diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go index 90a4882fd5b8..11d2f294b252 100644 --- a/cloud/google/machineactuator.go +++ b/cloud/google/machineactuator.go @@ -185,6 +185,20 @@ func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initia return nil } +func (gce *GCEClient) ProvisionClusterDependencies(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { + err := gce.CreateMasterNodeServiceAccount(cluster, initialMachines) + if err != nil { + return err + } + + err = gce.CreateWorkerNodeServiceAccount(cluster, initialMachines) + if err != nil { + return err + } + + return nil +} + func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Machine) error { config, err := gce.providerconfig(machine.Spec.ProviderConfig) if err != nil { @@ -235,7 +249,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach return errors.New("invalid cluster state: cannot create a Kubernetes node without an API endpoint") } var err error - metadata, err = nodeMetadata(gce.kubeadmToken, cluster, machine, &machineSetupMetadata) + metadata, err = nodeMetadata(gce.kubeadmToken, cluster, machine, config.Project, &machineSetupMetadata) if err != nil { return err } @@ -266,18 +280,6 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach labels[BootstrapLabelKey] = "true" } - // The service account is needed for the Kubernetes GCE cloud provider code. It is needed on the master VM. - serviceAccounts := []*compute.ServiceAccount{nil} - if util.IsMaster(machine) { - serviceAccounts = append(serviceAccounts, - &compute.ServiceAccount{ - Email: "default", - Scopes: []string{ - "https://www.googleapis.com/auth/cloud-platform", - }, - }) - } - op, err := gce.computeService.InstancesInsert(project, zone, &compute.Instance{ Name: name, MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, config.MachineType), @@ -311,8 +313,15 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach "https-server", fmt.Sprintf("%s-worker", cluster.Name)}, }, - Labels: labels, - ServiceAccounts: serviceAccounts, + Labels: labels, + ServiceAccounts: []*compute.ServiceAccount{ + { + Email: gce.GetDefaultServiceAccountForMachine(cluster, machine), + Scopes: []string{ + compute.CloudPlatformScope, + }, + }, + }, }) if err == nil { @@ -391,7 +400,16 @@ func (gce *GCEClient) Delete(machine *clusterv1.Machine) error { } func (gce *GCEClient) PostDelete(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error { - return gce.DeleteMachineControllerServiceAccount(cluster, machines) + if err := gce.DeleteMasterNodeServiceAccount(cluster, machines); err != nil { + return fmt.Errorf("error deleting master node service account: %v", err) + } + if err := gce.DeleteWorkerNodeServiceAccount(cluster, machines); err != nil { + return fmt.Errorf("error deleting worker node service account: %v", err) + } + if err := gce.DeleteMachineControllerServiceAccount(cluster, machines); err != nil { + return fmt.Errorf("error deleting machine controller service account: %v", err) + } + return nil } func (gce *GCEClient) Update(cluster *clusterv1.Cluster, goalMachine *clusterv1.Machine) error { diff --git a/cloud/google/metadata.go b/cloud/google/metadata.go index ae25c80e9e46..af237d3260e7 100644 --- a/cloud/google/metadata.go +++ b/cloud/google/metadata.go @@ -39,11 +39,12 @@ type metadataParams struct { MasterEndpoint string } -func nodeMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.Machine, metadata *machinesetup.Metadata) (map[string]string, error) { +func nodeMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.Machine, project string, metadata *machinesetup.Metadata) (map[string]string, error) { params := metadataParams{ Token: token, Cluster: cluster, Machine: machine, + Project: project, Metadata: metadata, PodCIDR: getSubnet(cluster.Spec.ClusterNetwork.Pods), ServiceCIDR: getSubnet(cluster.Spec.ClusterNetwork.Services), @@ -124,4 +125,10 @@ MACHINE={{ .Machine.ObjectMeta.Name }} CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }} POD_CIDR={{ .PodCIDR }} SERVICE_CIDER={{ .ServiceCIDR }} +# Environment variables for GCE cloud config +PROJECT={{ .Project }} +NETWORK=default +SUBNETWORK=kubernetes +CLUSTER_NAME={{ .Cluster.Name }} +NODE_TAG="$CLUSTER_NAME-worker" ` diff --git a/cloud/google/serviceaccount.go b/cloud/google/serviceaccount.go index 6844194ad68a..dc6aecc78380 100644 --- a/cloud/google/serviceaccount.go +++ b/cloud/google/serviceaccount.go @@ -1,12 +1,9 @@ /* Copyright 2017 The Kubernetes Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,68 +23,142 @@ import ( ) const ( - ServiceAccountPrefix = "k8s-machine-controller-" - ServiceAccount = "service-account" - MachineControllerSecret = "machine-controller-credential" + MasterNodeServiceAccountPrefix = "k8s-master" + WorkerNodeServiceAccountPrefix = "k8s-worker" + MachineControllerServiceAccountPrefix = "k8s-machine-controller" + MachineControllerSecret = "machine-controller-credential" + ClusterAnnotationPrefix = "service-account-" +) + +var ( + MasterNodeRoles = []string{ + "compute.instanceAdmin", + "compute.networkAdmin", + "compute.securityAdmin", + "compute.viewer", + "iam.serviceAccountUser", + "storage.admin", + "storage.objectViewer", + } + WorkerNodeRoles = []string{ + "compute.instanceAdmin", + "compute.networkAdmin", + "compute.securityAdmin", + } + MachineControllerRoles = []string{ + "compute.instanceAdmin.v1", + "iam.serviceAccountActor", + } ) +// Returns the email address of the service account that should be used +// as the default service account for this machine +func (gce *GCEClient) GetDefaultServiceAccountForMachine(cluster *clusterv1.Cluster, machine *clusterv1.Machine) string { + if util.IsMaster(machine) { + return cluster.ObjectMeta.Annotations[ClusterAnnotationPrefix+MasterNodeServiceAccountPrefix] + } else { + return cluster.ObjectMeta.Annotations[ClusterAnnotationPrefix+WorkerNodeServiceAccountPrefix] + } +} + +// Creates a GCP service account for the master node, granted permissions +// that allow the control plane to provision disks and networking resources +func (gce *GCEClient) CreateMasterNodeServiceAccount(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { + _, _, err := gce.createServiceAccount(MasterNodeServiceAccountPrefix, MasterNodeRoles, cluster, initialMachines) + + return err +} + +// Creates a GCP service account for the worker node +func (gce *GCEClient) CreateWorkerNodeServiceAccount(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { + _, _, err := gce.createServiceAccount(WorkerNodeServiceAccountPrefix, WorkerNodeRoles, cluster, initialMachines) + + return err +} + // Creates a GCP service account for the machine controller, granted the // permissions to manage compute instances, and stores its credentials as a // Kubernetes secret. func (gce *GCEClient) CreateMachineControllerServiceAccount(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { + accountId, project, err := gce.createServiceAccount(MachineControllerServiceAccountPrefix, MachineControllerRoles, cluster, initialMachines) + if err != nil { + return err + } + + email := fmt.Sprintf("%s@%s.iam.gserviceaccount.com", accountId, project) + + localFile := accountId + "-key.json" + err = run("gcloud", "--project", project, "iam", "service-accounts", "keys", "create", localFile, "--iam-account", email) + if err != nil { + return fmt.Errorf("couldn't create service account key: %v", err) + } + + err = run("kubectl", "create", "secret", "generic", MachineControllerSecret, "--from-file=service-account.json="+localFile) + if err != nil { + return fmt.Errorf("couldn't import service account key as credential: %v", err) + } + if err := run("rm", localFile); err != nil { + glog.Error(err) + } + + return nil +} + +func (gce *GCEClient) createServiceAccount(serviceAccountPrefix string, roles []string, cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) (string, string, error) { if len(initialMachines) == 0 { - return fmt.Errorf("machine count is zero, cannot create service a/c") + return "", "", fmt.Errorf("machine count is zero, cannot create service a/c") } // TODO: use real go bindings // Figure out what projects the service account needs permission to. projects, err := gce.getProjects(initialMachines) if err != nil { - return err + return "", "", err } // The service account needs to be created in a single project, so just // use the first one, but grant permission to all projects in the list. project := projects[0] - accountId := ServiceAccountPrefix + util.RandomString(5) + accountId := serviceAccountPrefix + "-" + util.RandomString(5) - err = run("gcloud", "--project", project, "iam", "service-accounts", "create", "--display-name=k8s machines controller", accountId) + err = run("gcloud", "--project", project, "iam", "service-accounts", "create", "--display-name="+serviceAccountPrefix+" service account", accountId) if err != nil { - return fmt.Errorf("couldn't create service account: %v", err) + return "", "", fmt.Errorf("couldn't create service account: %v", err) } email := fmt.Sprintf("%s@%s.iam.gserviceaccount.com", accountId, project) - localFile := accountId + "-key.json" for _, project := range projects { - err = run("gcloud", "projects", "add-iam-policy-binding", project, "--member=serviceAccount:"+email, "--role=roles/compute.instanceAdmin.v1") - if err != nil { - return fmt.Errorf("couldn't grant permissions to service account: %v", err) + for _, role := range roles { + err = run("gcloud", "projects", "add-iam-policy-binding", project, "--member=serviceAccount:"+email, "--role=roles/"+role) + if err != nil { + return "", "", fmt.Errorf("couldn't grant permissions to service account: %v", err) + } } } - err = run("gcloud", "--project", project, "iam", "service-accounts", "keys", "create", localFile, "--iam-account", email) - if err != nil { - return fmt.Errorf("couldn't create service account key: %v", err) - } - - err = run("kubectl", "create", "secret", "generic", MachineControllerSecret, "--from-file=service-account.json="+localFile) - if err != nil { - return fmt.Errorf("couldn't import service account key as credential: %v", err) - } - if err := run("rm", localFile); err != nil { - glog.Error(err) - } - if cluster.ObjectMeta.Annotations == nil { cluster.ObjectMeta.Annotations = make(map[string]string) } - cluster.ObjectMeta.Annotations[ServiceAccount] = email - return nil + cluster.ObjectMeta.Annotations[ClusterAnnotationPrefix+serviceAccountPrefix] = email + + return accountId, project, nil +} + +func (gce *GCEClient) DeleteMasterNodeServiceAccount(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error { + return gce.deleteServiceAccount(MasterNodeServiceAccountPrefix, MasterNodeRoles, cluster, machines) +} + +func (gce *GCEClient) DeleteWorkerNodeServiceAccount(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error { + return gce.deleteServiceAccount(WorkerNodeServiceAccountPrefix, WorkerNodeRoles, cluster, machines) } func (gce *GCEClient) DeleteMachineControllerServiceAccount(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error { + return gce.deleteServiceAccount(MachineControllerServiceAccountPrefix, MachineControllerRoles, cluster, machines) +} + +func (gce *GCEClient) deleteServiceAccount(serviceAccountPrefix string, roles []string, cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error { if len(machines) == 0 { glog.Info("machine count is zero, cannot determine project for service a/c deletion") return nil @@ -100,7 +171,7 @@ func (gce *GCEClient) DeleteMachineControllerServiceAccount(cluster *clusterv1.C project := projects[0] var email string if cluster.ObjectMeta.Annotations != nil { - email = cluster.ObjectMeta.Annotations[ServiceAccount] + email = cluster.ObjectMeta.Annotations[ClusterAnnotationPrefix+serviceAccountPrefix] } if email == "" { @@ -108,7 +179,9 @@ func (gce *GCEClient) DeleteMachineControllerServiceAccount(cluster *clusterv1.C return nil } - err = run("gcloud", "projects", "remove-iam-policy-binding", project, "--member=serviceAccount:"+email, "--role=roles/compute.instanceAdmin.v1") + for _, role := range roles { + err = run("gcloud", "projects", "remove-iam-policy-binding", project, "--member=serviceAccount:"+email, "--role=roles/"+role) + } if err != nil { return fmt.Errorf("couldn't remove permissions to service account: %v", err) diff --git a/gcp-deployer/README.md b/gcp-deployer/README.md index 0e4f2013b409..4f46f1149eee 100644 --- a/gcp-deployer/README.md +++ b/gcp-deployer/README.md @@ -74,6 +74,10 @@ to find the broken node (using the dry run flag) and fix it. ### Deleting a cluster +***NOTE***: Before deleting your cluster, it is recommended that you delete any Kubernetes +objects which have created resources external to the cluster, like services with type LoadBalancer, +some types of persistent volume claims, and ingress resources. + To delete your cluster run `./gcp-deployer delete` diff --git a/gcp-deployer/deploy/deploy_helper.go b/gcp-deployer/deploy/deploy_helper.go index 337a1e366380..5ae0e08f9608 100644 --- a/gcp-deployer/deploy/deploy_helper.go +++ b/gcp-deployer/deploy/deploy_helper.go @@ -55,6 +55,12 @@ func (d *deployer) createCluster(c *clusterv1.Cluster, machines []*clusterv1.Mac master.Name = master.GetGenerateName() + c.GetName() } + glog.Infof("Starting cluster dependency creation %s", c.GetName()) + + if err := d.machineDeployer.ProvisionClusterDependencies(c, machines); err != nil { + return err + } + glog.Infof("Starting cluster creation %s", c.GetName()) glog.Infof("Starting master creation %s", master.GetName()) diff --git a/gcp-deployer/machine_setup_configs.yaml b/gcp-deployer/machine_setup_configs.yaml index 92e5d05073bd..d24cb235cb2b 100644 --- a/gcp-deployer/machine_setup_configs.yaml +++ b/gcp-deployer/machine_setup_configs.yaml @@ -70,6 +70,7 @@ items: Environment="KUBELET_NETWORK_ARGS=--network-plugin=kubenet" Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}" EOF + sed -i "2iEnvironment=\"KUBELET_EXTRA_ARGS=--cloud-provider=gce --cloud-config=/etc/kubernetes/cloud-config\"" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf systemctl daemon-reload systemctl restart kubelet.service PRIVATEIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip"` @@ -112,6 +113,21 @@ items: modprobe br_netfilter kubeadm init --config /etc/kubernetes/kubeadm_config.yaml + + # add default storage class + cat > /etc/kubernetes/default-storage-class.yaml < /etc/kubernetes/cloud-config < Date: Tue, 8 May 2018 09:33:50 -0700 Subject: [PATCH 2/7] Issue #81 Adding instructions for deploying ingress controller --- .../examples/gce-ingress-controller/README.md | 12 ++ .../ingress-controller.yaml | 165 ++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 docs/examples/gce-ingress-controller/README.md create mode 100644 docs/examples/gce-ingress-controller/ingress-controller.yaml diff --git a/docs/examples/gce-ingress-controller/README.md b/docs/examples/gce-ingress-controller/README.md new file mode 100644 index 000000000000..f4b9f38c8e6a --- /dev/null +++ b/docs/examples/gce-ingress-controller/README.md @@ -0,0 +1,12 @@ +# Deploy GCE Ingress Controller + +Instructions for how to deploy an ingress controller in a cluster +that was deployed by gcp-deployer + +1. Replace `` and `` in +`ingress-controller.yml`. +1. Run `kubectl create -f ingress-controller.yml`. This will create +Kubernetes service account with the correct permissions in the cluster, +a default backend for the ingress controller, and the glbc app + +Now you will be able to create ingress objects. \ No newline at end of file diff --git a/docs/examples/gce-ingress-controller/ingress-controller.yaml b/docs/examples/gce-ingress-controller/ingress-controller.yaml new file mode 100644 index 000000000000..205845a4cfbf --- /dev/null +++ b/docs/examples/gce-ingress-controller/ingress-controller.yaml @@ -0,0 +1,165 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: glbc + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/ +kind: ClusterRole +metadata: + name: system:controller:glbc +rules: +- apiGroups: [""] + resources: ["secrets", "endpoints", "services", "pods", "nodes", "namespaces", "configmaps", "events"] + verbs: ["describe", "get", "list", "watch", "update", "create", "patch"] +- apiGroups: ["extensions"] + resources: ["ingresses"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["extensions"] + resources: ["ingresses/status"] + verbs: ["update"] +--- +apiVersion: rbac.authorization.k8s.io/ +kind: ClusterRoleBinding +metadata: + name: system:controller:glbc +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:glbc +subjects: +- kind: ServiceAccount + name: glbc + namespace: kube-system +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: l7-default-backend + namespace: kube-system + labels: + k8s-app: glbc + kubernetes.io/name: "GLBC" + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: glbc + template: + metadata: + labels: + k8s-app: glbc + name: glbc + spec: + containers: + - name: default-http-backend + # Any image is permissible as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.4 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ingress-controller-config + namespace: kube-system +data: + gce.conf: | + [global] + project-id = + node-tags = +--- +apiVersion: v1 +kind: Service +metadata: + # This must match the --default-backend-service argument of the l7 lb + # controller and is required because GCE mandates a default backend. + name: default-http-backend + namespace: kube-system + labels: + k8s-app: glbc + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "GLBCDefaultBackend" +spec: + # The default backend must be of type NodePort. + type: NodePort + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + k8s-app: glbc +--- +apiVersion: v1 +kind: ReplicationController +metadata: + namespace: kube-system + name: l7-lb-controller + labels: + k8s-app: glbc + version: v1.1.1 +spec: + # There should never be more than 1 controller alive simultaneously. + replicas: 1 + selector: + k8s-app: glbc + version: v1.1.1 + template: + metadata: + labels: + k8s-app: glbc + version: v1.1.1 + name: glbc + spec: + serviceAccountName: glbc + terminationGracePeriodSeconds: 600 + containers: + - image: k8s.gcr.io/ingress-gce-glbc-amd64:v1.1.1 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + name: l7-lb-controller + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 50Mi + args: + - --default-backend-service=kube-system/default-http-backend + - --sync-period=300s + - --config-file-path=/etc/ingress-config/gce.conf + volumeMounts: + - mountPath: /etc/ingress-config + name: cloudconfig + readOnly: true + volumes: + - configMap: + name: ingress-controller-config + name: cloudconfig \ No newline at end of file From 1c94324bd134515314d70fad43415fcce93e663e Mon Sep 17 00:00:00 2001 From: Meaghan Kjelland Date: Tue, 8 May 2018 15:14:13 -0700 Subject: [PATCH 3/7] Fixing build issue by adding new actuator method to terraform cloud provider --- cloud/terraform/machineactuator.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloud/terraform/machineactuator.go b/cloud/terraform/machineactuator.go index 3baebddda36b..f0cd810230bd 100644 --- a/cloud/terraform/machineactuator.go +++ b/cloud/terraform/machineactuator.go @@ -116,6 +116,10 @@ func (tf *TerraformClient) CreateMachineController(cluster *clusterv1.Cluster, i return nil } +func (tf *TerraformClient) ProvisionClusterDependencies(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { + return nil +} + func saveFile(contents, path string, perm os.FileMode) error { return ioutil.WriteFile(path, []byte(contents), perm) } From 9e19e0bb24cb2ba0ce9d7d845cbdffceb8374687 Mon Sep 17 00:00:00 2001 From: Meaghan Kjelland Date: Tue, 8 May 2018 15:27:19 -0700 Subject: [PATCH 4/7] Fixing build issue by adding new actuator method to test actuator --- pkg/controller/machine/testactuator.go | 37 ++++++++++++++++++-------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/pkg/controller/machine/testactuator.go b/pkg/controller/machine/testactuator.go index 359383af3e1f..d1ec99186e5f 100644 --- a/pkg/controller/machine/testactuator.go +++ b/pkg/controller/machine/testactuator.go @@ -23,17 +23,32 @@ import ( ) type TestActuator struct { - unblock chan string - BlockOnCreate bool - BlockOnDelete bool - BlockOnUpdate bool - BlockOnExists bool - CreateCallCount int64 - DeleteCallCount int64 - UpdateCallCount int64 - ExistsCallCount int64 - ExistsValue bool - Lock sync.Mutex + unblock chan string + BlockOnProvisionDependencies bool + BlockOnCreate bool + BlockOnDelete bool + BlockOnUpdate bool + BlockOnExists bool + ProvisionDependenciesCallCount int64 + CreateCallCount int64 + DeleteCallCount int64 + UpdateCallCount int64 + ExistsCallCount int64 + ExistsValue bool + Lock sync.Mutex +} + +func (a *TestActuator) ProvisionClusterDependencies(*v1alpha1.Cluster, []*v1alpha1.Machine) error { + defer func() { + if a.BlockOnProvisionDependencies { + <-a.unblock + } + }() + + a.Lock.Lock() + defer a.Lock.Unlock() + a.ProvisionDependenciesCallCount++ + return nil } func (a *TestActuator) Create(*v1alpha1.Cluster, *v1alpha1.Machine) error { From 4eb0d896f23f5a109086a9841bbc0a0f4a205486 Mon Sep 17 00:00:00 2001 From: Meaghan Kjelland Date: Wed, 9 May 2018 11:20:51 -0700 Subject: [PATCH 5/7] Addressing code review feedback: - moving ProvisionClusterDependencies method to machinedpeloyer - Updating ingress controller service account permissions - various small cleanup fixes --- cloud/google/serviceaccount.go | 4 +- cloud/terraform/machineactuator.go | 4 -- .../examples/gce-ingress-controller/README.md | 4 +- .../ingress-controller.yaml | 19 ++++++---- gcp-deployer/deploy/machinedeployer.go | 3 ++ gcp-deployer/machine_setup_configs.yaml | 10 +++-- pkg/controller/machine/actuator.go | 4 -- pkg/controller/machine/testactuator.go | 37 ++++++------------- 8 files changed, 36 insertions(+), 49 deletions(-) diff --git a/cloud/google/serviceaccount.go b/cloud/google/serviceaccount.go index dc6aecc78380..82dcf4820188 100644 --- a/cloud/google/serviceaccount.go +++ b/cloud/google/serviceaccount.go @@ -27,7 +27,7 @@ const ( WorkerNodeServiceAccountPrefix = "k8s-worker" MachineControllerServiceAccountPrefix = "k8s-machine-controller" MachineControllerSecret = "machine-controller-credential" - ClusterAnnotationPrefix = "service-account-" + ClusterAnnotationPrefix = "gce.clusterapi.k8s.io/service-account-" ) var ( @@ -105,6 +105,8 @@ func (gce *GCEClient) CreateMachineControllerServiceAccount(cluster *clusterv1.C return nil } +// creates a service account with the roles specifed. Returns the account id +// of the created account and the project it belongs to. func (gce *GCEClient) createServiceAccount(serviceAccountPrefix string, roles []string, cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) (string, string, error) { if len(initialMachines) == 0 { return "", "", fmt.Errorf("machine count is zero, cannot create service a/c") diff --git a/cloud/terraform/machineactuator.go b/cloud/terraform/machineactuator.go index f0cd810230bd..3baebddda36b 100644 --- a/cloud/terraform/machineactuator.go +++ b/cloud/terraform/machineactuator.go @@ -116,10 +116,6 @@ func (tf *TerraformClient) CreateMachineController(cluster *clusterv1.Cluster, i return nil } -func (tf *TerraformClient) ProvisionClusterDependencies(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { - return nil -} - func saveFile(contents, path string, perm os.FileMode) error { return ioutil.WriteFile(path, []byte(contents), perm) } diff --git a/docs/examples/gce-ingress-controller/README.md b/docs/examples/gce-ingress-controller/README.md index f4b9f38c8e6a..41efa6f0c51e 100644 --- a/docs/examples/gce-ingress-controller/README.md +++ b/docs/examples/gce-ingress-controller/README.md @@ -4,8 +4,8 @@ Instructions for how to deploy an ingress controller in a cluster that was deployed by gcp-deployer 1. Replace `` and `` in -`ingress-controller.yml`. -1. Run `kubectl create -f ingress-controller.yml`. This will create +`ingress-controller.yaml`. +1. Run `kubectl create -f ingress-controller.yaml`. This will create Kubernetes service account with the correct permissions in the cluster, a default backend for the ingress controller, and the glbc app diff --git a/docs/examples/gce-ingress-controller/ingress-controller.yaml b/docs/examples/gce-ingress-controller/ingress-controller.yaml index 205845a4cfbf..74b9b157f1fb 100644 --- a/docs/examples/gce-ingress-controller/ingress-controller.yaml +++ b/docs/examples/gce-ingress-controller/ingress-controller.yaml @@ -1,4 +1,3 @@ ---- apiVersion: v1 kind: ServiceAccount metadata: @@ -11,7 +10,10 @@ metadata: name: system:controller:glbc rules: - apiGroups: [""] - resources: ["secrets", "endpoints", "services", "pods", "nodes", "namespaces", "configmaps", "events"] + resources: ["secrets", "endpoints", "services", "pods", "nodes", "namespaces"] + verbs: ["describe", "get", "list", "watch"] +- apiGroups: [""] + resources: ["events", "configmaps"] verbs: ["describe", "get", "list", "watch", "update", "create", "patch"] - apiGroups: ["extensions"] resources: ["ingresses"] @@ -86,7 +88,7 @@ data: gce.conf: | [global] project-id = - node-tags = + node-tags = -worker --- apiVersion: v1 kind: Service @@ -111,8 +113,8 @@ spec: selector: k8s-app: glbc --- -apiVersion: v1 -kind: ReplicationController +apiVersion: extensions/v1beta1 +kind: Deployment metadata: namespace: kube-system name: l7-lb-controller @@ -123,8 +125,9 @@ spec: # There should never be more than 1 controller alive simultaneously. replicas: 1 selector: - k8s-app: glbc - version: v1.1.1 + matchLabels: + k8s-app: glbc + version: v1.1.1 template: metadata: labels: @@ -162,4 +165,4 @@ spec: volumes: - configMap: name: ingress-controller-config - name: cloudconfig \ No newline at end of file + name: cloudconfig diff --git a/gcp-deployer/deploy/machinedeployer.go b/gcp-deployer/deploy/machinedeployer.go index 449494a3ea8d..ab31b90e932b 100644 --- a/gcp-deployer/deploy/machinedeployer.go +++ b/gcp-deployer/deploy/machinedeployer.go @@ -12,6 +12,9 @@ type machineDeployer interface { GetIP(machine *clusterv1.Machine) (string, error) GetKubeConfig(master *clusterv1.Machine) (string, error) + // Provision infrastructure that the cluster needs before it + // can be created + ProvisionClusterDependencies(*clusterv1.Cluster, []*clusterv1.Machine) error // Create and start the machine controller. The list of initial // machines don't have to be reconciled as part of this function, but // are provided in case the function wants to refer to them (and their diff --git a/gcp-deployer/machine_setup_configs.yaml b/gcp-deployer/machine_setup_configs.yaml index d24cb235cb2b..d2fae882067f 100644 --- a/gcp-deployer/machine_setup_configs.yaml +++ b/gcp-deployer/machine_setup_configs.yaml @@ -64,13 +64,14 @@ items: mv /usr/bin/kubeadm.dl /usr/bin/kubeadm chmod a+rx /usr/bin/kubeadm - # Override network args to use kubenet instead of cni, and override Kubelet DNS args. + # Override network args to use kubenet instead of cni, override Kubelet DNS args and + # add cloud provider args. cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf < /etc/systemd/system/kubelet.service.d/20-kubenet.conf < Date: Thu, 10 May 2018 14:55:31 -0700 Subject: [PATCH 6/7] Adding post create step to cluster creation to allow creation of ingress service account and ingress controller --- cloud/google/config/configtemplate.go | 199 ++++++++++++++++++ cloud/google/machineactuator.go | 37 +++- cloud/google/pods.go | 71 +++++++ cloud/google/serviceaccount.go | 36 +++- .../examples/gce-ingress-controller/README.md | 12 -- .../ingress-controller.yaml | 168 --------------- gcp-deployer/deploy/deploy_helper.go | 5 + gcp-deployer/deploy/machinedeployer.go | 2 + gcp-deployer/machine_setup_configs.yaml | 14 -- 9 files changed, 337 insertions(+), 207 deletions(-) delete mode 100644 docs/examples/gce-ingress-controller/README.md delete mode 100644 docs/examples/gce-ingress-controller/ingress-controller.yaml diff --git a/cloud/google/config/configtemplate.go b/cloud/google/config/configtemplate.go index 81c600706ccc..c6f00bd6b327 100644 --- a/cloud/google/config/configtemplate.go +++ b/cloud/google/config/configtemplate.go @@ -282,3 +282,202 @@ data: tls.crt: {{ .TLSCrt }} tls.key: {{ .TLSKey }} ` + +const StorageClassConfigTemplate = ` +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: standard + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-standard +` + +const IngressControllerConfigTemplate = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + name: glbc + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/ +kind: ClusterRole +metadata: + name: system:controller:glbc +rules: +- apiGroups: [""] + resources: ["secrets", "endpoints", "services", "pods", "nodes", "namespaces"] + verbs: ["describe", "get", "list", "watch"] +- apiGroups: [""] + resources: ["events", "configmaps"] + verbs: ["describe", "get", "list", "watch", "update", "create", "patch"] +- apiGroups: ["extensions"] + resources: ["ingresses"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["extensions"] + resources: ["ingresses/status"] + verbs: ["update"] +--- +apiVersion: rbac.authorization.k8s.io/ +kind: ClusterRoleBinding +metadata: + name: system:controller:glbc +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:glbc +subjects: +- kind: ServiceAccount + name: glbc + namespace: kube-system +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: l7-default-backend + namespace: kube-system + labels: + k8s-app: glbc + kubernetes.io/name: "GLBC" + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: glbc + template: + metadata: + labels: + k8s-app: glbc + name: glbc + spec: + containers: + - name: default-http-backend + # Any image is permissible as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.4 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ingress-controller-config + namespace: kube-system +data: + gce.conf: | + [global] + token-url = nil + network = default + project-id = {{ .Project }} + node-tags = {{ .NodeTag }} +--- +apiVersion: v1 +kind: Service +metadata: + # This must match the --default-backend-service argument of the l7 lb + # controller and is required because GCE mandates a default backend. + name: default-http-backend + namespace: kube-system + labels: + k8s-app: glbc + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "GLBCDefaultBackend" +spec: + # The default backend must be of type NodePort. + type: NodePort + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + k8s-app: glbc +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + namespace: kube-system + name: l7-lb-controller + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + labels: + k8s-app: glbc + version: v1.1.1 + kubernetes.io/name: "GLBC" +spec: + # There should never be more than 1 controller alive simultaneously. + replicas: 1 + selector: + matchLabels: + k8s-app: glbc + version: v1.1.1 + template: + metadata: + labels: + k8s-app: glbc + version: v1.1.1 + name: glbc + spec: + serviceAccountName: glbc + terminationGracePeriodSeconds: 600 + containers: + - image: k8s.gcr.io/ingress-gce-glbc-amd64:v1.1.1 + livenessProbe: + httpGet: + path: /healthz + port: 8086 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/credentials/service-account.json + name: l7-lb-controller + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 50Mi + command: + - sh + - -c + - 'exec /glbc --gce-ratelimit=ga.Operations.Get,qps,10,100 --gce-ratelimit=alpha.Operations.Get,qps,10,100 --gce-ratelimit=ga.BackendServices.Get,qps,1.8,1 --gce-ratelimit=ga.HealthChecks.Get,qps,1.8,1 --gce-ratelimit=alpha.HealthChecks.Get,qps,1.8,1 --verbose --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=true --use-real-cloud=true --config-file-path=/etc/ingress-config/gce.conf --healthz-port=8086 2>&1' + volumeMounts: + - mountPath: /etc/ingress-config + name: cloudconfig + readOnly: true + - mountPath: /etc/credentials + name: credentials + readOnly: true + volumes: + - name: cloudconfig + configMap: + name: ingress-controller-config + - name: credentials + secret: + secretName: glbc-gcp-key +` diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go index 11d2f294b252..4ca00befb1a8 100644 --- a/cloud/google/machineactuator.go +++ b/cloud/google/machineactuator.go @@ -186,17 +186,12 @@ func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initia } func (gce *GCEClient) ProvisionClusterDependencies(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { - err := gce.CreateMasterNodeServiceAccount(cluster, initialMachines) + err := gce.CreateWorkerNodeServiceAccount(cluster, initialMachines) if err != nil { return err } - err = gce.CreateWorkerNodeServiceAccount(cluster, initialMachines) - if err != nil { - return err - } - - return nil + return gce.CreateMasterNodeServiceAccount(cluster, initialMachines) } func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Machine) error { @@ -399,6 +394,31 @@ func (gce *GCEClient) Delete(machine *clusterv1.Machine) error { return err } +func (gce *GCEClient) PostCreate(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error { + err := CreateDefaultStorageClass() + if err != nil { + return fmt.Errorf("error creating default storage class: %v", err) + } + + err = gce.CreateIngressControllerServiceAccount(cluster, machines) + if err != nil { + return fmt.Errorf("error creating service account for ingress controller: %v", err) + } + + if len(machines) > 0 { + config, err := gce.providerconfig(machines[0].Spec.ProviderConfig) + if err != nil { + return fmt.Errorf("error creating ingress controller: %v", err) + } + err = CreateIngressController(config.Project, cluster.Name) + if err != nil { + return fmt.Errorf("error creating ingress controller: %v", err) + } + } + + return nil +} + func (gce *GCEClient) PostDelete(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error { if err := gce.DeleteMasterNodeServiceAccount(cluster, machines); err != nil { return fmt.Errorf("error deleting master node service account: %v", err) @@ -406,6 +426,9 @@ func (gce *GCEClient) PostDelete(cluster *clusterv1.Cluster, machines []*cluster if err := gce.DeleteWorkerNodeServiceAccount(cluster, machines); err != nil { return fmt.Errorf("error deleting worker node service account: %v", err) } + if err := gce.DeleteIngressControllerServiceAccount(cluster, machines); err != nil { + return fmt.Errorf("error deleting ingress controller service account: %v", err) + } if err := gce.DeleteMachineControllerServiceAccount(cluster, machines); err != nil { return fmt.Errorf("error deleting machine controller service account: %v", err) } diff --git a/cloud/google/pods.go b/cloud/google/pods.go index 6000d0fe42f8..32ffedab7851 100644 --- a/cloud/google/pods.go +++ b/cloud/google/pods.go @@ -177,6 +177,77 @@ func CreateApiServerAndController(token string) error { } } +func CreateIngressController(project string, clusterName string) error { + tmpl, err := template.New("config").Parse(config.IngressControllerConfigTemplate) + if err != nil { + return err + } + + type params struct { + Project string + NodeTag string + } + + var tmplBuf bytes.Buffer + err = tmpl.Execute(&tmplBuf, params{ + Project: project, + NodeTag: clusterName + "-worker", + }) + if err != nil { + return err + } + + maxTries := 5 + for tries := 0; tries < maxTries; tries++ { + err = deployConfig(tmplBuf.Bytes()) + if err == nil { + return nil + } else { + if tries < maxTries-1 { + glog.Infof("Error scheduling ingress controller. Will retry... %v\n", err) + time.Sleep(3 * time.Second) + } + } + } + + if err != nil { + return fmt.Errorf("couldn't start ingress controller: %v\n", err) + } else { + return nil + } +} + +func CreateDefaultStorageClass() error { + tmpl, err := template.New("config").Parse(config.StorageClassConfigTemplate) + if err != nil { + return err + } + var tmplBuf bytes.Buffer + err = tmpl.Execute(&tmplBuf, nil) + if err != nil { + return err + } + + maxTries := 5 + for tries := 0; tries < maxTries; tries++ { + err = deployConfig(tmplBuf.Bytes()) + if err == nil { + return nil + } else { + if tries < maxTries-1 { + glog.Info("Error creating default storage class. Will retry...\n") + time.Sleep(3 * time.Second) + } + } + } + + if err != nil { + return fmt.Errorf("couldn't create default storage class: %v\n", err) + } else { + return nil + } +} + func deployConfig(manifest []byte) error { cmd := exec.Command("kubectl", "create", "-f", "-") stdin, err := cmd.StdinPipe() diff --git a/cloud/google/serviceaccount.go b/cloud/google/serviceaccount.go index 82dcf4820188..d155088a5dd6 100644 --- a/cloud/google/serviceaccount.go +++ b/cloud/google/serviceaccount.go @@ -25,9 +25,13 @@ import ( const ( MasterNodeServiceAccountPrefix = "k8s-master" WorkerNodeServiceAccountPrefix = "k8s-worker" + IngressControllerServiceAccountPrefix = "k8s-ingress-controller" MachineControllerServiceAccountPrefix = "k8s-machine-controller" - MachineControllerSecret = "machine-controller-credential" - ClusterAnnotationPrefix = "gce.clusterapi.k8s.io/service-account-" + + IngressControllerSecret = "glbc-gcp-key" + MachineControllerSecret = "machine-controller-credential" + + ClusterAnnotationPrefix = "gce.clusterapi.k8s.io/service-account-" ) var ( @@ -40,10 +44,12 @@ var ( "storage.admin", "storage.objectViewer", } - WorkerNodeRoles = []string{ - "compute.instanceAdmin", + WorkerNodeRoles = []string{} + IngressControllerRoles = []string{ + "compute.instanceAdmin.v1", "compute.networkAdmin", "compute.securityAdmin", + "iam.serviceAccountActor", } MachineControllerRoles = []string{ "compute.instanceAdmin.v1", @@ -76,6 +82,16 @@ func (gce *GCEClient) CreateWorkerNodeServiceAccount(cluster *clusterv1.Cluster, return err } +// Creates a GCP service account for the ingress controller +func (gce *GCEClient) CreateIngressControllerServiceAccount(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error { + accountId, project, err := gce.createServiceAccount(IngressControllerServiceAccountPrefix, IngressControllerRoles, cluster, initialMachines) + if err != nil { + return err + } + + return gce.createSecretForServiceAccountKey(accountId, project, IngressControllerSecret, "kube-system") +} + // Creates a GCP service account for the machine controller, granted the // permissions to manage compute instances, and stores its credentials as a // Kubernetes secret. @@ -85,15 +101,19 @@ func (gce *GCEClient) CreateMachineControllerServiceAccount(cluster *clusterv1.C return err } + return gce.createSecretForServiceAccountKey(accountId, project, MachineControllerSecret, "default") +} + +func (gce *GCEClient) createSecretForServiceAccountKey(accountId string, project string, secretName string, namespace string) error { email := fmt.Sprintf("%s@%s.iam.gserviceaccount.com", accountId, project) localFile := accountId + "-key.json" - err = run("gcloud", "--project", project, "iam", "service-accounts", "keys", "create", localFile, "--iam-account", email) + err := run("gcloud", "--project", project, "iam", "service-accounts", "keys", "create", localFile, "--iam-account", email) if err != nil { return fmt.Errorf("couldn't create service account key: %v", err) } - err = run("kubectl", "create", "secret", "generic", MachineControllerSecret, "--from-file=service-account.json="+localFile) + err = run("kubectl", "create", "secret", "generic", secretName, "--from-file=service-account.json="+localFile, "--namespace="+namespace) if err != nil { return fmt.Errorf("couldn't import service account key as credential: %v", err) } @@ -156,6 +176,10 @@ func (gce *GCEClient) DeleteWorkerNodeServiceAccount(cluster *clusterv1.Cluster, return gce.deleteServiceAccount(WorkerNodeServiceAccountPrefix, WorkerNodeRoles, cluster, machines) } +func (gce *GCEClient) DeleteIngressControllerServiceAccount(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error { + return gce.deleteServiceAccount(IngressControllerServiceAccountPrefix, IngressControllerRoles, cluster, machines) +} + func (gce *GCEClient) DeleteMachineControllerServiceAccount(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error { return gce.deleteServiceAccount(MachineControllerServiceAccountPrefix, MachineControllerRoles, cluster, machines) } diff --git a/docs/examples/gce-ingress-controller/README.md b/docs/examples/gce-ingress-controller/README.md deleted file mode 100644 index 41efa6f0c51e..000000000000 --- a/docs/examples/gce-ingress-controller/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Deploy GCE Ingress Controller - -Instructions for how to deploy an ingress controller in a cluster -that was deployed by gcp-deployer - -1. Replace `` and `` in -`ingress-controller.yaml`. -1. Run `kubectl create -f ingress-controller.yaml`. This will create -Kubernetes service account with the correct permissions in the cluster, -a default backend for the ingress controller, and the glbc app - -Now you will be able to create ingress objects. \ No newline at end of file diff --git a/docs/examples/gce-ingress-controller/ingress-controller.yaml b/docs/examples/gce-ingress-controller/ingress-controller.yaml deleted file mode 100644 index 74b9b157f1fb..000000000000 --- a/docs/examples/gce-ingress-controller/ingress-controller.yaml +++ /dev/null @@ -1,168 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: glbc - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/ -kind: ClusterRole -metadata: - name: system:controller:glbc -rules: -- apiGroups: [""] - resources: ["secrets", "endpoints", "services", "pods", "nodes", "namespaces"] - verbs: ["describe", "get", "list", "watch"] -- apiGroups: [""] - resources: ["events", "configmaps"] - verbs: ["describe", "get", "list", "watch", "update", "create", "patch"] -- apiGroups: ["extensions"] - resources: ["ingresses"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: ["extensions"] - resources: ["ingresses/status"] - verbs: ["update"] ---- -apiVersion: rbac.authorization.k8s.io/ -kind: ClusterRoleBinding -metadata: - name: system:controller:glbc -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:controller:glbc -subjects: -- kind: ServiceAccount - name: glbc - namespace: kube-system ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: l7-default-backend - namespace: kube-system - labels: - k8s-app: glbc - kubernetes.io/name: "GLBC" - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: glbc - template: - metadata: - labels: - k8s-app: glbc - name: glbc - spec: - containers: - - name: default-http-backend - # Any image is permissible as long as: - # 1. It serves a 404 page at / - # 2. It serves 200 on a /healthz endpoint - image: gcr.io/google_containers/defaultbackend:1.4 - livenessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - ports: - - containerPort: 8080 - resources: - limits: - cpu: 10m - memory: 20Mi - requests: - cpu: 10m - memory: 20Mi ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: ingress-controller-config - namespace: kube-system -data: - gce.conf: | - [global] - project-id = - node-tags = -worker ---- -apiVersion: v1 -kind: Service -metadata: - # This must match the --default-backend-service argument of the l7 lb - # controller and is required because GCE mandates a default backend. - name: default-http-backend - namespace: kube-system - labels: - k8s-app: glbc - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/name: "GLBCDefaultBackend" -spec: - # The default backend must be of type NodePort. - type: NodePort - ports: - - port: 80 - targetPort: 8080 - protocol: TCP - name: http - selector: - k8s-app: glbc ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - namespace: kube-system - name: l7-lb-controller - labels: - k8s-app: glbc - version: v1.1.1 -spec: - # There should never be more than 1 controller alive simultaneously. - replicas: 1 - selector: - matchLabels: - k8s-app: glbc - version: v1.1.1 - template: - metadata: - labels: - k8s-app: glbc - version: v1.1.1 - name: glbc - spec: - serviceAccountName: glbc - terminationGracePeriodSeconds: 600 - containers: - - image: k8s.gcr.io/ingress-gce-glbc-amd64:v1.1.1 - livenessProbe: - httpGet: - path: /healthz - port: 8081 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - name: l7-lb-controller - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 50Mi - args: - - --default-backend-service=kube-system/default-http-backend - - --sync-period=300s - - --config-file-path=/etc/ingress-config/gce.conf - volumeMounts: - - mountPath: /etc/ingress-config - name: cloudconfig - readOnly: true - volumes: - - configMap: - name: ingress-controller-config - name: cloudconfig diff --git a/gcp-deployer/deploy/deploy_helper.go b/gcp-deployer/deploy/deploy_helper.go index 5ae0e08f9608..06e1984c2cdf 100644 --- a/gcp-deployer/deploy/deploy_helper.go +++ b/gcp-deployer/deploy/deploy_helper.go @@ -98,6 +98,11 @@ func (d *deployer) createCluster(c *clusterv1.Cluster, machines []*clusterv1.Mac return fmt.Errorf("can't create machine controller: %v", err) } + glog.Info("Creating additional cluster resources...") + if err := d.machineDeployer.PostCreate(c, machines); err != nil { + return fmt.Errorf("can't create additional cluster resources: %v", err) + } + if err := d.waitForClusterResourceReady(); err != nil { return err } diff --git a/gcp-deployer/deploy/machinedeployer.go b/gcp-deployer/deploy/machinedeployer.go index ab31b90e932b..2eaee0f10771 100644 --- a/gcp-deployer/deploy/machinedeployer.go +++ b/gcp-deployer/deploy/machinedeployer.go @@ -21,5 +21,7 @@ type machineDeployer interface { // ProviderConfigs) to know how to configure the machine controller. // Not idempotent. CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine, clientSet kubernetes.Clientset) error + // Create GCE and kubernetes resources after the cluster is created + PostCreate(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error PostDelete(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error } diff --git a/gcp-deployer/machine_setup_configs.yaml b/gcp-deployer/machine_setup_configs.yaml index d2fae882067f..314ca0facb06 100644 --- a/gcp-deployer/machine_setup_configs.yaml +++ b/gcp-deployer/machine_setup_configs.yaml @@ -115,20 +115,6 @@ items: kubeadm init --config /etc/kubernetes/kubeadm_config.yaml - # add default storage class - cat > /etc/kubernetes/default-storage-class.yaml < Date: Tue, 15 May 2018 10:30:52 -0700 Subject: [PATCH 7/7] Fixing typo --- cloud/google/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/google/metadata.go b/cloud/google/metadata.go index af237d3260e7..5e6f09c2e26c 100644 --- a/cloud/google/metadata.go +++ b/cloud/google/metadata.go @@ -124,7 +124,7 @@ MASTER={{ .MasterEndpoint }} MACHINE={{ .Machine.ObjectMeta.Name }} CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }} POD_CIDR={{ .PodCIDR }} -SERVICE_CIDER={{ .ServiceCIDR }} +SERVICE_CIDR={{ .ServiceCIDR }} # Environment variables for GCE cloud config PROJECT={{ .Project }} NETWORK=default