Merge pull request #141 from mkjelland/gce-cloud-provider

Enable GCE cloud provider support: Issue #81
kubernetes-sigs · May 15, 2018 · 4e00ee9 · 4e00ee9
2 parents d1e212f + 3d9c8e1
commit 4e00ee9
Show file tree

Hide file tree

Showing 9 changed files with 501 additions and 50 deletions.
diff --git a/cloud/google/config/configtemplate.go b/cloud/google/config/configtemplate.go
@@ -282,3 +282,202 @@ data:
   tls.crt: {{ .TLSCrt }}
   tls.key: {{ .TLSKey }}
 `
+
+const StorageClassConfigTemplate = `
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: standard
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-standard
+`
+
+const IngressControllerConfigTemplate = `
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: glbc
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/
+kind: ClusterRole
+metadata:
+  name: system:controller:glbc
+rules:
+- apiGroups: [""]
+  resources: ["secrets", "endpoints", "services", "pods", "nodes", "namespaces"]
+  verbs: ["describe", "get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["events", "configmaps"]
+  verbs: ["describe", "get", "list", "watch", "update", "create", "patch"]
+- apiGroups: ["extensions"]
+  resources: ["ingresses"]
+  verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["extensions"]
+  resources: ["ingresses/status"]
+  verbs: ["update"]
+---
+apiVersion: rbac.authorization.k8s.io/
+kind: ClusterRoleBinding
+metadata:
+  name: system:controller:glbc
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:controller:glbc
+subjects:
+- kind: ServiceAccount
+  name: glbc
+  namespace: kube-system
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: l7-default-backend
+  namespace: kube-system
+  labels:
+    k8s-app: glbc
+    kubernetes.io/name: "GLBC"
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: glbc
+  template:
+    metadata:
+      labels:
+        k8s-app: glbc
+        name: glbc
+    spec:
+      containers:
+      - name: default-http-backend
+        # Any image is permissible as long as:
+        # 1. It serves a 404 page at /
+        # 2. It serves 200 on a /healthz endpoint
+        image: gcr.io/google_containers/defaultbackend:1.4
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        ports:
+        - containerPort: 8080
+        resources:
+          limits:
+            cpu: 10m
+            memory: 20Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ingress-controller-config
+  namespace: kube-system
+data:
+  gce.conf: |
+    [global]
+    token-url = nil
+    network = default
+    project-id = {{ .Project }}
+    node-tags = {{ .NodeTag }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  # This must match the --default-backend-service argument of the l7 lb
+  # controller and is required because GCE mandates a default backend.
+  name: default-http-backend
+  namespace: kube-system
+  labels:
+    k8s-app: glbc
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "GLBCDefaultBackend"
+spec:
+  # The default backend must be of type NodePort.
+  type: NodePort
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    k8s-app: glbc
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  namespace: kube-system
+  name: l7-lb-controller
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
+  labels:
+    k8s-app: glbc
+    version: v1.1.1
+    kubernetes.io/name: "GLBC"
+spec:
+  # There should never be more than 1 controller alive simultaneously.
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: glbc
+      version: v1.1.1
+  template:
+    metadata:
+      labels:
+        k8s-app: glbc
+        version: v1.1.1
+        name: glbc
+    spec:
+      serviceAccountName: glbc
+      terminationGracePeriodSeconds: 600
+      containers:
+      - image: k8s.gcr.io/ingress-gce-glbc-amd64:v1.1.1
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8086
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+        env:
+        - name: GOOGLE_APPLICATION_CREDENTIALS
+          value: /etc/credentials/service-account.json
+        name: l7-lb-controller
+        resources:
+          limits:
+            cpu: 100m
+            memory: 100Mi
+          requests:
+            cpu: 100m
+            memory: 50Mi
+        command:
+        - sh
+        - -c
+        - 'exec /glbc --gce-ratelimit=ga.Operations.Get,qps,10,100 --gce-ratelimit=alpha.Operations.Get,qps,10,100 --gce-ratelimit=ga.BackendServices.Get,qps,1.8,1 --gce-ratelimit=ga.HealthChecks.Get,qps,1.8,1 --gce-ratelimit=alpha.HealthChecks.Get,qps,1.8,1 --verbose --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=true --use-real-cloud=true --config-file-path=/etc/ingress-config/gce.conf --healthz-port=8086 2>&1'
+        volumeMounts:
+        - mountPath: /etc/ingress-config
+          name: cloudconfig
+          readOnly: true
+        - mountPath: /etc/credentials
+          name: credentials
+          readOnly: true
+      volumes:
+      - name: cloudconfig
+        configMap:
+          name: ingress-controller-config
+      - name: credentials
+        secret:
+          secretName: glbc-gcp-key
+`
diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go
@@ -181,6 +181,15 @@ func (gce *GCEClient) CreateMachineController(cluster *clusterv1.Cluster, initia
 	return nil
 }
 
+func (gce *GCEClient) ProvisionClusterDependencies(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error {
+	err := gce.CreateWorkerNodeServiceAccount(cluster, initialMachines)
+	if err != nil {
+		return err
+	}
+
+	return gce.CreateMasterNodeServiceAccount(cluster, initialMachines)
+}
+
 func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Machine) error {
 	if gce.machineSetupConfigGetter == nil {
 		return errors.New("a valid machineSetupConfigGetter is required")
@@ -234,7 +243,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 			return errors.New("invalid cluster state: cannot create a Kubernetes node without an API endpoint")
 		}
 		var err error
-		metadata, err = nodeMetadata(gce.kubeadmToken, cluster, machine, &machineSetupMetadata)
+		metadata, err = nodeMetadata(gce.kubeadmToken, cluster, machine, config.Project, &machineSetupMetadata)
 		if err != nil {
 			return err
 		}
@@ -265,18 +274,6 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 			labels[BootstrapLabelKey] = "true"
 		}
 
-		// The service account is needed for the Kubernetes GCE cloud provider code. It is needed on the master VM.
-		serviceAccounts := []*compute.ServiceAccount{nil}
-		if util.IsMaster(machine) {
-			serviceAccounts = append(serviceAccounts,
-				&compute.ServiceAccount{
-					Email: "default",
-					Scopes: []string{
-						"https://www.googleapis.com/auth/cloud-platform",
-					},
-				})
-		}
-
 		op, err := gce.computeService.InstancesInsert(project, zone, &compute.Instance{
 			Name:         name,
 			MachineType:  fmt.Sprintf("zones/%s/machineTypes/%s", zone, config.MachineType),
@@ -310,8 +307,15 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 					"https-server",
 					fmt.Sprintf("%s-worker", cluster.Name)},
 			},
-			Labels:          labels,
-			ServiceAccounts: serviceAccounts,
+			Labels: labels,
+			ServiceAccounts: []*compute.ServiceAccount{
+				{
+					Email: gce.GetDefaultServiceAccountForMachine(cluster, machine),
+					Scopes: []string{
+						compute.CloudPlatformScope,
+					},
+				},
+			},
 		})
 
 		if err == nil {
@@ -389,8 +393,45 @@ func (gce *GCEClient) Delete(machine *clusterv1.Machine) error {
 	return err
 }
 
+func (gce *GCEClient) PostCreate(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error {
+	err := CreateDefaultStorageClass()
+	if err != nil {
+		return fmt.Errorf("error creating default storage class: %v", err)
+	}
+
+	err = gce.CreateIngressControllerServiceAccount(cluster, machines)
+	if err != nil {
+		return fmt.Errorf("error creating service account for ingress controller: %v", err)
+	}
+
+	if len(machines) > 0 {
+		config, err := gce.providerconfig(machines[0].Spec.ProviderConfig)
+		if err != nil {
+			return fmt.Errorf("error creating ingress controller: %v", err)
+		}
+		err = CreateIngressController(config.Project, cluster.Name)
+		if err != nil {
+			return fmt.Errorf("error creating ingress controller: %v", err)
+		}
+	}
+
+	return nil
+}
+
 func (gce *GCEClient) PostDelete(cluster *clusterv1.Cluster, machines []*clusterv1.Machine) error {
-	return gce.DeleteMachineControllerServiceAccount(cluster, machines)
+	if err := gce.DeleteMasterNodeServiceAccount(cluster, machines); err != nil {
+		return fmt.Errorf("error deleting master node service account: %v", err)
+	}
+	if err := gce.DeleteWorkerNodeServiceAccount(cluster, machines); err != nil {
+		return fmt.Errorf("error deleting worker node service account: %v", err)
+	}
+	if err := gce.DeleteIngressControllerServiceAccount(cluster, machines); err != nil {
+		return fmt.Errorf("error deleting ingress controller service account: %v", err)
+	}
+	if err := gce.DeleteMachineControllerServiceAccount(cluster, machines); err != nil {
+		return fmt.Errorf("error deleting machine controller service account: %v", err)
+	}
+	return nil
 }
 
 func (gce *GCEClient) Update(cluster *clusterv1.Cluster, goalMachine *clusterv1.Machine) error {

diff --git a/cloud/google/metadata.go b/cloud/google/metadata.go
@@ -39,11 +39,12 @@ type metadataParams struct {
 	MasterEndpoint string
 }
 
-func nodeMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.Machine, metadata *machinesetup.Metadata) (map[string]string, error) {
+func nodeMetadata(token string, cluster *clusterv1.Cluster, machine *clusterv1.Machine, project string, metadata *machinesetup.Metadata) (map[string]string, error) {
 	params := metadataParams{
 		Token:          token,
 		Cluster:        cluster,
 		Machine:        machine,
+		Project:        project,
 		Metadata:       metadata,
 		PodCIDR:        getSubnet(cluster.Spec.ClusterNetwork.Pods),
 		ServiceCIDR:    getSubnet(cluster.Spec.ClusterNetwork.Services),
@@ -124,4 +125,10 @@ MACHINE={{ .Machine.ObjectMeta.Name }}
 CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }}
 POD_CIDR={{ .PodCIDR }}
 SERVICE_CIDR={{ .ServiceCIDR }}
+# Environment variables for GCE cloud config
+PROJECT={{ .Project }}
+NETWORK=default
+SUBNETWORK=kubernetes
+CLUSTER_NAME={{ .Cluster.Name }}
+NODE_TAG="$CLUSTER_NAME-worker"
 `