From ebb8c5228fc21654101676c0e0004f4fde79ad75 Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Mon, 23 Apr 2018 10:27:20 -0700
Subject: [PATCH 01/11] Use kubenet instead of weave net

- add a worker tag to worker nodes so firewall rules created for
loadbalancers will be applied to the worker nodes.
- add the default service account to new VM instances.
- add serviceAccountActor role to the machine controller service account
so the master can spin up new nodes.
- add a function to the startup scripts to prevent docker from starting
on installation so we can configure DOCKER_OPTS and then start it
manually. This is needed to prevent it from messing with the IP tables.
---
 cloud/google/machineactuator.go         |  15 +-
 cloud/google/metadata.go                |   5 +-
 cloud/google/serviceaccount.go          |   4 +
 gcp-deployer/machine_setup_configs.yaml | 174 ++----------------------
 4 files changed, 34 insertions(+), 164 deletions(-)

diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go
index a75a0a8f8609..483e2dbdc736 100644
--- a/cloud/google/machineactuator.go
+++ b/cloud/google/machineactuator.go
@@ -266,10 +266,15 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 		if gce.machineClient == nil {
 			labels[BootstrapLabelKey] = "true"
 		}
+		tags := []string{"https-server"}
+		if !util.IsMaster(machine) {
+			tags = append(tags, fmt.Sprintf("%s-worker", cluster.Name))
+		}
 
 		op, err := gce.computeService.InstancesInsert(project, zone, &compute.Instance{
 			Name:        name,
 			MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, config.MachineType),
+			CanIpForward: true,
 			NetworkInterfaces: []*compute.NetworkInterface{
 				{
 					Network: "global/networks/default",
@@ -295,9 +300,17 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 				Items: metadataItems,
 			},
 			Tags: &compute.Tags{
-				Items: []string{"https-server"},
+				Items: tags,
 			},
 			Labels: labels,
+			ServiceAccounts: []*compute.ServiceAccount{
+				{
+					Email: "default",
+					Scopes: []string{
+						"https://www.googleapis.com/auth/cloud-platform",
+					},
+				},
+			},
 		})
 
 		if err == nil {
diff --git a/cloud/google/metadata.go b/cloud/google/metadata.go
index 2469b0552ed3..dc90abc16630 100644
--- a/cloud/google/metadata.go
+++ b/cloud/google/metadata.go
@@ -95,7 +95,7 @@ func init() {
 	nodeEnvironmentVarsTemplate = template.Must(template.New("nodeEnvironmentVars").Parse(nodeEnvironmentVars))
 }
 
-// TODO(kcoronado): replace with actual network and node tag args when they are added into provider config.
+// TODO(kcoronado): replace with actual network args when they are added into provider config.
 const masterEnvironmentVars = `
 #!/bin/bash
 KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }}
@@ -111,7 +111,8 @@ SERVICE_CIDR={{ .ServiceCIDR }}
 PROJECT={{ .Project }}
 NETWORK=default
 SUBNETWORK=kubernetes
-NODE_TAG=worker
+CLUSTER_NAME={{ .Cluster.Name }}
+NODE_TAG="$CLUSTER_NAME-worker"
 `
 
 const nodeEnvironmentVars = `
diff --git a/cloud/google/serviceaccount.go b/cloud/google/serviceaccount.go
index 6844194ad68a..935e561ea892 100644
--- a/cloud/google/serviceaccount.go
+++ b/cloud/google/serviceaccount.go
@@ -65,6 +65,10 @@ func (gce *GCEClient) CreateMachineControllerServiceAccount(cluster *clusterv1.C
 		if err != nil {
 			return fmt.Errorf("couldn't grant permissions to service account: %v", err)
 		}
+		err = run("gcloud", "projects", "add-iam-policy-binding", project, "--member=serviceAccount:"+email, "--role=roles/iam.serviceAccountActor")
+		if err != nil {
+			return fmt.Errorf("couldn't grant permissions to service account: %v", err)
+		}
 	}
 
 	err = run("gcloud", "--project", project, "iam", "service-accounts", "keys", "create", localFile, "--iam-account", email)
diff --git a/gcp-deployer/machine_setup_configs.yaml b/gcp-deployer/machine_setup_configs.yaml
index cbdcd98540b1..f9c4366c7214 100644
--- a/gcp-deployer/machine_setup_configs.yaml
+++ b/gcp-deployer/machine_setup_configs.yaml
@@ -1,160 +1,4 @@
 items:
-- machineParams:
-  - os: ubuntu-1710-weave
-    roles:
-    - Node
-    versions:
-      kubelet: 1.9.4
-      containerRuntime:
-        name: docker
-        version: 1.12.0
-  image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710
-  metadata:
-    startupScript: |
-      set -e
-      set -x
-      (
-      apt-get update
-      apt-get install -y apt-transport-https prips
-      apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys F76221572C52609D
-      cat <<EOF > /etc/apt/sources.list.d/k8s.list
-      deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main
-      EOF
-      apt-get update
-      apt-get install -y docker-engine=1.12.0-0~xenial
-      curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
-      cat <<EOF > /etc/apt/sources.list.d/kubernetes.list
-      deb http://apt.kubernetes.io/ kubernetes-xenial main
-      EOF
-      apt-get update
-      # Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix
-      # search based on our SemVer to find the right (newest) package version.
-      function getversion() {
-      	name=$1
-      	prefix=$2
-      	version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1)
-      	if [[ -z "$version" ]]; then
-      		echo Can\'t find package $name with prefix $prefix
-      		exit 1
-      	fi
-      	echo $version
-      }
-      KUBELET=$(getversion kubelet ${KUBELET_VERSION}-)
-      KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-)
-      KUBECTL=$(getversion kubectl ${KUBELET_VERSION}-)
-      apt-get install -y kubelet=${KUBELET} kubeadm=${KUBEADM} kubectl=${KUBECTL}
-      systemctl enable docker || true
-      systemctl start docker || true
-      # kubeadm uses 10th IP as DNS server
-      CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1)
-      # Override Kubelet DNS args.
-      cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<EOF
-      [Service]
-      Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
-      EOF
-      systemctl daemon-reload
-      systemctl restart kubelet.service
-      kubeadm join --token "${TOKEN}" "${MASTER}" --ignore-preflight-errors=all --discovery-token-unsafe-skip-ca-verification
-      for tries in $(seq 1 60); do
-      	kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
-      	sleep 1
-      done
-      echo done.
-      ) 2>&1 | tee /var/log/startup.log
-- machineParams:
-  - os: ubuntu-1710-weave
-    roles:
-    - Master
-    versions:
-      kubelet: 1.9.4
-      controlPlane: 1.9.4
-      containerRuntime:
-        name: docker
-        version: 1.12.0
-  image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710
-  metadata:
-    startupScript: |
-      set -e
-      set -x
-      (
-      ARCH=amd64
-      curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-      touch /etc/apt/sources.list.d/kubernetes.list
-      sh -c 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list'
-      apt-get update -y
-      apt-get install -y \
-          socat \
-          ebtables \
-          docker.io \
-          apt-transport-https \
-          cloud-utils \
-          prips
-      curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl
-      chmod a+rx /usr/bin/kubeadm.dl
-      # kubeadm uses 10th IP as DNS server
-      CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1)
-      # Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix
-      # search based on our SemVer to find the right (newest) package version.
-      function getversion() {
-      	name=$1
-      	prefix=$2
-      	version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1)
-      	if [[ -z "$version" ]]; then
-      		echo Can\'t find package $name with prefix $prefix
-      		exit 1
-      	fi
-      	echo $version
-      }
-      KUBELET=$(getversion kubelet ${KUBELET_VERSION}-)
-      KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-)
-      apt-get install -y \
-          kubelet=${KUBELET} \
-          kubeadm=${KUBEADM}
-      mv /usr/bin/kubeadm.dl /usr/bin/kubeadm
-      chmod a+rx /usr/bin/kubeadm
-      systemctl enable docker
-      systemctl start docker
-      # Override Kubelet DNS args.
-      cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<EOF
-      [Service]
-      Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
-      EOF
-      systemctl daemon-reload
-      systemctl restart kubelet.service
-      PRIVATEIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip"`
-      echo $PRIVATEIP > /tmp/.ip
-      PUBLICIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"`
-      # Set up kubeadm config file to pass parameters to kubeadm init.
-      cat > /etc/kubernetes/kubeadm_config.yaml <<EOF
-      apiVersion: kubeadm.k8s.io/v1alpha1
-      kind: MasterConfiguration
-      api:
-        advertiseAddress: ${PUBLICIP}
-        bindPort: ${PORT}
-      networking:
-        serviceSubnet: ${SERVICE_CIDR}
-      kubernetesVersion: v${CONTROL_PLANE_VERSION}
-      token: ${TOKEN}
-      controllerManagerExtraArgs:
-        cluster-cidr: ${POD_CIDR}
-        service-cluster-ip-range: ${SERVICE_CIDR}
-        allocate-node-cidrs: "true"
-      apiServerCertSANs:
-      - ${PUBLICIP}
-      - ${PRIVATEIP}
-      EOF
-      kubeadm init --config /etc/kubernetes/kubeadm_config.yaml
-      # install weavenet
-      sysctl net.bridge.bridge-nf-call-iptables=1
-      export kubever=$(kubectl version --kubeconfig /etc/kubernetes/admin.conf | base64 | tr -d '\n')
-      kubectl apply --kubeconfig /etc/kubernetes/admin.conf -f "https://cloud.weave.works/k8s/net?k8s-version=$kubever"
-      for tries in $(seq 1 60); do
-      	kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
-      	sleep 1
-      done
-      echo done.
-      ) 2>&1 | tee /var/log/startup.log
-# These configs currently don't work - they need service accounts.
 - machineParams:
   - os: ubuntu-1604-lts
     roles:
@@ -179,10 +23,21 @@ items:
       apt-get install -y \
           socat \
           ebtables \
-          docker.io \
           apt-transport-https \
           cloud-utils \
           prips
+      function install_configure_docker () {
+          # prevent docker from auto-starting
+          echo "exit 101" > /usr/sbin/policy-rc.d
+          chmod +x /usr/sbin/policy-rc.d
+          trap "rm /usr/sbin/policy-rc.d" RETURN
+          apt install -y docker.io
+          echo 'DOCKER_OPTS="--iptables=false --ip-masq=false"' > /etc/default/docker
+          systemctl daemon-reload
+          systemctl enable docker
+          systemctl start docker
+      }
+      install_configure_docker
       curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl
       chmod a+rx /usr/bin/kubeadm.dl
       # kubeadm uses 10th IP as DNS server
@@ -206,8 +61,6 @@ items:
           kubeadm=${KUBEADM}
       mv /usr/bin/kubeadm.dl /usr/bin/kubeadm
       chmod a+rx /usr/bin/kubeadm
-      systemctl enable docker
-      systemctl start docker
       # Override network args to use kubenet instead of cni, and override Kubelet DNS args.
       cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<EOF
       [Service]
@@ -247,13 +100,12 @@ items:
       - ${PUBLICIP}
       - ${PRIVATEIP}
       EOF
-      kubeadm init --config /etc/kubernetes/kubeadm_config.yaml
+      kubeadm init --config /etc/kubernetes/kubeadm_config.yaml --ignore-preflight-errors=all
       for tries in $(seq 1 60); do
           kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
           sleep 1
       done
       echo done.
-      # kubectl create configmap cloud-config --namespace=kube-system --from-file=./gce.conf
       ) 2>&1 | tee /var/log/startup.log
 - machineParams:
   - os: ubuntu-1604-lts

From e391bd48e0261f16883acca9d47f680fc1f9249f Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Mon, 23 Apr 2018 13:30:30 -0700
Subject: [PATCH 02/11] Remove --ignore-preflight-errors=all flag

The flag was replaced with a workaround to create the missing file that
triggered the error. The cluster was functioning when all preflight
errors were ignored, but in case another error comes up in the future,
we don't want to accidentally ignore it.
---
 gcp-deployer/machine_setup_configs.yaml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/gcp-deployer/machine_setup_configs.yaml b/gcp-deployer/machine_setup_configs.yaml
index f9c4366c7214..db409bb7ed9b 100644
--- a/gcp-deployer/machine_setup_configs.yaml
+++ b/gcp-deployer/machine_setup_configs.yaml
@@ -26,6 +26,7 @@ items:
           apt-transport-https \
           cloud-utils \
           prips
+
       function install_configure_docker () {
           # prevent docker from auto-starting
           echo "exit 101" > /usr/sbin/policy-rc.d
@@ -38,6 +39,7 @@ items:
           systemctl start docker
       }
       install_configure_docker
+
       curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl
       chmod a+rx /usr/bin/kubeadm.dl
       # kubeadm uses 10th IP as DNS server
@@ -61,6 +63,7 @@ items:
           kubeadm=${KUBEADM}
       mv /usr/bin/kubeadm.dl /usr/bin/kubeadm
       chmod a+rx /usr/bin/kubeadm
+
       # Override network args to use kubenet instead of cni, and override Kubelet DNS args.
       cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<EOF
       [Service]
@@ -72,6 +75,7 @@ items:
       PRIVATEIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip"`
       echo $PRIVATEIP > /tmp/.ip
       PUBLICIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"`
+
       # Set up the GCE cloud config, which gets picked up by kubeadm init since cloudProvider is set to GCE.
       cat > /etc/kubernetes/cloud-config <<EOF
       [global]
@@ -80,6 +84,7 @@ items:
       subnetwork-name = ${SUBNETWORK}
       node-tags = ${NODE_TAG}
       EOF
+
       # Set up kubeadm config file to pass parameters to kubeadm init.
       cat > /etc/kubernetes/kubeadm_config.yaml <<EOF
       apiVersion: kubeadm.k8s.io/v1alpha1
@@ -100,7 +105,13 @@ items:
       - ${PUBLICIP}
       - ${PRIVATEIP}
       EOF
-      kubeadm init --config /etc/kubernetes/kubeadm_config.yaml --ignore-preflight-errors=all
+
+      # Create and set bridge-nf-call-iptables to 1 to pass the kubeadm preflight check.
+      # Workaround was found here:
+      # http://zeeshanali.com/sysadmin/fixed-sysctl-cannot-stat-procsysnetbridgebridge-nf-call-iptables/
+      modprobe br_netfilter
+
+      kubeadm init --config /etc/kubernetes/kubeadm_config.yaml
       for tries in $(seq 1 60); do
           kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
           sleep 1

From ee2219e8ab81a6828dd1fa4496c7c33b3acfcd9b Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Mon, 23 Apr 2018 15:26:37 -0700
Subject: [PATCH 03/11] Fix issue with assigning conflicting IPs to pods

---
 gcp-deployer/machine_setup_configs.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gcp-deployer/machine_setup_configs.yaml b/gcp-deployer/machine_setup_configs.yaml
index db409bb7ed9b..e20f3a4d8d39 100644
--- a/gcp-deployer/machine_setup_configs.yaml
+++ b/gcp-deployer/machine_setup_configs.yaml
@@ -67,7 +67,7 @@ items:
       # Override network args to use kubenet instead of cni, and override Kubelet DNS args.
       cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<EOF
       [Service]
-      Environment="KUBELET_NETWORK_ARGS=--network-plugin=kubenet --pod-cidr=${POD_CIDR}"
+      Environment="KUBELET_NETWORK_ARGS=--network-plugin=kubenet"
       Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
       EOF
       systemctl daemon-reload
@@ -169,7 +169,7 @@ items:
       # Override network args to use kubenet instead of cni, and override Kubelet DNS args.
       cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<EOF
       [Service]
-      Environment="KUBELET_NETWORK_ARGS=--network-plugin=kubenet --pod-cidr=${POD_CIDR}"
+      Environment="KUBELET_NETWORK_ARGS=--network-plugin=kubenet"
       Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
       EOF
       systemctl daemon-reload

From d03a96fe91bc0ac4b3d99240fa3fd124673fd310 Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Thu, 26 Apr 2018 11:51:54 -0700
Subject: [PATCH 04/11] Add check to only add service account to masters

---
 cloud/google/machineactuator.go | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go
index 483e2dbdc736..2f06cd8c4b47 100644
--- a/cloud/google/machineactuator.go
+++ b/cloud/google/machineactuator.go
@@ -270,6 +270,16 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 		if !util.IsMaster(machine) {
 			tags = append(tags, fmt.Sprintf("%s-worker", cluster.Name))
 		}
+		serviceAccounts := []*compute.ServiceAccount{nil}
+		if util.IsMaster(machine) {
+			serviceAccounts = append(serviceAccounts,
+				&compute.ServiceAccount{
+					Email: "default",
+					Scopes: []string{
+						"https://www.googleapis.com/auth/cloud-platform",
+					},
+				})
+		}
 
 		op, err := gce.computeService.InstancesInsert(project, zone, &compute.Instance{
 			Name:        name,
@@ -302,15 +312,8 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 			Tags: &compute.Tags{
 				Items: tags,
 			},
-			Labels: labels,
-			ServiceAccounts: []*compute.ServiceAccount{
-				{
-					Email: "default",
-					Scopes: []string{
-						"https://www.googleapis.com/auth/cloud-platform",
-					},
-				},
-			},
+			Labels:          labels,
+			ServiceAccounts: serviceAccounts,
 		})
 
 		if err == nil {

From d30c4f5a90e90d704d0f1cfec534ca29470a59be Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Thu, 26 Apr 2018 14:13:25 -0700
Subject: [PATCH 05/11] Use apt-get instead of apt in scripts

---
 gcp-deployer/machine_setup_configs.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gcp-deployer/machine_setup_configs.yaml b/gcp-deployer/machine_setup_configs.yaml
index e20f3a4d8d39..9a861caccdd3 100644
--- a/gcp-deployer/machine_setup_configs.yaml
+++ b/gcp-deployer/machine_setup_configs.yaml
@@ -32,7 +32,7 @@ items:
           echo "exit 101" > /usr/sbin/policy-rc.d
           chmod +x /usr/sbin/policy-rc.d
           trap "rm /usr/sbin/policy-rc.d" RETURN
-          apt install -y docker.io
+          apt-get install -y docker.io
           echo 'DOCKER_OPTS="--iptables=false --ip-masq=false"' > /etc/default/docker
           systemctl daemon-reload
           systemctl enable docker

From af87544242a62af9767c801c670408ce9f00813a Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Fri, 27 Apr 2018 11:03:40 -0700
Subject: [PATCH 06/11] Remove ServiceAccountActor role from machine controller

---
 cloud/google/serviceaccount.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/cloud/google/serviceaccount.go b/cloud/google/serviceaccount.go
index 935e561ea892..6844194ad68a 100644
--- a/cloud/google/serviceaccount.go
+++ b/cloud/google/serviceaccount.go
@@ -65,10 +65,6 @@ func (gce *GCEClient) CreateMachineControllerServiceAccount(cluster *clusterv1.C
 		if err != nil {
 			return fmt.Errorf("couldn't grant permissions to service account: %v", err)
 		}
-		err = run("gcloud", "projects", "add-iam-policy-binding", project, "--member=serviceAccount:"+email, "--role=roles/iam.serviceAccountActor")
-		if err != nil {
-			return fmt.Errorf("couldn't grant permissions to service account: %v", err)
-		}
 	}
 
 	err = run("gcloud", "--project", project, "iam", "service-accounts", "keys", "create", localFile, "--iam-account", email)

From 0ab825271a9200a6962cc742d4b3d26b7155ca50 Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Mon, 30 Apr 2018 09:27:45 -0700
Subject: [PATCH 07/11] Add worker tag to master to make it schedulable

---
 cloud/google/machineactuator.go | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go
index 2f06cd8c4b47..9aad5d344e13 100644
--- a/cloud/google/machineactuator.go
+++ b/cloud/google/machineactuator.go
@@ -40,13 +40,13 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/cluster-api/cloud/google/clients"
 	gceconfig "sigs.k8s.io/cluster-api/cloud/google/gceproviderconfig"
 	gceconfigv1 "sigs.k8s.io/cluster-api/cloud/google/gceproviderconfig/v1alpha1"
 	"sigs.k8s.io/cluster-api/cloud/google/machinesetup"
 	apierrors "sigs.k8s.io/cluster-api/errors"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 	client "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1"
-	"sigs.k8s.io/cluster-api/cloud/google/clients"
 	"sigs.k8s.io/cluster-api/util"
 )
 
@@ -78,12 +78,12 @@ type GCEClientComputeService interface {
 
 type GCEClient struct {
 	computeService GCEClientComputeService
-	scheme        *runtime.Scheme
-	codecFactory  *serializer.CodecFactory
-	kubeadmToken  string
-	sshCreds      SshCreds
-	machineClient client.MachineInterface
-	configWatch   *machinesetup.ConfigWatch
+	scheme         *runtime.Scheme
+	codecFactory   *serializer.CodecFactory
+	kubeadmToken   string
+	sshCreds       SshCreds
+	machineClient  client.MachineInterface
+	configWatch    *machinesetup.ConfigWatch
 }
 
 const (
@@ -266,10 +266,6 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 		if gce.machineClient == nil {
 			labels[BootstrapLabelKey] = "true"
 		}
-		tags := []string{"https-server"}
-		if !util.IsMaster(machine) {
-			tags = append(tags, fmt.Sprintf("%s-worker", cluster.Name))
-		}
 		serviceAccounts := []*compute.ServiceAccount{nil}
 		if util.IsMaster(machine) {
 			serviceAccounts = append(serviceAccounts,
@@ -282,8 +278,8 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 		}
 
 		op, err := gce.computeService.InstancesInsert(project, zone, &compute.Instance{
-			Name:        name,
-			MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, config.MachineType),
+			Name:         name,
+			MachineType:  fmt.Sprintf("zones/%s/machineTypes/%s", zone, config.MachineType),
 			CanIpForward: true,
 			NetworkInterfaces: []*compute.NetworkInterface{
 				{
@@ -310,7 +306,9 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 				Items: metadataItems,
 			},
 			Tags: &compute.Tags{
-				Items: tags,
+				Items: []string{
+					"https-server",
+					fmt.Sprintf("%s-worker", cluster.Name)},
 			},
 			Labels:          labels,
 			ServiceAccounts: serviceAccounts,

From f705c9a605d5a96db21eaa609d8ba3a2df646c71 Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Mon, 30 Apr 2018 10:52:26 -0700
Subject: [PATCH 08/11] Add docker change to nodes to prevent auto-start

---
 gcp-deployer/machine_setup_configs.yaml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/gcp-deployer/machine_setup_configs.yaml b/gcp-deployer/machine_setup_configs.yaml
index 9a861caccdd3..21f97fa8842f 100644
--- a/gcp-deployer/machine_setup_configs.yaml
+++ b/gcp-deployer/machine_setup_configs.yaml
@@ -140,7 +140,20 @@ items:
       deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main
       EOF
       apt-get update
-      apt-get install -y docker-engine=1.12.0-0~xenial
+
+      function install_configure_docker () {
+          # prevent docker from auto-starting
+          echo "exit 101" > /usr/sbin/policy-rc.d
+          chmod +x /usr/sbin/policy-rc.d
+          trap "rm /usr/sbin/policy-rc.d" RETURN
+          apt-get install -y docker-engine=1.12.0-0~xenial
+          echo 'DOCKER_OPTS="--iptables=false --ip-masq=false"' > /etc/default/docker
+          systemctl daemon-reload
+          systemctl enable docker
+          systemctl start docker
+      }
+      install_configure_docker
+
       curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
       cat <<EOF > /etc/apt/sources.list.d/kubernetes.list
       deb http://apt.kubernetes.io/ kubernetes-xenial main

From fc939137c5d46dd6fb2d750615fae67ef497948c Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Mon, 30 Apr 2018 10:54:10 -0700
Subject: [PATCH 09/11] Update machines yaml template to use kubenet configs

---
 gcp-deployer/machines.yaml.template | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gcp-deployer/machines.yaml.template b/gcp-deployer/machines.yaml.template
index 62c0edeb921b..3575f585ba08 100644
--- a/gcp-deployer/machines.yaml.template
+++ b/gcp-deployer/machines.yaml.template
@@ -13,7 +13,7 @@ items:
         project: "$GCLOUD_PROJECT"
         zone: "$ZONE"
         machineType: "n1-standard-2"
-        os: "ubuntu-1710-weave"
+        os: "ubuntu-1604-lts"
     versions:
       kubelet: 1.9.4
       controlPlane: 1.9.4
@@ -36,7 +36,7 @@ items:
         project: "$GCLOUD_PROJECT"
         zone: "$ZONE"
         machineType: "n1-standard-1"
-        os: "ubuntu-1710-weave"
+        os: "ubuntu-1604-lts"
     versions:
       kubelet: 1.9.4
       containerRuntime:

From 777f87293c6cebed446a3069b5179193406f1bfc Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Mon, 30 Apr 2018 10:58:25 -0700
Subject: [PATCH 10/11] Revert support for load balancing

These changes will be in a separate PR.
---
 cloud/google/machineactuator.go | 4 +---
 cloud/google/metadata.go        | 5 ++---
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go
index 9aad5d344e13..f4e39f4cfaaf 100644
--- a/cloud/google/machineactuator.go
+++ b/cloud/google/machineactuator.go
@@ -306,9 +306,7 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 				Items: metadataItems,
 			},
 			Tags: &compute.Tags{
-				Items: []string{
-					"https-server",
-					fmt.Sprintf("%s-worker", cluster.Name)},
+				Items: []string{"https-server"},
 			},
 			Labels:          labels,
 			ServiceAccounts: serviceAccounts,
diff --git a/cloud/google/metadata.go b/cloud/google/metadata.go
index dc90abc16630..2469b0552ed3 100644
--- a/cloud/google/metadata.go
+++ b/cloud/google/metadata.go
@@ -95,7 +95,7 @@ func init() {
 	nodeEnvironmentVarsTemplate = template.Must(template.New("nodeEnvironmentVars").Parse(nodeEnvironmentVars))
 }
 
-// TODO(kcoronado): replace with actual network args when they are added into provider config.
+// TODO(kcoronado): replace with actual network and node tag args when they are added into provider config.
 const masterEnvironmentVars = `
 #!/bin/bash
 KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }}
@@ -111,8 +111,7 @@ SERVICE_CIDR={{ .ServiceCIDR }}
 PROJECT={{ .Project }}
 NETWORK=default
 SUBNETWORK=kubernetes
-CLUSTER_NAME={{ .Cluster.Name }}
-NODE_TAG="$CLUSTER_NAME-worker"
+NODE_TAG=worker
 `
 
 const nodeEnvironmentVars = `

From 78cd0e2be44c92c2cafcef58229b48d95649c50e Mon Sep 17 00:00:00 2001
From: Katie Coronado <kcoronado@google.com>
Date: Mon, 30 Apr 2018 11:13:41 -0700
Subject: [PATCH 11/11] Add comment and remove extra enable/start docker in
 node script

---
 cloud/google/machineactuator.go         | 2 ++
 gcp-deployer/machine_setup_configs.yaml | 2 --
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cloud/google/machineactuator.go b/cloud/google/machineactuator.go
index f4e39f4cfaaf..60f8df6d4122 100644
--- a/cloud/google/machineactuator.go
+++ b/cloud/google/machineactuator.go
@@ -266,6 +266,8 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
 		if gce.machineClient == nil {
 			labels[BootstrapLabelKey] = "true"
 		}
+
+		// The service account is needed for the Kubernetes GCE cloud provider code. It is needed on the master VM.
 		serviceAccounts := []*compute.ServiceAccount{nil}
 		if util.IsMaster(machine) {
 			serviceAccounts = append(serviceAccounts,
diff --git a/gcp-deployer/machine_setup_configs.yaml b/gcp-deployer/machine_setup_configs.yaml
index 21f97fa8842f..92e5d05073bd 100644
--- a/gcp-deployer/machine_setup_configs.yaml
+++ b/gcp-deployer/machine_setup_configs.yaml
@@ -175,8 +175,6 @@ items:
       KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-)
       KUBECTL=$(getversion kubectl ${KUBELET_VERSION}-)
       apt-get install -y kubelet=${KUBELET} kubeadm=${KUBEADM} kubectl=${KUBECTL}
-      systemctl enable docker || true
-      systemctl start docker || true
       # kubeadm uses 10th IP as DNS server
       CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1)
       # Override network args to use kubenet instead of cni, and override Kubelet DNS args.