Ubuntu 18.04 & Kubernetes 1.13.2 & etcd 3.3.11 & calico & containerd

cmd/cluster_create.go

@@ -52,6 +52,7 @@ func RunClusterCreate(cmd *cobra.Command, args []string) {
 	if isolatedEtcd {
 		etcdCount, _ = cmd.Flags().GetInt("etcd-count")
 	}
+	debug, _ := cmd.Flags().GetBool("debug")
 
 	clusterName := randomName()
 	if name, _ := cmd.Flags().GetString("name"); name != "" {
@@ -73,7 +74,7 @@ func RunClusterCreate(cmd *cobra.Command, args []string) {
 		CloudInitFile: cloudInit,
 	}, AppConf.CurrentContext.Token)
 
-	sshClient := clustermanager.NewSSHCommunicator(AppConf.Config.SSHKeys)
+	sshClient := clustermanager.NewSSHCommunicator(AppConf.Config.SSHKeys, debug)
 	err := sshClient.(*clustermanager.SSHCommunicator).CapturePassphrase(sshKeyName)
 	FatalOnError(err)
 
@@ -164,7 +165,7 @@ func saveCluster(cluster *clustermanager.Cluster) {
 
 func renderProgressBars(cluster *clustermanager.Cluster, coordinator *pkg.UIProgressCoordinator) {
 	nodes := cluster.Nodes
-	provisionSteps := 5
+	provisionSteps := 8
 	netWorkSetupSteps := 2
 	etcdSteps := 4
 	masterInstallSteps := 2

cmd/config.go

@@ -41,7 +41,7 @@ func NewAppConfig() AppConfig {
 	}
 
 	makeConfigIfNotExists(&appConf)
-	appConf.SSHClient = clustermanager.NewSSHCommunicator(appConf.Config.SSHKeys)
+	appConf.SSHClient = clustermanager.NewSSHCommunicator(appConf.Config.SSHKeys, true)
 	return appConf
 }
 

pkg/clustermanager/cluster.go

@@ -129,8 +129,16 @@ func (manager *Manager) SetupEncryptedNetwork() error {
 				errChan <- err
 			}
 
-			_, err = manager.nodeCommunicator.RunCmd(node, "systemctl enable wg-quick@wg0 && systemctl restart wg-quick@wg0")
+			overlayRouteConf := GenerateOverlayRouteSystemdService(node)
+			err = manager.nodeCommunicator.WriteFile(node, "/etc/systemd/system/overlay-route.service", overlayRouteConf, false)
+			if err != nil {
+				errChan <- err
+			}
 
+			_, err = manager.nodeCommunicator.RunCmd(
+				node,
+				"systemctl enable wg-quick@wg0 && systemctl restart wg-quick@wg0"+
+					" && systemctl enable overlay-route.service && systemctl restart overlay-route.service")
 			if err != nil {
 				errChan <- err
 			}
@@ -150,12 +158,14 @@ func (manager *Manager) SetupEncryptedNetwork() error {
 
 // InstallMasters installs the kubernetes control plane to master nodes
 func (manager *Manager) InstallMasters() error {
-
 	commands := []NodeCommand{
-		{"kubeadm init", "kubeadm init --config /root/master-config.yaml"},
+		{"kubeadm init", "kubectl version > /dev/null &> /dev/null || kubeadm init --ignore-preflight-errors=all --config /root/master-config.yaml"},
 		{"configure kubectl", "rm -rf $HOME/.kube && mkdir -p $HOME/.kube && cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config"},
-		{"install flannel", "kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml"},
-		{"configure flannel", "kubectl -n kube-system patch ds kube-flannel-ds --type json -p '[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\",\"value\":\"true\",\"effect\":\"NoSchedule\"}}]'"},
+		//{"install Weave Net", "kubectl apply -f \"https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\\n')\""},
+		{"install canal (RBAC)", "kubectl apply -f https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/canal/rbac.yaml"},
+		{"install canal", "kubectl apply -f https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/canal/canal.yaml"},
+		//{"install flannel", "kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"},
+		//{"configure flannel", "kubectl -n kube-system patch ds kube-flannel-ds --type json -p '[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\",\"value\":\"true\",\"effect\":\"NoSchedule\"}}]'"},
 		//{"install hcloud integration", fmt.Sprintf("kubectl -n kube-system create secret generic hcloud --from-literal=token=%s", AppConf.CurrentContext.Token)},
 		//{"deploy cloud controller manager", "kubectl apply -f  https://raw.githubusercontent.com/hetznercloud/hcloud-cloud-controller-manager/master/deploy/v1.0.0.yaml"},
 	}
@@ -172,14 +182,14 @@ func (manager *Manager) InstallMasters() error {
 
 	for _, node := range manager.nodes {
 		if node.IsMaster {
-			_, err := manager.nodeCommunicator.RunCmd(node, "kubeadm reset")
+			_, err := manager.nodeCommunicator.RunCmd(node, "kubeadm reset -f")
 			if err != nil {
-				return nil
+				return err
 			}
 
 			_, err = manager.nodeCommunicator.RunCmd(node, "rm -rf /etc/kubernetes/pki && mkdir /etc/kubernetes/pki")
 			if err != nil {
-				return nil
+				return err
 			}
 			if len(manager.nodes) == 1 {
 				commands = append(commands, NodeCommand{"taint master", "kubectl taint nodes --all node-role.kubernetes.io/master-"})
@@ -212,7 +222,6 @@ func (manager *Manager) InstallMasters() error {
 
 // installs kubernetes control plane to a given node
 func (manager *Manager) installMasterStep(node Node, numMaster int, masterNode Node, commands []NodeCommand, trueChan chan bool, errChan chan error) {
-
 	// create master-configuration
 	var etcdNodes []Node
 	if manager.haEnabled {
@@ -275,10 +284,9 @@ func (manager *Manager) installMasterStep(node Node, numMaster int, masterNode N
 
 // InstallEtcdNodes installs the etcd cluster
 func (manager *Manager) InstallEtcdNodes(nodes []Node) error {
-
 	commands := []NodeCommand{
-		{"download etcd", "mkdir -p /opt/etcd && curl -L https://storage.googleapis.com/etcd/v3.2.13/etcd-v3.2.13-linux-amd64.tar.gz -o /opt/etcd-v3.2.13-linux-amd64.tar.gz"},
-		{"install etcd", "tar xzvf /opt/etcd-v3.2.13-linux-amd64.tar.gz -C /opt/etcd --strip-components=1"},
+		{"download etcd", "mkdir -p /opt/etcd && curl -L https://storage.googleapis.com/etcd/v3.3.11/etcd-v3.3.11-linux-amd64.tar.gz -o /opt/etcd-v3.3.11-linux-amd64.tar.gz"},
+		{"install etcd", "tar xzvf /opt/etcd-v3.3.11-linux-amd64.tar.gz -C /opt/etcd --strip-components=1"},
 		{"configure etcd", "systemctl enable etcd.service && systemctl stop etcd.service && rm -rf /var/lib/etcd && systemctl start etcd.service"},
 	}
 
@@ -328,6 +336,7 @@ func (manager *Manager) InstallWorkers(nodes []Node) error {
 	if err != nil {
 		return err
 	}
+	joinCommand = fmt.Sprintf("%s --cri-socket /var/run/docker/containerd/docker-containerd.sock", strings.TrimRight(joinCommand, "\n"))
 
 	errChan := make(chan error)
 	trueChan := make(chan bool)
@@ -338,7 +347,10 @@ func (manager *Manager) InstallWorkers(nodes []Node) error {
 			numProcs++
 			go func(node Node) {
 				manager.eventService.AddEvent(node.Name, "registering node")
-				_, err := manager.nodeCommunicator.RunCmd(node, "kubeadm reset && "+joinCommand)
+				_, err := manager.nodeCommunicator.RunCmd(
+					node,
+					"for i in ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack_ipv4; do modprobe $i; done"+
+						" && kubeadm reset -f && "+joinCommand)
 				if err != nil {
 					errChan <- err
 				}

pkg/clustermanager/configs.go

@@ -7,30 +7,53 @@ import (
 
 // GenerateMasterConfiguration generate the kubernetes config for master
 func GenerateMasterConfiguration(masterNode Node, masterNodes, etcdNodes []Node) string {
-	masterConfigTpl := `apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-api:
-  advertiseAddress: %s
+	masterConfigTpl := `apiVersion: kubeadm.k8s.io/v1alpha3
+kind: ClusterConfiguration
 networking:
-  podSubnet: 10.244.0.0/16
+  serviceSubnet: "10.96.0.0/12"
+  podSubnet: "10.244.0.0/16"
+  dnsDomain: "cluster.local"
 apiServerCertSANs:
-  - %s
   - 127.0.0.1
+  - %s
+%s%s
+---
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: InitConfiguration
+apiEndpoint:
+  advertiseAddress: %s
+  bindPort: 6443
+nodeRegistration:
+  criSocket: /var/run/docker/containerd/docker-containerd.sock
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/master
 `
-	etcdConfig := `etcd:
-  endpoints:`
-	masterConfig := fmt.Sprintf(masterConfigTpl, masterNode.PrivateIPAddress, masterNode.IPAddress)
+
+	masterNodesIps := ""
 	for _, node := range masterNodes {
-		masterConfig = fmt.Sprintf("%s%s\n", masterConfig, "  - "+node.PrivateIPAddress)
+		masterNodesIps = fmt.Sprintf("%s  - %s\n", masterNodesIps, node.PrivateIPAddress)
 	}
 
+	etcdConfig := ""
 	if len(etcdNodes) > 0 {
-		masterConfig = masterConfig + etcdConfig + "\n"
+		etcdConfig = `etcd:
+  external:
+    endpoints:` + "\n"
+
 		for _, node := range etcdNodes {
-			masterConfig = fmt.Sprintf("%s%s\n", masterConfig, "  - http://"+node.PrivateIPAddress+":2379")
+			etcdConfig = fmt.Sprintf("%s%s\n", etcdConfig, "    - http://"+node.PrivateIPAddress+":2379")
 		}
 	}
 
+	masterConfig := fmt.Sprintf(
+		masterConfigTpl,
+		masterNode.IPAddress,
+		masterNodesIps,
+		etcdConfig,
+		masterNode.PrivateIPAddress,
+	)
+
 	return masterConfig
 }
 

pkg/clustermanager/configs_test.go

@@ -7,34 +7,59 @@ import (
 )
 
 func TestGenerateMasterConfiguration(t *testing.T) {
-	expectedConf := `apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-api:
-  advertiseAddress: 10.0.0.1
+	expectedConf := `apiVersion: kubeadm.k8s.io/v1alpha3
+kind: ClusterConfiguration
 networking:
-  podSubnet: 10.244.0.0/16
+  serviceSubnet: "10.96.0.0/12"
+  podSubnet: "10.244.0.0/16"
+  dnsDomain: "cluster.local"
 apiServerCertSANs:
-  - 1.1.1.1
   - 127.0.0.1
+  - 1.1.1.1
   - 10.0.0.1
   - 10.0.0.2
-`
 
-	expectedConfWithEtcd := `apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-api:
+---
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: InitConfiguration
+apiEndpoint:
   advertiseAddress: 10.0.0.1
+  bindPort: 6443
+nodeRegistration:
+  criSocket: /var/run/docker/containerd/docker-containerd.sock
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/master
+`
+
+	expectedConfWithEtcd := `apiVersion: kubeadm.k8s.io/v1alpha3
+kind: ClusterConfiguration
 networking:
-  podSubnet: 10.244.0.0/16
+  serviceSubnet: "10.96.0.0/12"
+  podSubnet: "10.244.0.0/16"
+  dnsDomain: "cluster.local"
 apiServerCertSANs:
-  - 1.1.1.1
   - 127.0.0.1
+  - 1.1.1.1
   - 10.0.0.1
   - 10.0.0.2
 etcd:
-  endpoints:
-  - http://10.0.0.1:2379
-  - http://10.0.0.2:2379
+  external:
+    endpoints:
+    - http://10.0.0.1:2379
+    - http://10.0.0.2:2379
+
+---
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: InitConfiguration
+apiEndpoint:
+  advertiseAddress: 10.0.0.1
+  bindPort: 6443
+nodeRegistration:
+  criSocket: /var/run/docker/containerd/docker-containerd.sock
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/master
 `
 	nodes := []Node{
 		{Name: "node1", IPAddress: "1.1.1.1", PrivateIPAddress: "10.0.0.1"},
@@ -44,15 +69,14 @@ etcd:
 	noEtcdConf := GenerateMasterConfiguration(nodes[0], nodes, nil)
 
 	if noEtcdConf != expectedConf {
-		t.Errorf("master config without etcd does not match to expected.\n%s\n", noEtcdConf)
+		t.Errorf("master config without etcd does not match to expected.\n%s\n", diff.LineDiff(noEtcdConf, expectedConf))
 	}
 
 	etcdConf := GenerateMasterConfiguration(nodes[0], nodes, nodes)
 
 	if etcdConf != expectedConfWithEtcd {
 		t.Errorf("master config with etcd does not match to expected.\n%s\n", diff.LineDiff(etcdConf, expectedConfWithEtcd))
 	}
-
 }
 
 func TestGenerateEtcdSystemdService(t *testing.T) {
@@ -90,5 +114,4 @@ WantedBy=multi-user.target
 	if etcdService != expectedString {
 		t.Errorf("etcd systemd service does not match expected\n%s", diff.LineDiff(expectedString, etcdService))
 	}
-
 }

pkg/clustermanager/provision_node.go

@@ -10,8 +10,7 @@ import (
 const maxErrors = 3
 
 // K8sVersion is the version that will be used to install kubernetes
-var K8sVersion = flag.String("k8s-version", "1.9.11-00",
-	"The version of the k8s debian packages that will be used during provisioning")
+var K8sVersion = flag.String("k8s-version", "1.13.2-00", "The version of the k8s debian packages that will be used during provisioning")
 
 // NodeProvisioner provisions all basic packages to install docker, kubernetes and wireguard
 type NodeProvisioner struct {
@@ -51,7 +50,8 @@ func (provisioner *NodeProvisioner) Provision(node Node, communicator NodeCommun
 	}
 
 	eventService.AddEvent(node.Name, "packages installed")
-	return nil
+
+	return provisioner.disableSwap()
 }
 
 func (provisioner *NodeProvisioner) packagesAreInstalled(node Node, communicator NodeCommunicator) bool {
@@ -84,6 +84,18 @@ func (provisioner *NodeProvisioner) prepareAndInstall() error {
 	return nil
 }
 
+func (provisioner *NodeProvisioner) disableSwap() error {
+	provisioner.eventService.AddEvent(provisioner.node.Name, "disabling swap")
+
+	_, err := provisioner.communicator.RunCmd(provisioner.node, "swapoff -a")
+	if err != nil {
+		return err
+	}
+
+	_, err = provisioner.communicator.RunCmd(provisioner.node, "sed -i '/ swap / s/^/#/' /etc/fstab")
+	return err
+}
+
 func (provisioner *NodeProvisioner) installTransportTools() error {
 
 	provisioner.eventService.AddEvent(provisioner.node.Name, "installing transport tools")
@@ -139,7 +151,7 @@ func (provisioner *NodeProvisioner) prepareDocker() error {
 	// docker-ce
 	aptPreferencesDocker := `
 Package: docker-ce
-Pin: version 17.03.*
+Pin: version 18.06.0~ce~3-0~ubuntu
 Pin-Priority: 1000
 	`
 	err := provisioner.communicator.WriteFile(provisioner.node, "/etc/apt/preferences.d/docker-ce", aptPreferencesDocker, false)
@@ -168,7 +180,7 @@ func (provisioner *NodeProvisioner) updateAndInstall() error {
 	}
 
 	provisioner.eventService.AddEvent(provisioner.node.Name, "installing packages")
-	command := fmt.Sprintf("apt-get install -y docker-ce kubelet=%s kubeadm=%s kubectl=%s wireguard linux-headers-$(uname -r) linux-headers-virtual",
+	command := fmt.Sprintf("apt-get install -y docker-ce kubelet=%s kubeadm=%s kubectl=%s kubernetes-cni wireguard linux-headers-$(uname -r) linux-headers-virtual",
 		*K8sVersion, *K8sVersion, *K8sVersion)
 	_, err = provisioner.communicator.RunCmd(provisioner.node, command)
 	if err != nil {