fix: Switch to containerd and pin k8s version

terraform/modules/AWS/Kubernetes/master-cloud-config.tf

@@ -7,6 +7,6 @@ data "template_file" "master_cloud_config" {
     master_bashrc  = filebase64("${path.module}/bashrc")
     master_inputrc = filebase64("${path.module}/inputrc")
     master_aliases = filebase64("${path.module}/bash_aliases")
+    version        = var.kubernetes_version
   }
 }
-

terraform/modules/AWS/Kubernetes/master-cloud-config.yaml

@@ -46,29 +46,27 @@ write_files:
 
 
 runcmd:
+  - 'echo "br_netfilter" >> /etc/modules-load.d/k8s.conf && modprobe br_netfilter'
+  - 'echo "net.bridge.bridge-nf-call-iptables = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'echo "net.bridge.bridge-nf-call-ip6tables = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'sysctl --load=/etc/sysctl.d/k8s.conf'
   - 'curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -'
   - 'echo "deb https://apt.kubernetes.io/ kubernetes-xenial main"  > /etc/apt/sources.list.d/kubernetes.list'
-  - 'mkdir /run/download'
   - 'apt update'
-  - 'apt install -y kubelet kubeadm kubectl docker.io awscli'
-  - 'wget https://github.com/kubernetes-incubator/cri-tools/releases/download/v1.11.1/crictl-v1.11.1-linux-amd64.tar.gz -O /run/download/crictl.tgz'
-  - 'tar -C /usr/bin -xzf /run/download/crictl.tgz'
-  - 'chmod 754 /usr/bin/crictl'
-  - 'chown root:root /usr/bin/crictl'
-  - 'systemctl enable docker'
-  - 'systemctl daemon-reload'
-  - 'systemctl restart docker'
-  - 'systemctl restart kubelet'
-  - 'wget https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml -O /run/download/rbac-kdd.yaml'
-  - 'wget https://docs.projectcalico.org/v3.9/manifests/calico.yaml -O /run/download/calico.yaml'
-  - 'kubeadm init --pod-network-cidr=192.168.0.0/16'
+  - 'apt install -y containerd kubelet=${version}-00 kubeadm=${version}-00 kubectl=${version}-00 awscli'
+  - 'systemctl enable --now containerd'
+  - 'systemctl enable --now kubelet'
+  - 'kubeadm init --pod-network-cidr=192.168.0.0/16 --kubernetes-version=v${version}'
   - 'mkdir /root/.kube'
   - 'cp /etc/kubernetes/admin.conf /root/.kube/config'
-  - 'kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /run/download/rbac-kdd.yaml'
-  - 'kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /run/download/calico.yaml'
-  - "egrep -A 1 'kubeadm join' /var/log/cloud-init-output.log  |tr -d '\\' | tr -d '\n' > /tmp/join.txt"
+  - 'kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://projectcalico.docs.tigera.io/manifests/calico.yaml' # needs to be versioned
+  - "grep -A 1 'kubeadm join' /var/log/cloud-init-output.log | tr -d '\\\n\r\t' > /tmp/join.txt"
   - 'aws s3 cp /tmp/join.txt s3://${s3_bucket_name}'
   - 'aws s3 cp /root/.kube/config s3://${s3_bucket_name}'
+  - 'curl -L https://github.com/containerd/nerdctl/releases/download/v0.17.0/nerdctl-0.17.0-linux-amd64.tar.gz -o /tmp/nerdctl.tar.gz'
+  - 'tar -xf /tmp/nerdctl.tar.gz nerdctl -O > /tmp/nerdctl'
+  - 'install /tmp/nerdctl /usr/local/bin/'
 
 output:
   all: '| tee -a /var/log/cloud-init-output.log'

terraform/modules/AWS/Kubernetes/master-goss.yaml

@@ -17,16 +17,12 @@ port:
     ip:
     - '::'
 service:
-  docker:
+  containerd:
     enabled: true
     running: true
   kubelet:
     enabled: true
     running: true
-group:
-  docker:
-    exists: true
-    gid: 115
 process:
   kubelet:
     running: true

terraform/modules/AWS/Kubernetes/node-cloud-config.tf

@@ -7,6 +7,6 @@ data "template_file" "node_cloud_config" {
     node_bashrc    = filebase64("${path.module}/bashrc")
     node_inputrc   = filebase64("${path.module}/inputrc")
     node_aliases   = filebase64("${path.module}/bash_aliases")
+    version        = var.kubernetes_version
   }
 }
-

terraform/modules/AWS/Kubernetes/node-cloud-config.yaml

@@ -45,21 +45,22 @@ write_files:
       @include common-password
 
 runcmd:
+  - 'echo "br_netfilter" >> /etc/modules-load.d/k8s.conf && modprobe br_netfilter'
+  - 'echo "net.bridge.bridge-nf-call-iptables = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'echo "net.bridge.bridge-nf-call-ip6tables = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'sysctl --load=/etc/sysctl.d/k8s.conf'
   - "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -"
   - 'echo "deb https://apt.kubernetes.io/ kubernetes-xenial main"  > /etc/apt/sources.list.d/kubernetes.list'
-  - "mkdir /run/download"
   - "apt update"
-  - "apt install -y kubelet kubeadm kubectl docker.io awscli"
-  - "wget https://github.com/kubernetes-incubator/cri-tools/releases/download/v1.11.1/crictl-v1.11.1-linux-amd64.tar.gz -O /run/download/crictl.tgz"
-  - "tar -C /usr/bin -xzf /run/download/crictl.tgz"
-  - "rm -rf /run/download"
-  - "chmod 754 /usr/bin/crictl"
-  - "chown root:root /usr/bin/crictl"
-  - "systemctl enable docker"
-  - "systemctl daemon-reload"
-  - "systemctl restart docker"
-  - "systemctl restart kubelet"
-  - "while true; do aws s3 ls s3://${s3_bucket_name}/join.txt > /dev/null; if [ $? -ne 0 ]; then sleep 10; else break; fi; done && aws s3 cp s3://${s3_bucket_name}/join.txt /tmp  && sh /tmp/join.txt"
+  - "apt install -y containerd kubelet=${version}-00 kubeadm=${version}-00 kubectl=${version}-00 awscli"
+  - 'systemctl enable --now containerd'
+  - 'systemctl enable --now kubelet'
+  - "until aws s3 ls s3://${s3_bucket_name}/join.txt > /dev/null; do sleep 5; done && aws s3 cp s3://${s3_bucket_name}/join.txt /tmp"
+  - "until sh /tmp/join.txt; do sleep 5; done"
+  - 'curl -L https://github.com/containerd/nerdctl/releases/download/v0.17.0/nerdctl-0.17.0-linux-amd64.tar.gz -o /tmp/nerdctl.tar.gz'
+  - 'tar -xf /tmp/nerdctl.tar.gz nerdctl -O > /tmp/nerdctl'
+  - 'install /tmp/nerdctl /usr/local/bin/'
 
 output:
   all: "| tee -a /var/log/cloud-init-output.log"

terraform/modules/AWS/Kubernetes/node-goss.yaml

@@ -17,16 +17,12 @@ port:
     ip:
     - '::'
 service:
-  docker:
+  containerd:
     enabled: true
     running: true
   kubelet:
     enabled: true
     running: true
-group:
-  docker:
-    exists: true
-    gid: 115
 process:
   kubelet:
     running: true

terraform/modules/AWS/Kubernetes/variables.tf

@@ -54,3 +54,8 @@ variable "default_tags" {
   type        = map(string)
 }
 
+variable "kubernetes_version" {
+  description = "version of kubernetes to deploy"
+  type        = string
+  default     = "1.22.7"
+}