Merge pull request #327 from kubernetes-simulator/bugfixes

Several bugfixes to make simulator work with Kubernetes 1.22

Dockerfile

@@ -33,7 +33,7 @@ ENV PATH $PATH:/usr/local/go/bin
 ENV GOPATH /go
 ENV PATH $PATH:/go/bin
 
-ENV GO111MODULE on 
+ENV GO111MODULE on
 RUN mkdir -p /go/ && \
     go get github.com/hashicorp/terraform/tools/[email protected]
 
@@ -44,7 +44,7 @@ RUN terraform-bundle package terraform-bundle.hcl && \
 
 # Default configuration for dep
 ARG JQ_VERSION=1.6
-ARG YQ_VERSION=2.7.2
+ARG YQ_VERSION=3.4.1
 ARG GOSS_VERSION=v0.3.7
 ARG HADOLINT_VERSION=v1.16.3
 ARG lint_user=lint
@@ -55,7 +55,7 @@ RUN curl -sL https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/j
     && chmod +x /usr/local/bin/jq
 
 ## Install YQ
-RUN curl -sL https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 \
+RUN curl -sL https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 \
       -o /usr/local/bin/yq                                                           \
     && chmod +x /usr/local/bin/yq
 

launch-environment

@@ -4,7 +4,4 @@ AWS_SECRET_ACCESS_KEY
 
 # AWS Configuration
 AWS_REGION
-AWS_DEFAULT_REGION
 AWS_PROFILE
-AWS_DEFAULT_PROFILE
-

simulation-scripts/perturb.sh

@@ -188,10 +188,11 @@ get_pods() {
     error "Timed out waiting for pods to be ready"
   fi
 
-  local QUERY_DOCKER="docker inspect \$(docker ps -aq)"
+  local QUERY_DOCKER="nerdctl -n k8s.io inspect \$(nerdctl -n k8s.io ps -q)"
   local QUERY_KUBECTL="kubectl get pods --all-namespaces -o json"
   local TMP_FILE="${TMP_DIR}/docker-"
 
+  info "Querying pod information"
   echo "${QUERY_DOCKER}" | run_ssh "$(get_master)" >|"${TMP_FILE}"master
   echo "${QUERY_KUBECTL}" | run_ssh "$(get_master)" >|"${TMP_FILE}"all-pods
   echo "${QUERY_DOCKER}" | run_ssh "$(get_node 1)" >|"${TMP_FILE}"node-1

terraform/modules/AWS/Ami/main.tf

@@ -7,7 +7,7 @@ data "aws_ami" "find_ami" {
 
   filter {
     name   = "name"
-    values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"]
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
   }
 
   filter {
@@ -20,4 +20,3 @@ data "aws_ami" "find_ami" {
     values = ["hvm"]
   }
 }
-

terraform/modules/AWS/InternalHost/internal-config.yaml

@@ -1,5 +1,9 @@
 #cloud-config
 
+# wait for an internet connection
+bootcmd:
+  - until ping -W 1 -c1 8.8.8.8 >/dev/null; do sleep 1; done
+
 package_update: true
 package_upgrade: true
 disable_root: false

terraform/modules/AWS/InternalHost/main.tf

@@ -14,4 +14,3 @@ resource "aws_instance" "simulator_internal_host" {
     },
   )
 }
-

terraform/modules/AWS/InternalHost/variables.tf

@@ -36,4 +36,3 @@ variable "iam_instance_profile_id" {
 variable "s3_bucket_name" {
   description = "Name  of s3 bucket"
 }
-

terraform/modules/AWS/Kubernetes/main.tf

@@ -28,7 +28,6 @@ resource "aws_instance" "simulator_node_instances" {
   associate_public_ip_address = false
   subnet_id                   = var.private_subnet_id
   user_data                   = element(data.template_file.node_cloud_config.*.rendered, count.index)
-  depends_on                  = [aws_instance.simulator_master_instances]
   iam_instance_profile        = var.iam_instance_profile_id
   tags = merge(
     var.default_tags,
@@ -37,4 +36,3 @@ resource "aws_instance" "simulator_node_instances" {
     },
   )
 }
-

terraform/modules/AWS/Kubernetes/master-cloud-config.tf

@@ -7,6 +7,6 @@ data "template_file" "master_cloud_config" {
     master_bashrc  = filebase64("${path.module}/bashrc")
     master_inputrc = filebase64("${path.module}/inputrc")
     master_aliases = filebase64("${path.module}/bash_aliases")
+    version        = var.kubernetes_version
   }
 }
-

terraform/modules/AWS/Kubernetes/master-cloud-config.yaml

@@ -1,5 +1,9 @@
 #cloud-config
 
+# wait for an internet connection
+bootcmd:
+  - until ping -W 1 -c1 8.8.8.8 >/dev/null; do sleep 1; done
+
 package_update: true
 package_upgrade: true
 disable_root: false
@@ -46,29 +50,27 @@ write_files:
 
 
 runcmd:
+  - 'echo "br_netfilter" >> /etc/modules-load.d/k8s.conf && modprobe br_netfilter'
+  - 'echo "net.bridge.bridge-nf-call-iptables = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'echo "net.bridge.bridge-nf-call-ip6tables = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'sysctl --load=/etc/sysctl.d/k8s.conf'
   - 'curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -'
   - 'echo "deb https://apt.kubernetes.io/ kubernetes-xenial main"  > /etc/apt/sources.list.d/kubernetes.list'
-  - 'mkdir /run/download'
   - 'apt update'
-  - 'apt install -y kubelet kubeadm kubectl docker.io awscli'
-  - 'wget https://github.com/kubernetes-incubator/cri-tools/releases/download/v1.11.1/crictl-v1.11.1-linux-amd64.tar.gz -O /run/download/crictl.tgz'
-  - 'tar -C /usr/bin -xzf /run/download/crictl.tgz'
-  - 'chmod 754 /usr/bin/crictl'
-  - 'chown root:root /usr/bin/crictl'
-  - 'systemctl enable docker'
-  - 'systemctl daemon-reload'
-  - 'systemctl restart docker'
-  - 'systemctl restart kubelet'
-  - 'wget https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml -O /run/download/rbac-kdd.yaml'
-  - 'wget https://docs.projectcalico.org/v3.9/manifests/calico.yaml -O /run/download/calico.yaml'
-  - 'kubeadm init --pod-network-cidr=192.168.0.0/16'
+  - 'apt install -y containerd kubelet=${version}-00 kubeadm=${version}-00 kubectl=${version}-00 awscli'
+  - 'systemctl enable --now containerd'
+  - 'systemctl enable --now kubelet'
+  - 'kubeadm init --pod-network-cidr=192.168.0.0/16 --kubernetes-version=v${version}'
   - 'mkdir /root/.kube'
   - 'cp /etc/kubernetes/admin.conf /root/.kube/config'
-  - 'kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /run/download/rbac-kdd.yaml'
-  - 'kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /run/download/calico.yaml'
-  - "egrep -A 1 'kubeadm join' /var/log/cloud-init-output.log  |tr -d '\\' | tr -d '\n' > /tmp/join.txt"
+  - 'kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://projectcalico.docs.tigera.io/manifests/calico.yaml' # needs to be versioned
+  - "grep -A 1 'kubeadm join' /var/log/cloud-init-output.log | tr -d '\\\n\r\t' > /tmp/join.txt"
   - 'aws s3 cp /tmp/join.txt s3://${s3_bucket_name}'
   - 'aws s3 cp /root/.kube/config s3://${s3_bucket_name}'
+  - 'curl -L https://github.com/containerd/nerdctl/releases/download/v0.17.0/nerdctl-0.17.0-linux-amd64.tar.gz -o /tmp/nerdctl.tar.gz'
+  - 'tar -xf /tmp/nerdctl.tar.gz nerdctl -O > /tmp/nerdctl'
+  - 'install /tmp/nerdctl /usr/local/bin/'
 
 output:
   all: '| tee -a /var/log/cloud-init-output.log'

terraform/modules/AWS/Kubernetes/master-goss.yaml

@@ -17,16 +17,12 @@ port:
     ip:
     - '::'
 service:
-  docker:
+  containerd:
     enabled: true
     running: true
   kubelet:
     enabled: true
     running: true
-group:
-  docker:
-    exists: true
-    gid: 115
 process:
   kubelet:
     running: true

terraform/modules/AWS/Kubernetes/node-cloud-config.tf

@@ -7,6 +7,6 @@ data "template_file" "node_cloud_config" {
     node_bashrc    = filebase64("${path.module}/bashrc")
     node_inputrc   = filebase64("${path.module}/inputrc")
     node_aliases   = filebase64("${path.module}/bash_aliases")
+    version        = var.kubernetes_version
   }
 }
-

terraform/modules/AWS/Kubernetes/node-cloud-config.yaml

@@ -1,5 +1,9 @@
 #cloud-config
 
+# wait for an internet connection
+bootcmd:
+  - until ping -W 1 -c1 8.8.8.8 >/dev/null; do sleep 1; done
+
 package_update: true
 package_upgrade: true
 disable_root: false
@@ -45,21 +49,22 @@ write_files:
       @include common-password
 
 runcmd:
+  - 'echo "br_netfilter" >> /etc/modules-load.d/k8s.conf && modprobe br_netfilter'
+  - 'echo "net.bridge.bridge-nf-call-iptables = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'echo "net.bridge.bridge-nf-call-ip6tables = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.d/k8s.conf'
+  - 'sysctl --load=/etc/sysctl.d/k8s.conf'
   - "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -"
   - 'echo "deb https://apt.kubernetes.io/ kubernetes-xenial main"  > /etc/apt/sources.list.d/kubernetes.list'
-  - "mkdir /run/download"
   - "apt update"
-  - "apt install -y kubelet kubeadm kubectl docker.io awscli"
-  - "wget https://github.com/kubernetes-incubator/cri-tools/releases/download/v1.11.1/crictl-v1.11.1-linux-amd64.tar.gz -O /run/download/crictl.tgz"
-  - "tar -C /usr/bin -xzf /run/download/crictl.tgz"
-  - "rm -rf /run/download"
-  - "chmod 754 /usr/bin/crictl"
-  - "chown root:root /usr/bin/crictl"
-  - "systemctl enable docker"
-  - "systemctl daemon-reload"
-  - "systemctl restart docker"
-  - "systemctl restart kubelet"
-  - "while true; do aws s3 ls s3://${s3_bucket_name}/join.txt > /dev/null; if [ $? -ne 0 ]; then sleep 10; else break; fi; done && aws s3 cp s3://${s3_bucket_name}/join.txt /tmp  && sh /tmp/join.txt"
+  - "apt install -y containerd kubelet=${version}-00 kubeadm=${version}-00 kubectl=${version}-00 awscli"
+  - 'systemctl enable --now containerd'
+  - 'systemctl enable --now kubelet'
+  - "until aws s3 ls s3://${s3_bucket_name}/join.txt > /dev/null; do sleep 5; done && aws s3 cp s3://${s3_bucket_name}/join.txt /tmp"
+  - "until sh /tmp/join.txt; do sleep 5; done"
+  - 'curl -L https://github.com/containerd/nerdctl/releases/download/v0.17.0/nerdctl-0.17.0-linux-amd64.tar.gz -o /tmp/nerdctl.tar.gz'
+  - 'tar -xf /tmp/nerdctl.tar.gz nerdctl -O > /tmp/nerdctl'
+  - 'install /tmp/nerdctl /usr/local/bin/'
 
 output:
   all: "| tee -a /var/log/cloud-init-output.log"

terraform/modules/AWS/Kubernetes/node-goss.yaml

@@ -17,16 +17,12 @@ port:
     ip:
     - '::'
 service:
-  docker:
+  containerd:
     enabled: true
     running: true
   kubelet:
     enabled: true
     running: true
-group:
-  docker:
-    exists: true
-    gid: 115
 process:
   kubelet:
     running: true

terraform/modules/AWS/Kubernetes/test.tf

@@ -1,6 +1,10 @@
 resource "null_resource" "master_test" {
   count = var.number_of_master_instances
 
+  triggers = {
+    cluster_instance_ids = "${join(",", aws_instance.simulator_master_instances.*.id)}"
+  }
+
   connection {
     bastion_host        = var.bastion_public_ip
     bastion_private_key = file(pathexpand("~/.kubesim/cp_simulator_rsa"))
@@ -41,6 +45,10 @@ resource "null_resource" "master_test" {
 resource "null_resource" "node_test" {
   count = var.number_of_cluster_instances
 
+  triggers = {
+    cluster_instance_ids = "${join(",", aws_instance.simulator_node_instances.*.id)}"
+  }
+
   connection {
     bastion_host        = var.bastion_public_ip
     bastion_private_key = file(pathexpand("~/.kubesim/cp_simulator_rsa"))
@@ -55,6 +63,7 @@ resource "null_resource" "node_test" {
     agent       = "false"
     private_key = file(pathexpand("~/.kubesim/cp_simulator_rsa"))
 
+
     // Increase the timeout so the server has time to reboot
     timeout = "10m"
   }
@@ -77,4 +86,3 @@ resource "null_resource" "node_test" {
     ]
   }
 }
-

terraform/modules/AWS/Kubernetes/variables.tf

@@ -54,3 +54,8 @@ variable "default_tags" {
   type        = map(string)
 }
 
+variable "kubernetes_version" {
+  description = "version of kubernetes to deploy"
+  type        = string
+  default     = "1.22.7"
+}

terraform/modules/scripts/run-goss.sh

@@ -31,8 +31,14 @@ install_goss() {
 readonly -f install_goss
 
 wait_for_cloud_init() {
-  while ! grep -q "finish: modules-final: SUCCESS: running modules for final" /var/log/cloud-init.log; do
-    echo "Waiting 5s for cloud-init to finish"
+  while :; do
+    if grep -q "finish: modules-final: SUCCESS: running modules for final" /var/log/cloud-init.log; then
+        echo "Waiting 5s for cloud-init to finish"
+        break
+    elif grep -q "finish: modules-final: FAIL" /var/log/cloud-init.log; then
+        echo "cloud init failed"
+        exit 1
+    fi
     sleep 5
   done
 }