From 6a15e99dd3211a5010b89e047cef48c5aac50316 Mon Sep 17 00:00:00 2001
From: Vasilis Remmas <vremmas@nvidia.com>
Date: Thu, 11 Apr 2024 19:21:36 +0200
Subject: [PATCH] Add test to ensure pods on control plane can access Kube API
 server

This test is added to showcase primary network not working as expected
on Kind control plane nodes with thick plugin installed.

Signed-off-by: Vasilis Remmas <vremmas@nvidia.com>
---
 e2e/templates/simple-macvlan1.yml.j2 | 25 +++++++++++++++++++
 e2e/templates/simple-pod.yml.j2      | 15 ------------
 e2e/templates/simple-pods.yml.j2     | 35 +++++++++++++++++++++++++++
 e2e/test-simple-macvlan1.sh          | 16 +++++++++++++
 e2e/test-simple-pod.sh               | 36 ++++++++++++++++++++++++++--
 5 files changed, 110 insertions(+), 17 deletions(-)
 delete mode 100644 e2e/templates/simple-pod.yml.j2
 create mode 100644 e2e/templates/simple-pods.yml.j2

diff --git a/e2e/templates/simple-macvlan1.yml.j2 b/e2e/templates/simple-macvlan1.yml.j2
index 7d31dcdbc..e276a14a2 100644
--- a/e2e/templates/simple-macvlan1.yml.j2
+++ b/e2e/templates/simple-macvlan1.yml.j2
@@ -61,3 +61,28 @@ spec:
       privileged: true
   nodeSelector:
     kubernetes.io/hostname: kind-worker2
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: macvlan1-control-plane
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan1-config",
+              "ips": [ "10.1.1.13/24" ] }
+    ]'
+  labels:
+    app: macvlan
+spec:
+  containers:
+  - name: macvlan-control-plane
+    image: nicolaka/netshoot:v0.12
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+  nodeSelector:
+    kubernetes.io/hostname: kind-control-plane
+  tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: "Exists"
+    effect: NoSchedule
diff --git a/e2e/templates/simple-pod.yml.j2 b/e2e/templates/simple-pod.yml.j2
deleted file mode 100644
index 82ca926c0..000000000
--- a/e2e/templates/simple-pod.yml.j2
+++ /dev/null
@@ -1,15 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: simple-centos1
-  annotations:
-  labels:
-    app: simple
-spec:
-  containers:
-  - name: simple-centos1
-    image: centos:8
-    command: ["/bin/sleep", "10000"]
-    securityContext:
-      privileged: true
diff --git a/e2e/templates/simple-pods.yml.j2 b/e2e/templates/simple-pods.yml.j2
new file mode 100644
index 000000000..58e582ed8
--- /dev/null
+++ b/e2e/templates/simple-pods.yml.j2
@@ -0,0 +1,35 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: simple-worker
+  annotations:
+  labels:
+    app: simple
+spec:
+  containers:
+  - name: simple-worker
+    image: nicolaka/netshoot:v0.12
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: simple-control-plane
+  labels:
+    app: simple
+spec:
+  containers:
+  - name: simple-control-plane
+    image: nicolaka/netshoot:v0.12
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+  nodeSelector:
+    kubernetes.io/hostname: kind-control-plane
+  tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: "Exists"
+    effect: NoSchedule
diff --git a/e2e/test-simple-macvlan1.sh b/e2e/test-simple-macvlan1.sh
index 52ad37d21..bdd1aa9f2 100755
--- a/e2e/test-simple-macvlan1.sh
+++ b/e2e/test-simple-macvlan1.sh
@@ -38,5 +38,21 @@ if [ $ipaddr != "10.1.1.12" ]; then
 	echo "macvlan1-worker2 IP address is different: ${ipaddr}"
 fi
 
+echo "check eventual connectivity of macvlan1-control-plane Pod to the Kubernetes API server"
+for i in `seq 1 10`;
+do
+    if [ $(kubectl exec macvlan1-control-plane -- nc -zvw1 kubernetes 443 >/dev/null && echo $? || echo $?) -eq 0 ]; then
+        echo "macvlan1-control-plane reached the Kubernetes API server"
+        break
+    fi
+
+    if [ $i -eq 10 ]; then
+        echo "macvlan1-control-plane couldn't connect to the Kubernetes API server"
+        exit 1
+    fi
+
+    sleep 1
+done
+
 echo "cleanup resources"
 kubectl delete -f yamls/simple-macvlan1.yml
diff --git a/e2e/test-simple-pod.sh b/e2e/test-simple-pod.sh
index 5e9c66873..aab19a33c 100755
--- a/e2e/test-simple-pod.sh
+++ b/e2e/test-simple-pod.sh
@@ -3,8 +3,40 @@ set -o errexit
 
 export PATH=${PATH}:./bin
 
-kubectl create -f yamls/simple-pod.yml
+kubectl create -f yamls/simple-pods.yml
 kubectl wait --for=condition=ready -l app=simple --timeout=300s pod
 
+echo "check eventual connectivity of simple-worker Pod to the Kubernetes API server"
+for i in `seq 1 10`;
+do
+    if [ $(kubectl exec simple-worker -- nc -zvw1 kubernetes 443 >/dev/null && echo $? || echo $?) -eq 0 ]; then
+        echo "simple-worker reached the Kubernetes API server"
+        break
+    fi
+
+    if [ $i -eq 10 ]; then
+        echo "simple-worker couldn't connect to the Kubernetes API server"
+        exit 1
+    fi
+
+    sleep 1
+done
+
+echo "check eventual connectivity of simple-control-plane Pod to the Kubernetes API server"
+for i in `seq 1 10`;
+do
+    if [ $(kubectl exec simple-control-plane -- nc -zvw1 kubernetes 443 >/dev/null && echo $? || echo $?) -eq 0 ]; then
+        echo "simple-control-plane reached the Kubernetes API server"
+        break
+    fi
+
+    if [ $i -eq 10 ]; then
+        echo "simple-control-plane couldn't connect to the Kubernetes API server"
+        exit 1
+    fi
+
+    sleep 1
+done
+
 echo "cleanup resources"
-kubectl delete -f yamls/simple-pod.yml
+kubectl delete -f yamls/simple-pods.yml