Merge pull request #491 from nik-netlox/main

K8s-calico-ipvs test cases updated to run with github local runners

.github/workflows/k8s-calico-ipvs3-ha.yml

@@ -1,8 +1,8 @@
 name: K8s-Calico-Cluster-IPVS3-HA-Sanity-CI
 on:
- #schedule:
+ schedule:
  # Runs "At 17:00 UTC every day-of-week"
- #- cron: '0 17 * * *'
+ - cron: '0 17 * * *'
  workflow_dispatch:
     inputs:
       testName:

cicd/k8s-calico-ipvs3-ha/README

@@ -14,6 +14,6 @@ If you wish to create this scenario in your lab then install Vagrant and follow
 
 1. Run ./config.sh to setup the K8s cluster, client and LoxiLB nodes
 
-2. Run ./validation.sh to run the HA test. Test Results will be displayed at the end.
+2. Run ./validation.sh to run the TCP HA test or ./validation_with_sctp.sh to run TCP & SCTP HA Test. Test Results will be displayed at the end.
 
 3. Run ./rmconfig.sh to cleanup the setup.

cicd/k8s-calico-ipvs3-ha/host_validation.sh

@@ -2,6 +2,7 @@
 extIP=$(cat /vagrant/extIP)
 
 code=0
+
 echo Service IP: $extIP
 
 numECMP=$(birdc show route | grep $extIP -A 3 | grep via | wc -l)
@@ -18,26 +19,10 @@ echo "Testing Service"
 echo "*********************************************"
 
 # iperf client accessing fullnat service
-stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 100 -i 1 -b 100M &> iperff.out &
+stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 60 -i 1 -b 100M &> iperff.out &
 
 # iperf client accessing default service
-stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out &
-
-mkfifo sd1.pipe
-mkfifo sd2.pipe
-
-sleep infinity > sd1.pipe &
-sleep infinity > sd2.pipe &
-
-sleep 1
-
-stdbuf -oL nohup sctp_darn -H 192.168.90.9 -h 20.20.20.1 -p 56004 -s -I < sd1.pipe &> sdf.out &
-stdbuf -oL nohup sctp_darn -H 30.30.30.1 -h 20.20.20.1 -p 56005 -s -I < sd2.pipe &> sdd.out &
+stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 60 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out &
 
-sleep 2
-for((i=0;i<30;i++))
-do
-echo "snd=100" >> sd1.pipe
-echo "snd=100" >> sd2.pipe
-sleep 1
-done
+echo "iperf client started"
+echo "phase-1 done"

cicd/k8s-calico-ipvs3-ha/host_validation2.sh

@@ -1,30 +1,10 @@
 #!/bin/bash
-for((i=0;i<50;i++))
-do
-echo "snd=100" >> sd1.pipe
-echo "snd=100" >> sd2.pipe
-sleep 1
-done
-
-
-echo "stats" >> sd1.pipe
-echo "stats" >> sd2.pipe
-
-echo "shutdown" >> sd1.pipe
-echo "shutdown" >> sd2.pipe
 
 pkill iperf
-pkill sctp_darn
 
 iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
 iperfd_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
 
-sdf_res1=$(grep -i "packets sent" sdf.out | xargs | cut -d ' ' -f 3)
-sdf_res2=$(grep -i "packets rec" sdf.out | xargs | cut -d ' ' -f 3)
-
-sdd_res1=$(grep -i "packets sent" sdd.out | xargs | cut -d ' ' -f 3)
-sdd_res2=$(grep -i "packets rec" sdd.out | xargs | cut -d ' ' -f 3)
-
 if [[ $iperff_res != 0 ]]; then
     echo -e "K8s-calico-ipvs3-ha TCP\t\t(fullnat)\t[OK]"
 else
@@ -39,19 +19,5 @@ else
     code=1
 fi
 
-if [[ $sdf_res1 != 0 && $sdf_res2 != 0 && $sdf_res1 == $sdf_res2 ]]; then
-    echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[OK]"
-else
-    echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[FAILED]"
-    code=1
-fi
-
-if [[ $sdd_res1 != 0 && $sdd_res2 != 0 && $sdd_res1 == $sdd_res2 ]]; then
-    echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[OK]"
-else
-    echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[FAILED]"
-    code=1
-fi
-
-rm *.out *.pipe
+rm *.out
 exit $code

cicd/k8s-calico-ipvs3-ha/host_validation2_with_sctp.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+for((i=0;i<50;i++))
+do
+echo "snd=100" 1> sd1.pipe
+echo "snd=100" 1> sd2.pipe
+sleep 1
+done
+
+
+echo "stats" 1> sd1.pipe
+echo "stats" 1> sd2.pipe
+
+echo "shutdown" 1> sd1.pipe
+echo "shutdown" 1> sd2.pipe
+
+pkill iperf
+pkill sctp_darn
+
+iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
+iperfd_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
+
+sdf_res=$(grep -i "Client: Sending packets.(100000/100000)" sdf.out)
+sdd_res=$(grep -i "Client: Sending packets.(100000/100000)" sdd.out)
+
+sdf_res1=$(grep -i "packets sent" sdf.out | xargs | cut -d ' ' -f 3)
+sdf_res2=$(grep -i "packets rec" sdf.out | xargs | cut -d ' ' -f 3)
+
+sdd_res1=$(grep -i "packets sent" sdd.out | xargs | cut -d ' ' -f 3)
+sdd_res2=$(grep -i "packets rec" sdd.out | xargs | cut -d ' ' -f 3)
+
+if [[ $iperff_res != 0 ]]; then
+    echo -e "K8s-calico-ipvs3-ha TCP\t\t(fullnat)\t[OK]"
+else
+    echo -e "K8s-calico-ipvs3-ha TCP\t\t(fullnat)\t[FAILED]"
+    code=1
+fi
+
+if [[ $iperfd_res != 0 ]]; then
+    echo -e "K8s-calico-ipvs3-ha TCP\t\t(default\t[OK]"
+else
+    echo -e "K8s-calico-ipvs3-ha TCP\t\t(default)\t[FAILED]"
+    code=1
+fi
+
+if [[ $sdf_res1 != 0 && $sdf_res2 != 0 && $sdf_res1 == $sdf_res2 ]]; then
+    echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[OK]"
+else
+    echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[FAILED]"
+    code=1
+fi
+
+if [[ $sdd_res1 != 0 && $sdd_res2 != 0 && $sdd_res1 == $sdd_res2 ]]; then
+    echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[OK]"
+else
+    echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[FAILED]"
+    code=1
+fi
+rm *.pipe
+rm *.out
+exit $code

cicd/k8s-calico-ipvs3-ha/host_validation_with_sctp.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+extIP=$(cat /vagrant/extIP)
+
+code=0
+
+echo Service IP: $extIP
+
+numECMP=$(birdc show route | grep $extIP -A 3 | grep via | wc -l)
+
+birdc show route | grep $extIP -A 3
+
+if [ $numECMP == "2" ]; then
+    echo "Host route [OK]"
+else
+    echo "Host route [NOK]"
+fi
+echo -e "\n*********************************************"
+echo "Testing Service"
+echo "*********************************************"
+
+# iperf client accessing fullnat service
+stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 100 -i 1 -b 100M &> iperff.out &
+
+# iperf client accessing default service
+stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out &
+
+echo "iperf client started"
+
+sleep 1
+
+mkfifo sd1.pipe
+mkfifo sd2.pipe
+
+sleep infinity > sd1.pipe &
+sleep infinity > sd2.pipe &
+
+stdbuf -oL nohup sctp_darn -H 192.168.90.9 -h 20.20.20.1 -p 56004 -s -I < sd1.pipe &> sdf.out &
+stdbuf -oL nohup sctp_darn -H 30.30.30.1 -h 20.20.20.1 -p 56005 -s -I < sd2.pipe &> sdd.out &
+
+echo "sctp_test client started"
+
+sleep 2
+for((i=0;i<30;i++))
+do
+echo "snd=100" 1> sd1.pipe
+echo "snd=100" 1> sd2.pipe
+sleep 1
+done
+echo "phase-1 done"
+exit 0

cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb1.sh

@@ -7,4 +7,5 @@ add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu
 apt-get update
 apt-get install -y docker-ce
 docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest -b --cluster=192.168.80.253 --self=0
+docker cp loxilb:/usr/local/sbin/loxicmd ./
 #docker exec -dt loxilb /root/loxilb-io/loxilb/loxilb -b --cluster=192.168.80.253 --self=0

cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb2.sh

@@ -7,5 +7,7 @@ add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu
 apt-get update
 apt-get install -y docker-ce
 docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest -b --cluster=192.168.80.252 --self=1
+docker cp loxilb:/usr/local/sbin/loxicmd ./
 #docker exec -dt loxilb /root/loxilb-io/loxilb/loxilb -b --cluster=192.168.80.252 --self=1
 
+

cicd/k8s-calico-ipvs3-ha/node_scripts/master.sh

@@ -40,6 +40,9 @@ kubeadm token create --print-join-command > $config_path/join.sh
 curl https://raw.githubusercontent.com/projectcalico/calico/v${CALICO_VERSION}/manifests/calico.yaml -O
 
 kubectl apply -f calico.yaml
+kubectl patch configmap/calico-config -n kube-system --type merge \
+  -p '{"data":{"veth_mtu": "8900"}}'
+kubectl rollout restart daemonset calico-node -n kube-system
 
 sudo -i -u vagrant bash << EOF
 whoami

cicd/k8s-calico-ipvs3-ha/validation.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -eo pipefail
 source ../common.sh
 echo k8s-calico-ipvs3-ha
 
@@ -47,18 +48,18 @@ echo "**************************************************************************
 echo -e "\nLB List"
 echo -e "\n---- LLB1 ----"
 echo "******************************************************************************"
-vagrant ssh llb1 -c 'sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null
+vagrant ssh llb1 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null
 echo "******************************************************************************"
 echo -e "\n---- LLB2 ----"
-vagrant ssh llb2 -c 'sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null
+vagrant ssh llb2 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null
 echo "******************************************************************************"
 echo -e "\nEP List"
 echo -e "\n---- LLB1 ----"
 echo "******************************************************************************"
-vagrant ssh llb1 -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null
+vagrant ssh llb1 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null
 echo "******************************************************************************"
 echo -e "\n---- LLB2 ----"
-vagrant ssh llb2 -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null
+vagrant ssh llb2 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null
 echo "******************************************************************************"
 echo -e "\nTEST RESULTS"
 echo "******************************************************************************"
@@ -78,14 +79,17 @@ echo -e " BACKUP\t: $backup\n"
 
 vagrant ssh host -c 'sudo /vagrant/host_validation.sh' 2> /dev/null
 
+sleep 15
+echo -e "phase-2 begins..\n"
+
 count=1
 sync=0
 while [[ $count -le 5 ]] ; do
 echo -e "\nStatus at MASTER:$master\n"
-vagrant ssh $master -c "sudo docker exec -it loxilb loxicmd get ct | grep est" 2> /dev/null
+vagrant ssh $master -c "sudo ./loxicmd get ct | grep est" 2> /dev/null
 
 echo -e "\nStatus at BACKUP:$backup\n"
-vagrant ssh $backup -c "sudo docker exec -it loxilb loxicmd get ct | grep est" 2> /dev/null
+vagrant ssh $backup -c "sudo ./loxicmd get ct | grep est" 2> /dev/null
 
 nres1=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
 nres2=$(curl -sX 'GET' 'http://192.168.80.253:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
@@ -102,17 +106,14 @@ done
 
 if [[ $sync == 0 ]]; then
     echo -e "\nConnection Sync failed\n"
-    vagrant ssh host -c 'sudo pkill iperf; sudo pkill sctp_darn; sudo rm -rf *.pipe *.out'
+    vagrant ssh host -c 'sudo pkill iperf; sudo pkill sctp_test; sudo rm -rf *.out'
     exit 1
 fi
 
 echo "Restarting MASTER:$master.."
 vagrant ssh $master -c 'sudo docker restart loxilb' 2> /dev/null
 
-sleep 10
-
-vagrant ssh host -c 'sudo /vagrant/host_validation2.sh' 2> /dev/null
+sleep 50
 
 sudo rm extIP
-
-exit $code
+vagrant ssh host -c 'sudo /vagrant/host_validation2.sh' 2> /dev/null