From f11077d23762d2c0b8c39c1b234381c0d802a577 Mon Sep 17 00:00:00 2001
From: nik-netlox <66406124+nik-netlox@users.noreply.github.com>
Date: Mon, 8 Jan 2024 17:59:26 +0900
Subject: [PATCH 1/8] Update k8s-calico-ipvs3-ha.yml

---
 .github/workflows/k8s-calico-ipvs3-ha.yml | 26 +++++++++++------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/k8s-calico-ipvs3-ha.yml b/.github/workflows/k8s-calico-ipvs3-ha.yml
index 45076a612..4fcbd13ad 100644
--- a/.github/workflows/k8s-calico-ipvs3-ha.yml
+++ b/.github/workflows/k8s-calico-ipvs3-ha.yml
@@ -13,24 +13,24 @@ jobs:
   test-runner:
     name: k8s-calico-cluster-ipvs3-ha-sanity
     runs-on: self-hosted
-    if: github.repository == 'loxilb-io/loxilb' 
-          &&  github.event.inputs.tagName == ''
+    #if: github.repository == 'loxilb-io/loxilb' 
+    #      &&  github.event.inputs.tagName == ''
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-          submodules: recursive
+    #- name: Checkout
+    #  uses: actions/checkout@v2
+    #  with:
+    #      submodules: recursive
 
     - name: Run the test
       run: |
          cd cicd/k8s-calico-ipvs3-ha
-         ./config.sh
+         #./config.sh
          ./validation.sh
          cd -
          
-    - name: Clean test-bed
-      if: success() || failure()
-      run: |
-        cd cicd/k8s-calico-ipvs3-ha || true
-        ./rmconfig.sh
-        cd -
+    #- name: Clean test-bed
+    #  if: success() || failure()
+    #  run: |
+    #    cd cicd/k8s-calico-ipvs3-ha || true
+    #    ./rmconfig.sh
+    #    cd -

From 48dd7f8095065d11a5215ab1d528998d38480188 Mon Sep 17 00:00:00 2001
From: Nikhil Malik <nikhil.malik@netlox.io>
Date: Wed, 10 Jan 2024 12:44:36 +0900
Subject: [PATCH 2/8] k8s-calico-ipvs3-ha cicd test updated

---
 cicd/k8s-calico-ipvs3-ha/host_validation.sh   | 19 ++++--------
 cicd/k8s-calico-ipvs3-ha/host_validation2.sh  | 29 ++++---------------
 .../node_scripts/loxilb1.sh                   |  1 +
 .../node_scripts/loxilb2.sh                   |  2 ++
 cicd/k8s-calico-ipvs3-ha/validation.sh        | 24 +++++++--------
 .../k8s-calico-ipvs3-ha/yaml/sctp_default.yml |  4 ++-
 .../k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml |  4 ++-
 cicd/k8s-calico-ipvs3-ha/yaml/tcp_default.yml |  3 +-
 cicd/k8s-calico-ipvs3-ha/yaml/tcp_fullnat.yml |  1 +
 9 files changed, 35 insertions(+), 52 deletions(-)

diff --git a/cicd/k8s-calico-ipvs3-ha/host_validation.sh b/cicd/k8s-calico-ipvs3-ha/host_validation.sh
index 814bb14b4..e11dccb9b 100755
--- a/cicd/k8s-calico-ipvs3-ha/host_validation.sh
+++ b/cicd/k8s-calico-ipvs3-ha/host_validation.sh
@@ -2,6 +2,7 @@
 extIP=$(cat /vagrant/extIP)
 
 code=0
+
 echo Service IP: $extIP
 
 numECMP=$(birdc show route | grep $extIP -A 3 | grep via | wc -l)
@@ -23,21 +24,11 @@ stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 100 -i 1 -b 100M &> iperff.out
 # iperf client accessing default service
 stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out &
 
-mkfifo sd1.pipe
-mkfifo sd2.pipe
-
-sleep infinity > sd1.pipe &
-sleep infinity > sd2.pipe &
+echo "iperf client started"
 
 sleep 1
 
-stdbuf -oL nohup sctp_darn -H 192.168.90.9 -h 20.20.20.1 -p 56004 -s -I < sd1.pipe &> sdf.out &
-stdbuf -oL nohup sctp_darn -H 30.30.30.1 -h 20.20.20.1 -p 56005 -s -I < sd2.pipe &> sdd.out &
+stdbuf -oL nohup sctp_test -H 192.168.90.9 -h 20.20.20.1 -p 56004 -s -c 0 -x 100000  &> sdf.out &
+stdbuf -oL nohup sctp_test -H 30.30.30.1 -h 20.20.20.1 -p 56005 -s -c 0 -x 100000 &> sdd.out &
 
-sleep 2
-for((i=0;i<30;i++))
-do
-echo "snd=100" >> sd1.pipe
-echo "snd=100" >> sd2.pipe
-sleep 1
-done
+echo "sctp_test client started"
diff --git a/cicd/k8s-calico-ipvs3-ha/host_validation2.sh b/cicd/k8s-calico-ipvs3-ha/host_validation2.sh
index 11e870001..d61ac1367 100755
--- a/cicd/k8s-calico-ipvs3-ha/host_validation2.sh
+++ b/cicd/k8s-calico-ipvs3-ha/host_validation2.sh
@@ -1,29 +1,12 @@
 #!/bin/bash
-for((i=0;i<50;i++))
-do
-echo "snd=100" >> sd1.pipe
-echo "snd=100" >> sd2.pipe
-sleep 1
-done
-
-
-echo "stats" >> sd1.pipe
-echo "stats" >> sd2.pipe
-
-echo "shutdown" >> sd1.pipe
-echo "shutdown" >> sd2.pipe
-
 pkill iperf
-pkill sctp_darn
+pkill sctp_test
 
 iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
 iperfd_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
 
-sdf_res1=$(grep -i "packets sent" sdf.out | xargs | cut -d ' ' -f 3)
-sdf_res2=$(grep -i "packets rec" sdf.out | xargs | cut -d ' ' -f 3)
-
-sdd_res1=$(grep -i "packets sent" sdd.out | xargs | cut -d ' ' -f 3)
-sdd_res2=$(grep -i "packets rec" sdd.out | xargs | cut -d ' ' -f 3)
+sdf_res=$(grep -i "Client: Sending packets.(100000/100000)" sdf.out)
+sdd_res=$(grep -i "Client: Sending packets.(100000/100000)" sdd.out)
 
 if [[ $iperff_res != 0 ]]; then
     echo -e "K8s-calico-ipvs3-ha TCP\t\t(fullnat)\t[OK]"
@@ -39,19 +22,19 @@ else
     code=1
 fi
 
-if [[ $sdf_res1 != 0 && $sdf_res2 != 0 && $sdf_res1 == $sdf_res2 ]]; then
+if [[ ! -z $sdf_res ]]; then
     echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[OK]"
 else
     echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[FAILED]"
     code=1
 fi
 
-if [[ $sdd_res1 != 0 && $sdd_res2 != 0 && $sdd_res1 == $sdd_res2 ]]; then
+if [[ ! -z $sdd_res ]]; then
     echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[OK]"
 else
     echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[FAILED]"
     code=1
 fi
 
-rm *.out *.pipe
+rm *.out
 exit $code
diff --git a/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb1.sh b/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb1.sh
index cc999d846..2bae00c97 100644
--- a/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb1.sh
+++ b/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb1.sh
@@ -7,4 +7,5 @@ add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu
 apt-get update
 apt-get install -y docker-ce
 docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest -b --cluster=192.168.80.253 --self=0
+docker cp loxilb:/usr/local/sbin/loxicmd ./
 #docker exec -dt loxilb /root/loxilb-io/loxilb/loxilb -b --cluster=192.168.80.253 --self=0
diff --git a/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb2.sh b/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb2.sh
index 522ff30a9..bf2744b38 100644
--- a/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb2.sh
+++ b/cicd/k8s-calico-ipvs3-ha/node_scripts/loxilb2.sh
@@ -7,5 +7,7 @@ add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu
 apt-get update
 apt-get install -y docker-ce
 docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest -b --cluster=192.168.80.252 --self=1
+docker cp loxilb:/usr/local/sbin/loxicmd ./
 #docker exec -dt loxilb /root/loxilb-io/loxilb/loxilb -b --cluster=192.168.80.252 --self=1
 
+
diff --git a/cicd/k8s-calico-ipvs3-ha/validation.sh b/cicd/k8s-calico-ipvs3-ha/validation.sh
index 0ddc714fe..be2b9b899 100755
--- a/cicd/k8s-calico-ipvs3-ha/validation.sh
+++ b/cicd/k8s-calico-ipvs3-ha/validation.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -eo pipefail
 source ../common.sh
 echo k8s-calico-ipvs3-ha
 
@@ -47,18 +48,18 @@ echo "**************************************************************************
 echo -e "\nLB List"
 echo -e "\n---- LLB1 ----"
 echo "******************************************************************************"
-vagrant ssh llb1 -c 'sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null
+vagrant ssh llb1 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null
 echo "******************************************************************************"
 echo -e "\n---- LLB2 ----"
-vagrant ssh llb2 -c 'sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null
+vagrant ssh llb2 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null
 echo "******************************************************************************"
 echo -e "\nEP List"
 echo -e "\n---- LLB1 ----"
 echo "******************************************************************************"
-vagrant ssh llb1 -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null
+vagrant ssh llb1 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null
 echo "******************************************************************************"
 echo -e "\n---- LLB2 ----"
-vagrant ssh llb2 -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null
+vagrant ssh llb2 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null
 echo "******************************************************************************"
 echo -e "\nTEST RESULTS"
 echo "******************************************************************************"
@@ -78,14 +79,16 @@ echo -e " BACKUP\t: $backup\n"
 
 vagrant ssh host -c 'sudo /vagrant/host_validation.sh' 2> /dev/null
 
+sleep 15
+
 count=1
 sync=0
 while [[ $count -le 5 ]] ; do
 echo -e "\nStatus at MASTER:$master\n"
-vagrant ssh $master -c "sudo docker exec -it loxilb loxicmd get ct | grep est" 2> /dev/null
+vagrant ssh $master -c "sudo ./loxicmd get ct | grep est" 2> /dev/null
 
 echo -e "\nStatus at BACKUP:$backup\n"
-vagrant ssh $backup -c "sudo docker exec -it loxilb loxicmd get ct | grep est" 2> /dev/null
+vagrant ssh $backup -c "sudo ./loxicmd get ct | grep est" 2> /dev/null
 
 nres1=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
 nres2=$(curl -sX 'GET' 'http://192.168.80.253:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
@@ -102,17 +105,14 @@ done
 
 if [[ $sync == 0 ]]; then
     echo -e "\nConnection Sync failed\n"
-    vagrant ssh host -c 'sudo pkill iperf; sudo pkill sctp_darn; sudo rm -rf *.pipe *.out'
+    vagrant ssh host -c 'sudo pkill iperf; sudo pkill sctp_test; sudo rm -rf *.out'
     exit 1
 fi
 
 echo "Restarting MASTER:$master.."
 vagrant ssh $master -c 'sudo docker restart loxilb' 2> /dev/null
 
-sleep 10
-
-vagrant ssh host -c 'sudo /vagrant/host_validation2.sh' 2> /dev/null
+sleep 30
 
 sudo rm extIP
-
-exit $code
+vagrant ssh host -c 'sudo /vagrant/host_validation2.sh' 2> /dev/null
diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml
index 44de88b52..324087241 100644
--- a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml
+++ b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml
@@ -5,6 +5,7 @@ metadata:
   annotations:
     loxilb.io/lbmode: "default"
     loxilb.io/liveness: "yes"
+    loxilb.io/probetype: "ping"
 spec:
   externalTrafficPolicy: Local
   loadBalancerClass: loxilb.io/loxilb
@@ -28,6 +29,7 @@ spec:
       image: loxilbio/sctp-darn:latest
       imagePullPolicy: Always
       #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"]
-      command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"]
+      #command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"]
+      command: ["sctp_test","-H", "0.0.0.0","-P", "9999", "-l"]
       ports:
         - containerPort: 9999
diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml
index 87beb9d08..ff69e7058 100644
--- a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml
+++ b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml
@@ -5,6 +5,7 @@ metadata:
   annotations:
     loxilb.io/lbmode: "fullnat"
     loxilb.io/liveness: "yes"
+    loxilb.io/probetype: "ping"
 spec:
   externalTrafficPolicy: Local
   loadBalancerClass: loxilb.io/loxilb
@@ -28,6 +29,7 @@ spec:
       image: loxilbio/sctp-darn:latest
       imagePullPolicy: Always
       #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"]
-      command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"]
+      #command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"]
+      command: ["sctp_test","-H", "0.0.0.0","-P", "9999", "-l"]
       ports:
         - containerPort: 9999
diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/tcp_default.yml b/cicd/k8s-calico-ipvs3-ha/yaml/tcp_default.yml
index c3def4e05..2742671bb 100644
--- a/cicd/k8s-calico-ipvs3-ha/yaml/tcp_default.yml
+++ b/cicd/k8s-calico-ipvs3-ha/yaml/tcp_default.yml
@@ -3,8 +3,9 @@ kind: Service
 metadata:
   name: tcp-lb-default
   annotations:
-    loxilb.io/liveness: "yes"
     loxilb.io/lbmode: "default"
+      #loxilb.io/liveness: "yes"
+      #loxilb.io/probetype: "ping"
 spec:
   externalTrafficPolicy: Local
   loadBalancerClass: loxilb.io/loxilb
diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/tcp_fullnat.yml b/cicd/k8s-calico-ipvs3-ha/yaml/tcp_fullnat.yml
index d47eb27c3..0125b69b7 100644
--- a/cicd/k8s-calico-ipvs3-ha/yaml/tcp_fullnat.yml
+++ b/cicd/k8s-calico-ipvs3-ha/yaml/tcp_fullnat.yml
@@ -5,6 +5,7 @@ metadata:
   annotations:
     loxilb.io/liveness: "yes"
     loxilb.io/lbmode: "fullnat"
+    loxilb.io/probetype: "ping"
 spec:
   externalTrafficPolicy: Local
   loadBalancerClass: loxilb.io/loxilb

From a01f74a2c3c3e2a6aa813edb3ab93c9c391540d8 Mon Sep 17 00:00:00 2001
From: nik-netlox <66406124+nik-netlox@users.noreply.github.com>
Date: Wed, 10 Jan 2024 12:46:25 +0900
Subject: [PATCH 3/8] Update k8s-calico-ipvs3-ha.yml

---
 .github/workflows/k8s-calico-ipvs3-ha.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/k8s-calico-ipvs3-ha.yml b/.github/workflows/k8s-calico-ipvs3-ha.yml
index 4fcbd13ad..7447ff2a6 100644
--- a/.github/workflows/k8s-calico-ipvs3-ha.yml
+++ b/.github/workflows/k8s-calico-ipvs3-ha.yml
@@ -16,21 +16,21 @@ jobs:
     #if: github.repository == 'loxilb-io/loxilb' 
     #      &&  github.event.inputs.tagName == ''
     steps:
-    #- name: Checkout
-    #  uses: actions/checkout@v2
-    #  with:
-    #      submodules: recursive
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+          submodules: recursive
 
     - name: Run the test
       run: |
          cd cicd/k8s-calico-ipvs3-ha
-         #./config.sh
+         ./config.sh
          ./validation.sh
          cd -
          
-    #- name: Clean test-bed
-    #  if: success() || failure()
-    #  run: |
-    #    cd cicd/k8s-calico-ipvs3-ha || true
-    #    ./rmconfig.sh
-    #    cd -
+    - name: Clean test-bed
+      if: success() || failure()
+      run: |
+        cd cicd/k8s-calico-ipvs3-ha || true
+        ./rmconfig.sh
+        cd -

From 82f1c136cc485f80b63561a313397f07e2ec4397 Mon Sep 17 00:00:00 2001
From: nik-netlox <66406124+nik-netlox@users.noreply.github.com>
Date: Wed, 10 Jan 2024 13:16:58 +0900
Subject: [PATCH 4/8] Update k8s-calico-ipvs3-ha.yml

---
 .github/workflows/k8s-calico-ipvs3-ha.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-calico-ipvs3-ha.yml b/.github/workflows/k8s-calico-ipvs3-ha.yml
index 7447ff2a6..f2f853d91 100644
--- a/.github/workflows/k8s-calico-ipvs3-ha.yml
+++ b/.github/workflows/k8s-calico-ipvs3-ha.yml
@@ -32,5 +32,5 @@ jobs:
       if: success() || failure()
       run: |
         cd cicd/k8s-calico-ipvs3-ha || true
-        ./rmconfig.sh
+        #./rmconfig.sh
         cd -

From fd6978aa70258d2ee79506efba85042f11311ebf Mon Sep 17 00:00:00 2001
From: nik-netlox <66406124+nik-netlox@users.noreply.github.com>
Date: Wed, 10 Jan 2024 17:19:26 +0900
Subject: [PATCH 5/8] Update k8s-calico-ipvs3-ha.yml

---
 .github/workflows/k8s-calico-ipvs3-ha.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/k8s-calico-ipvs3-ha.yml b/.github/workflows/k8s-calico-ipvs3-ha.yml
index f2f853d91..4fcbd13ad 100644
--- a/.github/workflows/k8s-calico-ipvs3-ha.yml
+++ b/.github/workflows/k8s-calico-ipvs3-ha.yml
@@ -16,21 +16,21 @@ jobs:
     #if: github.repository == 'loxilb-io/loxilb' 
     #      &&  github.event.inputs.tagName == ''
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-          submodules: recursive
+    #- name: Checkout
+    #  uses: actions/checkout@v2
+    #  with:
+    #      submodules: recursive
 
     - name: Run the test
       run: |
          cd cicd/k8s-calico-ipvs3-ha
-         ./config.sh
+         #./config.sh
          ./validation.sh
          cd -
          
-    - name: Clean test-bed
-      if: success() || failure()
-      run: |
-        cd cicd/k8s-calico-ipvs3-ha || true
-        #./rmconfig.sh
-        cd -
+    #- name: Clean test-bed
+    #  if: success() || failure()
+    #  run: |
+    #    cd cicd/k8s-calico-ipvs3-ha || true
+    #    ./rmconfig.sh
+    #    cd -

From b093b7753c010e8d2e09bcbc5e2b680dfc696060 Mon Sep 17 00:00:00 2001
From: Nikhil Malik <nikhil.malik@netlox.io>
Date: Wed, 10 Jan 2024 19:07:03 +0900
Subject: [PATCH 6/8] k8s-calico-ipvs3-ha cicd test updated

---
 .github/workflows/k8s-calico-ipvs3-ha.yml     |  22 ++--
 cicd/k8s-calico-ipvs3-ha/README               |   2 +-
 cicd/k8s-calico-ipvs3-ha/host_validation.sh   |  12 +-
 cicd/k8s-calico-ipvs3-ha/host_validation2.sh  |  19 +--
 .../host_validation2_with_sctp.sh             |  60 +++++++++
 .../host_validation_with_sctp.sh              |  50 ++++++++
 .../node_scripts/master.sh                    |   3 +
 cicd/k8s-calico-ipvs3-ha/validation.sh        |   3 +-
 .../validation_with_sctp.sh                   | 119 ++++++++++++++++++
 .../k8s-calico-ipvs3-ha/yaml/sctp_default.yml |   4 +-
 .../k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml |   4 +-
 11 files changed, 254 insertions(+), 44 deletions(-)
 create mode 100755 cicd/k8s-calico-ipvs3-ha/host_validation2_with_sctp.sh
 create mode 100755 cicd/k8s-calico-ipvs3-ha/host_validation_with_sctp.sh
 create mode 100755 cicd/k8s-calico-ipvs3-ha/validation_with_sctp.sh

diff --git a/.github/workflows/k8s-calico-ipvs3-ha.yml b/.github/workflows/k8s-calico-ipvs3-ha.yml
index 4fcbd13ad..7447ff2a6 100644
--- a/.github/workflows/k8s-calico-ipvs3-ha.yml
+++ b/.github/workflows/k8s-calico-ipvs3-ha.yml
@@ -16,21 +16,21 @@ jobs:
     #if: github.repository == 'loxilb-io/loxilb' 
     #      &&  github.event.inputs.tagName == ''
     steps:
-    #- name: Checkout
-    #  uses: actions/checkout@v2
-    #  with:
-    #      submodules: recursive
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+          submodules: recursive
 
     - name: Run the test
       run: |
          cd cicd/k8s-calico-ipvs3-ha
-         #./config.sh
+         ./config.sh
          ./validation.sh
          cd -
          
-    #- name: Clean test-bed
-    #  if: success() || failure()
-    #  run: |
-    #    cd cicd/k8s-calico-ipvs3-ha || true
-    #    ./rmconfig.sh
-    #    cd -
+    - name: Clean test-bed
+      if: success() || failure()
+      run: |
+        cd cicd/k8s-calico-ipvs3-ha || true
+        ./rmconfig.sh
+        cd -
diff --git a/cicd/k8s-calico-ipvs3-ha/README b/cicd/k8s-calico-ipvs3-ha/README
index 1e6e9dc91..b21a14a52 100644
--- a/cicd/k8s-calico-ipvs3-ha/README
+++ b/cicd/k8s-calico-ipvs3-ha/README
@@ -14,6 +14,6 @@ If you wish to create this scenario in your lab then install Vagrant and follow
 
 1. Run ./config.sh to setup the K8s cluster, client and LoxiLB nodes
 
-2. Run ./validation.sh to run the HA test. Test Results will be displayed at the end.
+2. Run ./validation.sh to run the TCP HA test or ./validation_with_sctp.sh to run TCP & SCTP HA Test. Test Results will be displayed at the end.
 
 3. Run ./rmconfig.sh to cleanup the setup.
diff --git a/cicd/k8s-calico-ipvs3-ha/host_validation.sh b/cicd/k8s-calico-ipvs3-ha/host_validation.sh
index e11dccb9b..30817fb4b 100755
--- a/cicd/k8s-calico-ipvs3-ha/host_validation.sh
+++ b/cicd/k8s-calico-ipvs3-ha/host_validation.sh
@@ -19,16 +19,10 @@ echo "Testing Service"
 echo "*********************************************"
 
 # iperf client accessing fullnat service
-stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 100 -i 1 -b 100M &> iperff.out &
+stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 60 -i 1 -b 100M &> iperff.out &
 
 # iperf client accessing default service
-stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out &
+stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 60 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out &
 
 echo "iperf client started"
-
-sleep 1
-
-stdbuf -oL nohup sctp_test -H 192.168.90.9 -h 20.20.20.1 -p 56004 -s -c 0 -x 100000  &> sdf.out &
-stdbuf -oL nohup sctp_test -H 30.30.30.1 -h 20.20.20.1 -p 56005 -s -c 0 -x 100000 &> sdd.out &
-
-echo "sctp_test client started"
+echo "phase-1 done"
diff --git a/cicd/k8s-calico-ipvs3-ha/host_validation2.sh b/cicd/k8s-calico-ipvs3-ha/host_validation2.sh
index d61ac1367..b29b337c5 100755
--- a/cicd/k8s-calico-ipvs3-ha/host_validation2.sh
+++ b/cicd/k8s-calico-ipvs3-ha/host_validation2.sh
@@ -1,13 +1,10 @@
 #!/bin/bash
+
 pkill iperf
-pkill sctp_test
 
 iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
 iperfd_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
 
-sdf_res=$(grep -i "Client: Sending packets.(100000/100000)" sdf.out)
-sdd_res=$(grep -i "Client: Sending packets.(100000/100000)" sdd.out)
-
 if [[ $iperff_res != 0 ]]; then
     echo -e "K8s-calico-ipvs3-ha TCP\t\t(fullnat)\t[OK]"
 else
@@ -22,19 +19,5 @@ else
     code=1
 fi
 
-if [[ ! -z $sdf_res ]]; then
-    echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[OK]"
-else
-    echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[FAILED]"
-    code=1
-fi
-
-if [[ ! -z $sdd_res ]]; then
-    echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[OK]"
-else
-    echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[FAILED]"
-    code=1
-fi
-
 rm *.out
 exit $code
diff --git a/cicd/k8s-calico-ipvs3-ha/host_validation2_with_sctp.sh b/cicd/k8s-calico-ipvs3-ha/host_validation2_with_sctp.sh
new file mode 100755
index 000000000..5c44702be
--- /dev/null
+++ b/cicd/k8s-calico-ipvs3-ha/host_validation2_with_sctp.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+for((i=0;i<50;i++))
+do
+echo "snd=100" 1> sd1.pipe
+echo "snd=100" 1> sd2.pipe
+sleep 1
+done
+
+
+echo "stats" 1> sd1.pipe
+echo "stats" 1> sd2.pipe
+
+echo "shutdown" 1> sd1.pipe
+echo "shutdown" 1> sd2.pipe
+
+pkill iperf
+pkill sctp_darn
+
+iperff_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
+iperfd_res=$(tail -n 1 iperff.out | xargs | cut -d ' ' -f 7)
+
+sdf_res=$(grep -i "Client: Sending packets.(100000/100000)" sdf.out)
+sdd_res=$(grep -i "Client: Sending packets.(100000/100000)" sdd.out)
+
+sdf_res1=$(grep -i "packets sent" sdf.out | xargs | cut -d ' ' -f 3)
+sdf_res2=$(grep -i "packets rec" sdf.out | xargs | cut -d ' ' -f 3)
+
+sdd_res1=$(grep -i "packets sent" sdd.out | xargs | cut -d ' ' -f 3)
+sdd_res2=$(grep -i "packets rec" sdd.out | xargs | cut -d ' ' -f 3)
+
+if [[ $iperff_res != 0 ]]; then
+    echo -e "K8s-calico-ipvs3-ha TCP\t\t(fullnat)\t[OK]"
+else
+    echo -e "K8s-calico-ipvs3-ha TCP\t\t(fullnat)\t[FAILED]"
+    code=1
+fi
+
+if [[ $iperfd_res != 0 ]]; then
+    echo -e "K8s-calico-ipvs3-ha TCP\t\t(default\t[OK]"
+else
+    echo -e "K8s-calico-ipvs3-ha TCP\t\t(default)\t[FAILED]"
+    code=1
+fi
+
+if [[ $sdf_res1 != 0 && $sdf_res2 != 0 && $sdf_res1 == $sdf_res2 ]]; then
+    echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[OK]"
+else
+    echo -e "K8s-calico-ipvs3-ha SCTP\t(fullnat)\t[FAILED]"
+    code=1
+fi
+
+if [[ $sdd_res1 != 0 && $sdd_res2 != 0 && $sdd_res1 == $sdd_res2 ]]; then
+    echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[OK]"
+else
+    echo -e "K8s-calico-ipvs3-ha SCTP\t(default)\t[FAILED]"
+    code=1
+fi
+rm *.pipe
+rm *.out
+exit $code
diff --git a/cicd/k8s-calico-ipvs3-ha/host_validation_with_sctp.sh b/cicd/k8s-calico-ipvs3-ha/host_validation_with_sctp.sh
new file mode 100755
index 000000000..ff29729b4
--- /dev/null
+++ b/cicd/k8s-calico-ipvs3-ha/host_validation_with_sctp.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+extIP=$(cat /vagrant/extIP)
+
+code=0
+
+echo Service IP: $extIP
+
+numECMP=$(birdc show route | grep $extIP -A 3 | grep via | wc -l)
+
+birdc show route | grep $extIP -A 3
+
+if [ $numECMP == "2" ]; then
+    echo "Host route [OK]"
+else
+    echo "Host route [NOK]"
+fi
+echo -e "\n*********************************************"
+echo "Testing Service"
+echo "*********************************************"
+
+# iperf client accessing fullnat service
+stdbuf -oL nohup iperf -c 20.20.20.1 -p 56002 -t 100 -i 1 -b 100M &> iperff.out &
+
+# iperf client accessing default service
+stdbuf -oL nohup iperf -c 20.20.20.1 -p 56003 -t 100 -i 1 -b 100M -B 30.30.30.1 &> iperfd.out &
+
+echo "iperf client started"
+
+sleep 1
+
+mkfifo sd1.pipe
+mkfifo sd2.pipe
+
+sleep infinity > sd1.pipe &
+sleep infinity > sd2.pipe &
+
+stdbuf -oL nohup sctp_darn -H 192.168.90.9 -h 20.20.20.1 -p 56004 -s -I < sd1.pipe &> sdf.out &
+stdbuf -oL nohup sctp_darn -H 30.30.30.1 -h 20.20.20.1 -p 56005 -s -I < sd2.pipe &> sdd.out &
+
+echo "sctp_test client started"
+
+sleep 2
+for((i=0;i<30;i++))
+do
+echo "snd=100" 1> sd1.pipe
+echo "snd=100" 1> sd2.pipe
+sleep 1
+done
+echo "phase-1 done"
+exit 0
diff --git a/cicd/k8s-calico-ipvs3-ha/node_scripts/master.sh b/cicd/k8s-calico-ipvs3-ha/node_scripts/master.sh
index 8a4ee7477..a5cdecaac 100644
--- a/cicd/k8s-calico-ipvs3-ha/node_scripts/master.sh
+++ b/cicd/k8s-calico-ipvs3-ha/node_scripts/master.sh
@@ -40,6 +40,9 @@ kubeadm token create --print-join-command > $config_path/join.sh
 curl https://raw.githubusercontent.com/projectcalico/calico/v${CALICO_VERSION}/manifests/calico.yaml -O
 
 kubectl apply -f calico.yaml
+kubectl patch configmap/calico-config -n kube-system --type merge \
+  -p '{"data":{"veth_mtu": "8900"}}'
+kubectl rollout restart daemonset calico-node -n kube-system
 
 sudo -i -u vagrant bash << EOF
 whoami
diff --git a/cicd/k8s-calico-ipvs3-ha/validation.sh b/cicd/k8s-calico-ipvs3-ha/validation.sh
index be2b9b899..ee9716394 100755
--- a/cicd/k8s-calico-ipvs3-ha/validation.sh
+++ b/cicd/k8s-calico-ipvs3-ha/validation.sh
@@ -80,6 +80,7 @@ echo -e " BACKUP\t: $backup\n"
 vagrant ssh host -c 'sudo /vagrant/host_validation.sh' 2> /dev/null
 
 sleep 15
+echo -e "phase-2 begins..\n"
 
 count=1
 sync=0
@@ -112,7 +113,7 @@ fi
 echo "Restarting MASTER:$master.."
 vagrant ssh $master -c 'sudo docker restart loxilb' 2> /dev/null
 
-sleep 30
+sleep 50
 
 sudo rm extIP
 vagrant ssh host -c 'sudo /vagrant/host_validation2.sh' 2> /dev/null
diff --git a/cicd/k8s-calico-ipvs3-ha/validation_with_sctp.sh b/cicd/k8s-calico-ipvs3-ha/validation_with_sctp.sh
new file mode 100755
index 000000000..393eaa395
--- /dev/null
+++ b/cicd/k8s-calico-ipvs3-ha/validation_with_sctp.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+set -eo pipefail
+source ../common.sh
+echo k8s-calico-ipvs3-ha
+
+if [ "$1" ]; then
+  KUBECONFIG="$1"
+fi
+
+# Set space as the delimiter
+IFS=' '
+
+for((i=0; i<120; i++))
+do
+  extLB=$(vagrant ssh master -c 'kubectl get svc' 2> /dev/null | grep "tcp-lb-default")
+  read -a strarr <<< "$extLB"
+  len=${#strarr[*]}
+  if [[ $((len)) -lt 6 ]]; then
+    echo "Can't find tcp-lb service"
+    sleep 1
+    continue
+  fi 
+  if [[ ${strarr[3]} != *"none"* ]]; then
+    extIP="$(cut -d'-' -f2 <<<${strarr[3]})"
+    break
+  fi
+  echo "No external LB allocated"
+  sleep 1
+done
+
+## Any routing updates  ??
+#sleep 30
+
+echo Service IP : $extIP
+echo $extIP > extIP
+echo -e "\nEnd Points List"
+echo "******************************************************************************"
+vagrant ssh master -c 'kubectl get endpoints -A' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\nSVC List"
+echo "******************************************************************************"
+vagrant ssh master -c 'kubectl get svc' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\nPod List"
+echo "******************************************************************************"
+vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\nLB List"
+echo -e "\n---- LLB1 ----"
+echo "******************************************************************************"
+vagrant ssh llb1 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\n---- LLB2 ----"
+vagrant ssh llb2 -c 'sudo ./loxicmd get lb -o wide' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\nEP List"
+echo -e "\n---- LLB1 ----"
+echo "******************************************************************************"
+vagrant ssh llb1 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\n---- LLB2 ----"
+vagrant ssh llb2 -c 'sudo ./loxicmd get ep -o wide' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\nTEST RESULTS"
+echo "******************************************************************************"
+
+master="llb1"
+backup="llb2"
+
+state=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/cistate/all' -H 'accept: application/json')
+
+if [[ $state == *"BACKUP"* ]]; then
+  master="llb2"
+  backup="llb1"
+fi
+
+echo -e "\n MASTER\t: $master"
+echo -e " BACKUP\t: $backup\n"
+
+vagrant ssh host -c 'sudo /vagrant/host_validation_with_sctp.sh' 2> /dev/null
+
+#sleep 15
+echo -e "phase-2 begins..\n"
+
+count=1
+sync=0
+while [[ $count -le 5 ]] ; do
+echo -e "\nStatus at MASTER:$master\n"
+vagrant ssh $master -c "sudo ./loxicmd get ct | grep est" 2> /dev/null
+
+echo -e "\nStatus at BACKUP:$backup\n"
+vagrant ssh $backup -c "sudo ./loxicmd get ct | grep est" 2> /dev/null
+
+nres1=$(curl -sX 'GET' 'http://192.168.80.252:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
+nres2=$(curl -sX 'GET' 'http://192.168.80.253:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
+
+if [[ $nres1 == $nres2 ]]; then
+    echo -e "\nConnections sync successful!!!\n"
+    sync=1
+    break;
+fi
+echo -e "\nConnections sync pending.. Let's wait a little more..\n"
+count=$(( $count + 1 ))
+sleep 2
+done
+
+if [[ $sync == 0 ]]; then
+    echo -e "\nConnection Sync failed\n"
+    vagrant ssh host -c 'sudo pkill iperf; sudo pkill sctp_test; sudo rm -rf *.out'
+    exit 1
+fi
+
+echo "Restarting MASTER:$master.."
+vagrant ssh $master -c 'sudo docker restart loxilb' 2> /dev/null
+
+sleep 30
+
+sudo rm extIP
+vagrant ssh host -c 'sudo /vagrant/host_validation2_with_sctp.sh' 2> /dev/null
diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml
index 324087241..0359fcb70 100644
--- a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml
+++ b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_default.yml
@@ -29,7 +29,7 @@ spec:
       image: loxilbio/sctp-darn:latest
       imagePullPolicy: Always
       #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"]
-      #command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"]
-      command: ["sctp_test","-H", "0.0.0.0","-P", "9999", "-l"]
+      command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"]
+      #command: ["sctp_test","-H", "0.0.0.0","-P", "9999", "-l"]
       ports:
         - containerPort: 9999
diff --git a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml
index ff69e7058..e010624f5 100644
--- a/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml
+++ b/cicd/k8s-calico-ipvs3-ha/yaml/sctp_fullnat.yml
@@ -29,7 +29,7 @@ spec:
       image: loxilbio/sctp-darn:latest
       imagePullPolicy: Always
       #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 6 ; done"]
-      #command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"]
-      command: ["sctp_test","-H", "0.0.0.0","-P", "9999", "-l"]
+      command: ["sctp_darn","-H", "0.0.0.0","-P", "9999", "-l"]
+      #command: ["sctp_test","-H", "0.0.0.0","-P", "9999", "-l"]
       ports:
         - containerPort: 9999

From ab95389de415c9caf0aef69f60c871d588a59fe8 Mon Sep 17 00:00:00 2001
From: Nikhil Malik <nikhil.malik@netlox.io>
Date: Thu, 11 Jan 2024 11:04:39 +0900
Subject: [PATCH 7/8] k8s-calico-ipvs3 CICD test scripts updated

---
 cicd/k8s-calico-ipvs3/config.sh            | 27 ++++++++++++++++++++++
 cicd/k8s-calico-ipvs3/node_scripts/host.sh |  1 +
 2 files changed, 28 insertions(+)

diff --git a/cicd/k8s-calico-ipvs3/config.sh b/cicd/k8s-calico-ipvs3/config.sh
index 09b5c1767..7dcbcfb3e 100755
--- a/cicd/k8s-calico-ipvs3/config.sh
+++ b/cicd/k8s-calico-ipvs3/config.sh
@@ -36,3 +36,30 @@ sudo sysctl net.ipv4.conf.vboxnet1.arp_accept=1
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_onearm.yml' 2> /dev/null
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/udp_onearm.yml' 2> /dev/null
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_onearm.yml' 2> /dev/null
+
+for((i=1; i<=60; i++))
+do
+    fin=1
+    pods=$(vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null | grep -v "NAMESPACE")
+
+    while IFS= read -a pods; do
+        read -a pod <<< "$pods"
+        if [[ ${pod[3]} != *"Running"* ]]; then
+            echo "${pod[1]} is not UP yet"
+            fin=0
+        fi
+    done <<< "$pods"
+    if [ $fin == 1 ];
+    then
+        break;
+    fi
+    echo "Will try after 10s"
+    sleep 10
+done
+
+if [[ $fin == 0 ]]; then
+echo "Cluster is not ready"
+exit 1
+fi
+echo "Cluster is ready"
+
diff --git a/cicd/k8s-calico-ipvs3/node_scripts/host.sh b/cicd/k8s-calico-ipvs3/node_scripts/host.sh
index 0018bc1c1..5b5efd476 100755
--- a/cicd/k8s-calico-ipvs3/node_scripts/host.sh
+++ b/cicd/k8s-calico-ipvs3/node_scripts/host.sh
@@ -1,4 +1,5 @@
 # Install Bird to work with k3s
+sudo apt-get update
 sudo apt-get -y install bird2 lksctp-tools
 
 sleep 5

From 65755350c671b05f513d010161e6b4bda7de3c6d Mon Sep 17 00:00:00 2001
From: nik-netlox <66406124+nik-netlox@users.noreply.github.com>
Date: Thu, 11 Jan 2024 11:06:48 +0900
Subject: [PATCH 8/8] Update k8s-calico-ipvs3-ha.yml

---
 .github/workflows/k8s-calico-ipvs3-ha.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/k8s-calico-ipvs3-ha.yml b/.github/workflows/k8s-calico-ipvs3-ha.yml
index 7447ff2a6..09d88e298 100644
--- a/.github/workflows/k8s-calico-ipvs3-ha.yml
+++ b/.github/workflows/k8s-calico-ipvs3-ha.yml
@@ -1,8 +1,8 @@
 name: K8s-Calico-Cluster-IPVS3-HA-Sanity-CI
 on:
- #schedule:
+ schedule:
  # Runs "At 17:00 UTC every day-of-week"
- #- cron: '0 17 * * *'
+ - cron: '0 17 * * *'
  workflow_dispatch:
     inputs:
       testName:
@@ -13,8 +13,8 @@ jobs:
   test-runner:
     name: k8s-calico-cluster-ipvs3-ha-sanity
     runs-on: self-hosted
-    #if: github.repository == 'loxilb-io/loxilb' 
-    #      &&  github.event.inputs.tagName == ''
+    if: github.repository == 'loxilb-io/loxilb' 
+          &&  github.event.inputs.tagName == ''
     steps:
     - name: Checkout
       uses: actions/checkout@v2