From 38a894e43b2bb2a6f3046e9948182a5a0c61f949 Mon Sep 17 00:00:00 2001
From: Nikhil Malik <nikhil.malik@netlox.io>
Date: Wed, 10 Apr 2024 18:43:25 +0900
Subject: [PATCH] sctp multihoming cicd added

---
 cicd/sctpmh/config.sh      |  96 +++++++++++++++++++++++++++++++
 cicd/sctpmh/input          |   6 ++
 cicd/sctpmh/rmconfig.sh    |  32 +++++++++++
 cicd/sctpmh/validation.sh  |  14 +++++
 cicd/sctpmh/validation1.sh |  61 ++++++++++++++++++++
 cicd/sctpmh/validation2.sh |  93 ++++++++++++++++++++++++++++++
 cicd/sctpmh/validation3.sh | 113 +++++++++++++++++++++++++++++++++++++
 cicd/sctpmh/validation4.sh | 113 +++++++++++++++++++++++++++++++++++++
 8 files changed, 528 insertions(+)
 create mode 100755 cicd/sctpmh/config.sh
 create mode 100644 cicd/sctpmh/input
 create mode 100755 cicd/sctpmh/rmconfig.sh
 create mode 100755 cicd/sctpmh/validation.sh
 create mode 100755 cicd/sctpmh/validation1.sh
 create mode 100755 cicd/sctpmh/validation2.sh
 create mode 100755 cicd/sctpmh/validation3.sh
 create mode 100755 cicd/sctpmh/validation4.sh

diff --git a/cicd/sctpmh/config.sh b/cicd/sctpmh/config.sh
new file mode 100755
index 000000000..4f2f519ac
--- /dev/null
+++ b/cicd/sctpmh/config.sh
@@ -0,0 +1,96 @@
+#!/bin/bash
+
+source ../common.sh
+
+echo "#########################################"
+echo "Spawning all hosts"
+echo "#########################################"
+
+spawn_docker_host --dock-type loxilb --dock-name llb1
+spawn_docker_host --dock-type host --dock-name ep1
+spawn_docker_host --dock-type host --dock-name r1
+spawn_docker_host --dock-type host --dock-name r2
+spawn_docker_host --dock-type host --dock-name r3
+spawn_docker_host --dock-type host --dock-name r4
+spawn_docker_host --dock-type host --dock-name sw1
+spawn_docker_host --dock-type host --dock-name sw2
+spawn_docker_host --dock-type host --dock-name user
+
+echo "#########################################"
+echo "Connecting and configuring  hosts"
+echo "#########################################"
+
+connect_docker_hosts user r1
+connect_docker_hosts user r2
+connect_docker_hosts r1 sw1
+connect_docker_hosts r2 sw1
+connect_docker_hosts sw1 llb1
+connect_docker_hosts llb1 sw2
+connect_docker_hosts sw2 r3
+connect_docker_hosts sw2 r4
+connect_docker_hosts r3 ep1
+connect_docker_hosts r4 ep1
+
+create_docker_host_cnbridge --host1 sw1 --host2 llb1
+create_docker_host_cnbridge --host1 sw1 --host2 r1
+create_docker_host_cnbridge --host1 sw1 --host2 r2
+
+create_docker_host_cnbridge --host1 sw2 --host2 llb1
+create_docker_host_cnbridge --host1 sw2 --host2 r3
+create_docker_host_cnbridge --host1 sw2 --host2 r4
+
+#node1 config
+config_docker_host --host1 user --host2 r1 --ptype phy --addr 1.1.1.1/24 --gw 1.1.1.254
+config_docker_host --host1 user --host2 r2 --ptype phy --addr 2.2.2.1/24
+config_docker_host --host1 r1 --host2 user --ptype phy --addr 1.1.1.254/24
+config_docker_host --host1 r2 --host2 user --ptype phy --addr 2.2.2.254/24
+
+create_docker_host_vlan --host1 llb1 --host2 sw1 --id 11 --ptype untagged
+create_docker_host_vlan --host1 r1 --host2 sw1 --id 11 --ptype untagged
+create_docker_host_vlan --host1 r2 --host2 sw1 --id 11 --ptype untagged
+config_docker_host --host1 r1 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.253/24 --gw 11.11.11.1
+config_docker_host --host1 r2 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.254/24 --gw 11.11.11.1
+config_docker_host --host1 llb1 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.1/24
+
+create_docker_host_vlan --host1 llb1 --host2 sw2 --id 10 --ptype untagged
+create_docker_host_vlan --host1 r3 --host2 sw2 --id 10 --ptype untagged
+create_docker_host_vlan --host1 r4 --host2 sw2 --id 10 --ptype untagged
+
+config_docker_host --host1 llb1 --host2 sw2 --ptype vlan --id 10 --addr 10.10.10.1/24
+config_docker_host --host1 r3 --host2 sw2 --ptype vlan --id 10 --addr 10.10.10.253/24 --gw 10.10.10.1
+config_docker_host --host1 r4 --host2 sw2 --ptype vlan --id 10 --addr 10.10.10.254/24 --gw 10.10.10.1
+
+config_docker_host --host1 ep1 --host2 r3 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254
+config_docker_host --host1 ep1 --host2 r4 --ptype phy --addr 32.32.32.1/24
+config_docker_host --host1 r3 --host2 ep1 --ptype phy --addr 31.31.31.254/24
+config_docker_host --host1 r4 --host2 ep1 --ptype phy --addr 32.32.32.254/24
+
+$hexec user ip route change default via 1.1.1.254
+$hexec ep1 ip route change default via 31.31.31.254
+
+# Backup paths in user
+$hexec user ip route add 124.124.124.1/32 via 2.2.2.254
+$hexec user ip route add 125.125.125.1/32 via 2.2.2.254
+$hexec user ip route add 134.134.134.1/32 via 2.2.2.254
+$hexec user ip route add 135.135.135.1/32 via 2.2.2.254
+
+$hexec ep1 ip route add 124.124.124.1/32 via 32.32.32.254
+$hexec ep1 ip route add 125.125.125.1/32 via 32.32.32.254
+$hexec ep1 ip route add 134.134.134.1/32 via 32.32.32.254
+$hexec ep1 ip route add 135.135.135.1/32 via 32.32.32.254
+
+$hexec llb1 ip route add 1.1.1.0/24 via 11.11.11.253
+$hexec llb1 ip route add 2.2.2.0/24 via 11.11.11.254
+
+$hexec llb1 ip route add 31.31.31.0/24 via 10.10.10.253
+$hexec llb1 ip route add 32.32.32.0/24 via 10.10.10.254
+
+sleep 20
+##Create LB rule user->ep1
+create_lb_rule llb1 123.123.123.1 --secips=124.124.124.1,125.125.125.1 --sctp=2020:9999 --endpoints=31.31.31.1:1 --mode=fullnat
+
+##Create LB rule ep1->user
+create_lb_rule llb1 133.133.133.1 --secips=134.134.134.1,135.135.135.1 --sctp=2020:9999 --endpoints=1.1.1.1:1 --mode=fullnat
+
+$dexec llb1 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_9999 --probetype=none
+$dexec llb1 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_9999 --probetype=none
diff --git a/cicd/sctpmh/input b/cicd/sctpmh/input
new file mode 100644
index 000000000..6fb66a5e2
--- /dev/null
+++ b/cicd/sctpmh/input
@@ -0,0 +1,6 @@
+
+
+
+
+
+
diff --git a/cicd/sctpmh/rmconfig.sh b/cicd/sctpmh/rmconfig.sh
new file mode 100755
index 000000000..dd68b07eb
--- /dev/null
+++ b/cicd/sctpmh/rmconfig.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+echo "#########################################"
+echo "Removing testbed"
+echo "#########################################"
+
+source ../common.sh
+
+disconnect_docker_hosts user r1
+disconnect_docker_hosts user r2
+disconnect_docker_hosts r1 sw1
+disconnect_docker_hosts r2 sw1
+disconnect_docker_hosts sw1 llb1
+disconnect_docker_hosts llb1 sw2
+disconnect_docker_hosts sw2 r3
+disconnect_docker_hosts sw2 r4
+disconnect_docker_hosts r3 ep1
+disconnect_docker_hosts r4 ep1
+
+delete_docker_host llb1
+delete_docker_host user
+delete_docker_host r1
+delete_docker_host r2
+delete_docker_host r3
+delete_docker_host r4
+delete_docker_host sw1
+delete_docker_host sw2
+delete_docker_host ep1
+
+echo "#########################################"
+echo "Removed testbed"
+echo "#########################################"
diff --git a/cicd/sctpmh/validation.sh b/cicd/sctpmh/validation.sh
new file mode 100755
index 000000000..9e3fec23b
--- /dev/null
+++ b/cicd/sctpmh/validation.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+echo "SCTP Multihoming - Test case #1"
+echo -e "*********************************************************************************"
+./validation1.sh
+echo -e "\n\n\nSCTP Multihoming - Test case #2"
+echo -e "*********************************************************************************"
+./validation2.sh
+echo -e "\n\n\nSCTP Multihoming - Test case #3"
+echo -e "*********************************************************************************"
+./validation3.sh
+echo -e "\n\n\nSCTP Multihoming - Test case #4"
+echo -e "*********************************************************************************"
+./validation4.sh
+echo -e "\n\n\n*********************************************************************************"
diff --git a/cicd/sctpmh/validation1.sh b/cicd/sctpmh/validation1.sh
new file mode 100755
index 000000000..6043bf114
--- /dev/null
+++ b/cicd/sctpmh/validation1.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+source ../common.sh
+
+echo -e "sctpmh: SCTP Multihoming Basic Test - Client & EP Uni-homed and LB is Multi-homed\n"
+extIP="123.123.123.1"
+port=2020
+
+echo "SCTP Multihoming service sctp-lb -> $extIP:$port"
+echo -e "------------------------------------------------------------------------------------\n"
+
+$hexec ep1 sctp_darn -H 0.0.0.0  -P 9999 -l 2>&1> /dev/null &
+
+sleep 10
+$hexec user stdbuf -oL sctp_darn -H 1.1.1.1 -h $extIP -p $port -s < input > user.out
+sleep 3
+
+exp="New connection, peer addresses
+123.123.123.1:2020
+124.124.124.1:2020
+125.125.125.1:2020"
+
+res=`cat user.out | grep -A 3 "New connection, peer addresses"`
+sudo rm -rf user.out
+sudo pkill sctp_darn
+
+if [[ "$res" == "$exp" ]]; then
+    echo $res
+    echo -e "\nsctpmh SCTP Multihoming service Basic Test [OK]\n"
+else
+    echo "sctpmh SCTP Multihoming service Basic Test [NOK]"
+    echo "Expected : $exp"
+    echo "Received : $res"
+    ## Dump some debug info
+    echo "system route-info"
+    echo -e "\nuser"
+    sudo ip netns exec user ip route
+    echo -e "\nr1"
+    sudo ip netns exec r1 ip route
+    echo -e "\nr2"
+    sudo ip netns exec r2 ip route
+    echo -e "\nllb1"
+    sudo ip netns exec llb1 ip route
+    echo -e "\nr3"
+    sudo ip netns exec r3 ip route
+    echo -e "\nr4"
+    sudo ip netns exec r4 ip route
+    echo "-----------------------------"
+
+    echo -e "\nllb1 lb-info"
+    $dexec llb1 loxicmd get lb
+    echo "llb1 ep-info"
+    $dexec llb1 loxicmd get ep
+    echo "llb1 bpf-info"
+    $dexec llb1 ntc filter show dev eth0 ingress
+    echo "BFP trace -- "
+    sudo timeout 5 cat  /sys/kernel/debug/tracing/trace_pipe
+    sudo killall -9 cat
+    echo "BFP trace -- "
+    exit 1
+fi
+echo -e "------------------------------------------------------------------------------------\n\n\n"
diff --git a/cicd/sctpmh/validation2.sh b/cicd/sctpmh/validation2.sh
new file mode 100755
index 000000000..8b92fef37
--- /dev/null
+++ b/cicd/sctpmh/validation2.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+source ../common.sh
+
+echo -e "sctpmh: SCTP Multihoming - Multipath Test, Client and LB Multihomed, EP is uni-homed\n"
+extIP="123.123.123.1"
+port=2020
+
+echo "SCTP Multihoming service sctp-lb(Multipath traffic) -> $extIP:$port"
+echo -e "------------------------------------------------------------------------------------\n"
+
+$hexec ep1 sctp_test -H 0.0.0.0  -P 9999 -l > ep1.out &
+sleep 2
+
+$hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -m 100 -x 100000 > user.out &
+sleep 1
+#Path counters
+p1c_old=0
+p1c_new=0
+p2c_old=0
+p2c_new=0
+p3c_old=0
+p3c_new=0
+
+for((i=0;i<100;i++)) do
+    fin=`tail -n 100 user.out | grep "Client: Sending packets.(100000/100000)"`
+    if [[ ! -z $fin ]]; then
+        fin=1
+        echo "sctp_test done."
+        break;
+    fi
+    $dexec llb1 loxicmd get ct
+    echo -e "\n"
+    p1c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "123.123.123.1 | 1.1.1.1" | xargs | cut -d '|' -f 10)
+    p2c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "124.124.124.1 | 2.2.2.1" | xargs | cut -d '|' -f 10)
+    p3c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "125.125.125.1 | 1.1.1.1" | xargs | cut -d '|' -f 10)
+    
+    echo "Counters: $p1c_new $p2c_new $p3c_new"
+
+    if [[ $p1c_new -gt $p1c_old ]]; then
+        echo "Path 1: 1.1.1.1 -> 123.123.123.1 [ACTIVE]"
+        p1=1
+    else
+        echo "Path 1: 1.1.1.1 -> 123.123.123.1 [NOT ACTIVE]"
+    fi
+
+    if [[ $p2c_new -gt $p2c_old ]]; then
+        echo "Path 2: 2.2.2.1 -> 124.124.124.1 [ACTIVE]"
+        p2=1
+    else
+        echo "Path 2: 2.2.2.1 -> 124.124.124.1 [NOT ACTIVE]"
+    fi
+
+    if [[ $p3c_new -gt $p3c_old ]]; then
+        echo "Path 3: 1.1.1.1 -> 125.125.125.1 [ACTIVE]"
+        p3=1
+    else
+        echo "Path 3: 1.1.1.1 -> 125.125.125.1 [NOT ACTIVE]"
+    fi
+    p1c_old=$p1c_new
+    p2c_old=$p1c_new
+    p2c_old=$p1c_new
+    echo -e "\n"
+    sleep 5
+done
+
+#sudo rm -rf user.out
+sudo pkill sctp_test
+sudo rm *.out
+if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 ]]; then
+    echo "sctpmh SCTP Multihoming Multipath [OK]"
+else
+    echo "sctpmh SCTP Multihoming Multipath [NOK]"
+    echo -e "\nuser"
+    sudo ip netns exec user ip route
+    echo -e "\nr1"
+    sudo ip netns exec r1 ip route
+    echo -e "\nr2"
+    sudo ip netns exec r2 ip route
+    echo -e "\nllb1"
+    sudo ip netns exec llb1 ip route
+    echo -e "\nr3"
+    sudo ip netns exec r3 ip route
+    echo -e "\nr4"
+    sudo ip netns exec r4 ip route
+    echo "-----------------------------"
+
+    echo -e "\nllb1 lb-info"
+    $dexec llb1 loxicmd get lb
+    echo "llb1 ep-info"
+    $dexec llb1 loxicmd get ep
+    exit 1
+fi
+echo -e "------------------------------------------------------------------------------------\n\n\n"
diff --git a/cicd/sctpmh/validation3.sh b/cicd/sctpmh/validation3.sh
new file mode 100755
index 000000000..fa3b308e4
--- /dev/null
+++ b/cicd/sctpmh/validation3.sh
@@ -0,0 +1,113 @@
+#!/bin/bash
+source ../common.sh
+
+echo -e "sctpmh: SCTP Multihoming - Multipath Failover Test. Client and LB Multihomed, EP is uni-homed\n"
+extIP="123.123.123.1"
+port=2020
+
+echo "SCTP Multihoming service sctp-lb(Multipath traffic) -> $extIP:$port"
+echo -e "------------------------------------------------------------------------------------\n"
+
+echo -e "\nTraffic Flow: User -> LB -> EP "
+
+$hexec ep1 sctp_test -H 0.0.0.0  -P 9999 -l > ep1.out &
+sleep 2
+
+$hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -m 100 -x 100000 > user.out &
+sleep 5
+#Path counters
+p1c_old=0
+p1c_new=0
+p2c_old=0
+p2c_new=0
+p3c_old=0
+p3c_new=0
+down=0
+code=0
+for((i=0;i<200;i++)) do
+    fin=`tail -n 100 user.out | grep "Client: Sending packets.(100000/100000)"`
+    if [[ ! -z $fin ]]; then
+        fin=1
+        echo "sctp_test done."
+        break;
+    fi
+    $dexec llb1 loxicmd get ct
+    echo -e "\n"
+    p1c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "123.123.123.1 | 1.1.1.1" | xargs | cut -d '|' -f 10)
+    p2c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "124.124.124.1 | 2.2.2.1" | xargs | cut -d '|' -f 10)
+    p3c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "125.125.125.1 | 1.1.1.1" | xargs | cut -d '|' -f 10)
+    
+    echo "Counters: $p1c_new $p2c_new $p3c_new"
+
+    if [[ $p1c_new -gt $p1c_old ]]; then
+        echo "Path 1: 1.1.1.1 -> 123.123.123.1 [ACTIVE]"
+        p1=1
+        if [[ $down == 1 ]]; then
+            echo "This path shouldn't be ACTIVE"
+            code=1
+        fi
+        echo "Turning off this path from User->LB"
+        $hexec user ip link set euserr1 down;
+        down=1
+        p1c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "123.123.123.1 | 1.1.1.1" | xargs | cut -d '|' -f 10)
+    else
+        if [[ $down == 1 ]]; then
+            p1dok=1
+            echo "Path 1: 1.1.1.1 -> 123.123.123.1 NOT ACTIVE - [OK]"
+        else  
+            echo "Path 1: 1.1.1.1 -> 123.123.123.1 [NOT ACTIVE]"
+        fi
+    fi
+
+    if [[ $p2c_new -gt $p2c_old ]]; then
+        echo "Path 2: 2.2.2.1 -> 124.124.124.1 [ACTIVE]"
+        p2=1
+    else
+        echo "Path 2: 2.2.2.1 -> 124.124.124.1 [NOT ACTIVE]"
+    fi
+
+    if [[ $p3c_new -gt $p3c_old ]]; then
+        echo "Path 3: 1.1.1.1 -> 125.125.125.1 [ACTIVE]"
+        p3=1
+    else
+        echo "Path 3: 1.1.1.1 -> 125.125.125.1 [NOT ACTIVE]"
+    fi
+    p1c_old=$p1c_new
+    p2c_old=$p1c_new
+    p2c_old=$p1c_new
+    echo -e "\n"
+    sleep 5
+done
+
+sudo rm -rf *.out
+sudo pkill sctp_test
+
+#Restore
+$hexec user ip link set euserr1 up
+$hexec user ip route add default via 1.1.1.254
+
+if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $p1dok == 1 && $code == 0 ]]; then
+    echo "sctpmh SCTP Multihoming Multipath Failover [OK]"
+else
+    echo "sctpmh SCTP Multihoming Multipath Failover [NOK]"
+    echo -e "\nuser"
+    sudo ip netns exec user ip route
+    echo -e "\nr1"
+    sudo ip netns exec r1 ip route
+    echo -e "\nr2"
+    sudo ip netns exec r2 ip route
+    echo -e "\nllb1"
+    sudo ip netns exec llb1 ip route
+    echo -e "\nr3"
+    sudo ip netns exec r3 ip route
+    echo -e "\nr4"
+    sudo ip netns exec r4 ip route
+    echo "-----------------------------"
+
+    echo -e "\nllb1 lb-info"
+    $dexec llb1 loxicmd get lb
+    echo "llb1 ep-info"
+    $dexec llb1 loxicmd get ep
+    exit 1
+fi
+echo -e "------------------------------------------------------------------------------------\n\n\n"
diff --git a/cicd/sctpmh/validation4.sh b/cicd/sctpmh/validation4.sh
new file mode 100755
index 000000000..aef8b4c01
--- /dev/null
+++ b/cicd/sctpmh/validation4.sh
@@ -0,0 +1,113 @@
+#!/bin/bash
+source ../common.sh
+
+echo -e "sctpmh: SCTP Multihoming - Multipath Failover Test. Client, LB and EP all Multihomed\n"
+extIP="133.133.133.1"
+port=2020
+
+echo "SCTP Multihoming service sctp-lb(Multipath traffic) -> $extIP:$port"
+echo -e "------------------------------------------------------------------------------------\n"
+
+echo -e "\nTraffic Flow: EP ---> LB ---> User"
+
+$hexec user sctp_test -H 1.1.1.1 -B 2.2.2.1  -P 9999 -l > user.out &
+sleep 2
+
+$hexec ep1 stdbuf -oL sctp_test -H 31.31.31.1 -B 32.32.32.1 -P 20000 -h $extIP -p $port -s -m 100 -x 100000 > ep1.out &
+sleep 5
+#Path counters
+p1c_old=0
+p1c_new=0
+p2c_old=0
+p2c_new=0
+p3c_old=0
+p3c_new=0
+down=0
+code=0
+for((i=0;i<200;i++)) do
+    fin=`tail -n 100 ep1.out | grep "Client: Sending packets.(100000/100000)"`
+    if [[ ! -z $fin ]]; then
+        fin=1
+        echo "sctp_test done."
+        break;
+    fi
+    $dexec llb1 loxicmd get ct
+    echo -e "\n"
+    p1c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "133.133.133.1 | 31.31.31.1" | xargs | cut -d '|' -f 10)
+    p2c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "134.134.134.1 | 32.32.32.1" | xargs | cut -d '|' -f 10)
+    p3c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "135.135.135.1 | 31.31.31.1" | xargs | cut -d '|' -f 10)
+    
+    echo "Counters: $p1c_new $p2c_new $p3c_new"
+
+    if [[ $p1c_new -gt $p1c_old ]]; then
+        echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 [ACTIVE]"
+        p1=1
+        if [[ $down == 1 ]]; then
+            echo "This path shouldn't be ACTIVE"
+            code=1
+        fi
+        echo -e "Turning off this path at User.\nEP----->LB--x-->User"
+        $hexec user ip link set euserr1 down;
+        down=1
+        p1c_new=$(sudo docker exec -i llb1 loxicmd get ct | grep "133.133.133.1 | 31.31.31.1" | xargs | cut -d '|' -f 10)
+    else
+        if [[ $down == 1 ]]; then
+            p1dok=1
+            echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 NOT ACTIVE - [OK]"
+        else  
+            echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 [NOT ACTIVE]"
+        fi
+    fi
+
+    if [[ $p2c_new -gt $p2c_old ]]; then
+        echo "Path 2: 32.32.32.1 -> 134.134.134.1 -> 2.2.2.1 [ACTIVE]"
+        p2=1
+    else
+        echo "Path 2: 32.32.32.1 -> 134.134.134.1 -> 2.2.2.1 [NOT ACTIVE]"
+    fi
+
+    if [[ $p3c_new -gt $p3c_old ]]; then
+        echo "Path 3: 31.31.31.1 -> 135.135.135.1 -> 1.1.1.1 [ACTIVE]"
+        p3=1
+    else
+        echo "Path 3: 31.31.31.1 -> 135.135.135.1 -> 1.1.1.1 [NOT ACTIVE]"
+    fi
+    p1c_old=$p1c_new
+    p2c_old=$p1c_new
+    p2c_old=$p1c_new
+    echo -e "\n"
+    sleep 5
+done
+
+sudo rm -rf *.out
+sudo pkill sctp_test
+
+#Restore
+$hexec user ip link set euserr1 up
+$hexec user ip route add default via 1.1.1.254
+
+if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $p1dok == 1 && $code == 0 ]]; then
+    echo "sctpmh SCTP Multihoming E2E Multipath Failover [OK]"
+else
+    echo "sctpmh SCTP Multihoming E2E Multipath Failover [NOK]"
+    echo -e "\nuser"
+    sudo ip netns exec user ip route
+    echo -e "\nr1"
+    sudo ip netns exec r1 ip route
+    echo -e "\nr2"
+    sudo ip netns exec r2 ip route
+    echo -e "\nllb1"
+    sudo ip netns exec llb1 ip route
+    echo -e "\nr3"
+    sudo ip netns exec r3 ip route
+    echo -e "\nr4"
+    sudo ip netns exec r4 ip route
+    echo "-----------------------------"
+
+    echo -e "\nllb1 lb-info"
+    $dexec llb1 loxicmd get lb
+    echo "llb1 ep-info"
+    $dexec llb1 loxicmd get ep
+    exit 1
+fi
+echo -e "------------------------------------------------------------------------------------\n\n\n"