Merge pull request loxilb-io#296 from TrekkieCoder/main

PR - Fixes after multihoming cicd runs
nik-netlox · May 31, 2023 · ec84b69 · ec84b69
2 parents ec1730c + f5d02ce
commit ec84b69
Show file tree

Hide file tree

Showing 14 changed files with 445 additions and 26 deletions.
diff --git a/.github/workflows/advanced-sanity.yml b/.github/workflows/advanced-sanity.yml
@@ -132,4 +132,16 @@ jobs:
              ./validation.sh
              ./rmconfig.sh
              cd -
+      - run: |
+             cd cicd/nat66udp/
+             ./config.sh
+             ./validation.sh
+             ./rmconfig.sh
+             cd -
+      - run: |
+             cd cicd/nat66sctp/
+             #./config.sh
+             #./validation.sh
+             #./rmconfig.sh
+             cd -
       - run: echo ${{ github.event.inputs.userInput }}
diff --git a/.github/workflows/k0s-weave.yml b/.github/workflows/k0s-weave.yml
@@ -0,0 +1,44 @@
+name: K0s-Weave-Sanity-CI
+on:
+ #schedule:
+ # Runs "At 11:00 UTC every day-of-week"
+ #- cron: '0 11 * * *'
+ workflow_dispatch:
+    inputs:
+      testName:
+        description: 'Test Run-Name'     
+        required: true
+        default: 'k0s-weave'
+jobs:
+  build:
+    name: k0s-weave-sanity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - run: |
+         echo "KUBECONFIG=/var/lib/k0s/pki/admin.conf" >> $GITHUB_ENV
+      - run: sudo ip addr add 192.169.20.59/32 dev lo
+      - run: sudo swapoff -a
+      - run: sudo curl -sSLf https://get.k0s.sh | sudo sh
+      - run: sudo k0s install controller --enable-worker -c cicd/k0s-weave/k0s.yaml
+      - run: sudo k0s start
+      - run: |
+         sleep 120
+         sudo k0s kubectl apply -f https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s.yaml
+      - run: |
+         sleep 60
+         sudo k0s kubectl get pods -A
+      - run: |
+         sleep 10
+         sudo k0s kubectl taint nodes --all node-role.kubernetes.io/master:NoSchedule-
+         sleep 60
+         sudo k0s kubectl get nodes
+         sudo k0s kubectl get pods -A
+      - run: |
+         cd cicd/k0s-weave/
+         ./config.sh
+         ./validation.sh
+         ./rmconfig.sh
+         cd -
diff --git a/cicd/k0s-weave/config.sh b/cicd/k0s-weave/config.sh
@@ -121,14 +121,15 @@ else
   # Install k0s 
   sudo ip addr add 192.169.20.59/32 dev lo
   sudo curl -sSLf https://get.k0s.sh | sudo sh
-  sudo k0s install controller --enable-worker -c k0s.yaml
+  sudo k0s install controller --enable-worker 
+  #sudo k0s install controller --enable-worker -c k0s.yaml
   sudo k0s start
 
   sleep 30
   sudo k0s kubectl apply -f https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s.yaml
 
   # Check kubectl works
-  sudo k0s kubectl $KUBECONFIG get pods -A
+  sudo k0s kubectl get pods -A
 
   # Remove taints in k0s if any (usually happens if started without cloud-manager)
   sudo k0s kubectl taint nodes --all node-role.kubernetes.io/master:NoSchedule-
@@ -151,13 +152,13 @@ sudo systemctl restart bird
 sleep 30
 
 # Start nginx pods and services for test(using kube-loxilb)
-sudo k0s kubectl $KUBECONFIG apply -f kube-loxilb.yml
+sudo k0s kubectl apply -f kube-loxilb.yml
 sleep 15
-sudo k0s kubectl $KUBECONFIG apply -f nginx-svc-lb1.yml
+sudo k0s kubectl apply -f nginx-svc-lb1.yml
 sleep 15
-sudo k0s kubectl $KUBECONFIG apply -f udp-svc-lb.yml
+sudo k0s kubectl apply -f udp-svc-lb.yml
 
 sleep 30
 
 # External LB service must be created by now
-sudo k0s kubectl $KUBECONFIG get svc
+sudo k0s kubectl get svc
diff --git a/cicd/k0s-weave/k0s.yaml b/cicd/k0s-weave/k0s.yaml
@@ -38,7 +38,7 @@ spec:
   storage:
     etcd:
       externalCluster: null
-      peerAddress: 192.168.20.58
+      peerAddress: 192.169.20.59
     type: etcd
   telemetry:
     enabled: true

diff --git a/cicd/k0s-weave/rmconfig.sh b/cicd/k0s-weave/rmconfig.sh
@@ -10,9 +10,9 @@ fi
 
 source ../common.sh
 
-sudo k0s kubectl $KUBECONFIG delete -f nginx-svc-lb1.yml >> /dev/null 2>&1
-sudo k0s kubectl $KUBECONFIG delete -f udp-svc-lb.yml >> /dev/null 2>&1
-sudo k0s kubectl $KUBECONFIG delete -f kube-loxilb.yml >> /dev/null 2>&1
+sudo k0s kubectl delete -f nginx-svc-lb1.yml >> /dev/null 2>&1
+sudo k0s kubectl delete -f udp-svc-lb.yml >> /dev/null 2>&1
+sudo k0s kubectl delete -f kube-loxilb.yml >> /dev/null 2>&1
 
 disconnect_docker_hosts user r1
 disconnect_docker_hosts r1 llb1

diff --git a/cicd/k0s-weave/validation.sh b/cicd/k0s-weave/validation.sh
@@ -11,7 +11,7 @@ IFS=' '
 
 for((i=0; i<120; i++))
 do
-  extLB=$(sudo k0s kubectl $KUBECONFIG get svc | grep "udp-lb1")
+  extLB=$(sudo k0s kubectl get svc | grep "udp-lb1")
   read -a strarr <<< "$extLB"
   len=${#strarr[*]}
   if [[ $((len)) -lt 6 ]]; then
@@ -31,13 +31,7 @@ done
 sleep 30
 
 echo $extIP
-
-out=$($hexec user ../common/udp_client $extIP 55003)
-if [[ ${out} == *"Client"* ]]; then
-  echo "cluster-k0s (udp) [OK]"
-else
-  echo "cluster-k0s (udp) [FAILED]"
-fi
+sudo k0s kubectl get endpoints -A
 
 out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) 
 if [[ ${out} == *"Welcome to nginx"* ]]; then
@@ -57,3 +51,10 @@ else
   $dexec r1 ip route
   exit 1
 fi
+
+out=$($hexec user ../common/udp_client $extIP 55003)
+if [[ ${out} == *"Client"* ]]; then
+  echo "cluster-k0s (udp) [OK]"
+else
+  echo "cluster-k0s (udp) [FAILED]"
+fi
diff --git a/cicd/nat66sctp/config.sh b/cicd/nat66sctp/config.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+
+source ../common.sh
+
+echo "#########################################"
+echo "Spawning all hosts"
+echo "#########################################"
+
+spawn_docker_host --dock-type loxilb --dock-name llb1
+spawn_docker_host --dock-type host --dock-name l3h1
+spawn_docker_host --dock-type host --dock-name l3ep1
+spawn_docker_host --dock-type host --dock-name l3ep2
+spawn_docker_host --dock-type host --dock-name l3ep3
+
+echo "#########################################"
+echo "Connecting and configuring  hosts"
+echo "#########################################"
+
+
+connect_docker_hosts l3h1 llb1
+connect_docker_hosts l3ep1 llb1
+connect_docker_hosts l3ep2 llb1
+connect_docker_hosts l3ep3 llb1
+
+$hexec l3h1 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null
+$hexec l3h1 sysctl net.ipv6.conf.default.disable_ipv6=0
+$hexec l3ep1 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null
+$hexec l3ep1 sysctl net.ipv6.conf.default.disable_ipv6=0
+$hexec l3ep2 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null
+$hexec l3ep2 sysctl net.ipv6.conf.default.disable_ipv6=0
+$hexec l3ep3 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null
+$hexec l3ep3 sysctl net.ipv6.conf.default.disable_ipv6=0
+$hexec llb1 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null
+$hexec llb1 sysctl net.ipv6.conf.default.disable_ipv6=0 2>&1 >> /dev/null
+$hexec llb1 sysctl net.ipv6.conf.all.disable_ipv6=0 2>&1 >> /dev/null
+$hexec llb1 sysctl net.ipv6.conf.all.forwarding=1 2>&1 >> /dev/null
+
+sleep 5
+
+#L3 config
+config_docker_host --host1 l3h1 --host2 llb1 --ptype phy --addr 10.10.10.1/24 --gw 10.10.10.254
+config_docker_host --host1 l3ep1 --host2 llb1 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254
+config_docker_host --host1 l3ep2 --host2 llb1 --ptype phy --addr 32.32.32.1/24 --gw 32.32.32.254
+config_docker_host --host1 l3ep3 --host2 llb1 --ptype phy --addr 33.33.33.1/24 --gw 33.33.33.254
+config_docker_host --host1 llb1 --host2 l3h1 --ptype phy --addr 10.10.10.254/24
+config_docker_host --host1 llb1 --host2 l3ep1 --ptype phy --addr 31.31.31.254/24
+config_docker_host --host1 llb1 --host2 l3ep2 --ptype phy --addr 32.32.32.254/24
+config_docker_host --host1 llb1 --host2 l3ep3 --ptype phy --addr 33.33.33.254/24
+
+## IPV6 Stuff
+$hexec l3h1 ip -6 addr add 3ffe::1/64 dev el3h1llb1
+$hexec l3h1 ip -6 route add default via 3ffe::2
+$hexec l3h1 ethtool --offload  el3h1llb1 rx off  tx off
+$hexec l3h1 ethtool -K el3h1llb1 gso off
+
+$hexec l3ep1 ip -6 addr add 4ffe::1/64 dev el3ep1llb1
+$hexec l3ep1 ip -6 route add default via 4ffe::2
+$hexec l3ep1 ethtool --offload  el3ep1llb1 rx off  tx off
+$hexec l3ep1 ethtool -K el3ep1llb1 gso off
+
+$hexec l3ep2 ip -6 addr add 5ffe::1/64 dev el3ep2llb1
+$hexec l3ep2 ip -6 route add default via 5ffe::2
+$hexec l3ep2 ethtool --offload  el3ep2llb1 rx off  tx off
+$hexec l3ep2 ethtool -K el3ep2llb1 gso off
+
+$hexec l3ep3 ip -6 addr add 6ffe::1/64 dev el3ep3llb1
+$hexec l3ep3 ip -6 route add default via 6ffe::2
+$hexec l3ep3 ethtool --offload  el3ep3llb1 rx off  tx off
+$hexec l3ep3 ethtool -K el3ep3llb1 gso off
+
+$hexec llb1 ip -6 addr add 3ffe::2/64 dev ellb1l3h1
+$hexec llb1 ip -6 addr add 4ffe::2/64 dev ellb1l3ep1
+$hexec llb1 ip -6 addr add 5ffe::2/64 dev ellb1l3ep2
+$hexec llb1 ip -6 addr add 6ffe::2/64 dev ellb1l3ep3
+$hexec llb1 ethtool --offload ellb1l3h1 rx off tx on
+$hexec llb1 ethtool -K ellb1l3h1 gso off
+$hexec llb1 ethtool --offload ellb1l3ep1 rx off tx on
+$hexec llb1 ethtool -K ellb1l3ep1 gso off
+$hexec llb1 ethtool --offload ellb1l3ep2 rx off tx on
+$hexec llb1 ethtool -K ellb1l3ep2 gso off
+$hexec llb1 ethtool --offload ellb1l3ep3 rx off tx on
+$hexec llb1 ethtool -K ellb1l3ep3 gso off
+
+sleep 5
+$dexec llb1 loxicmd create lb 20.20.20.1 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1
+$dexec llb1 loxicmd create lb  2001::1 --tcp=2020:8080 --endpoints=4ffe::1:1,5ffe::1:1,6ffe::1:1
+$dexec llb1 loxicmd create lb  2001::1 --sctp=2020:8080 --endpoints=4ffe::1:1,5ffe::1:1,6ffe::1:1
diff --git a/cicd/nat66sctp/rmconfig.sh b/cicd/nat66sctp/rmconfig.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+source ../common.sh
+
+disconnect_docker_hosts l3h1 llb1
+disconnect_docker_hosts l3ep1 llb1
+disconnect_docker_hosts l3ep2 llb1
+disconnect_docker_hosts l3ep3 llb1
+
+delete_docker_host llb1
+delete_docker_host l3h1
+delete_docker_host l3ep1
+delete_docker_host l3ep2
+delete_docker_host l3ep3
+
+echo "#########################################"
+echo "Deleted testbed"
+echo "#########################################"
diff --git a/cicd/nat66sctp/validation.sh b/cicd/nat66sctp/validation.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+source ../common.sh
+echo "SCENARIO nat66sctp"
+$hexec l3ep1  socat -v -T2 SCTP6-LISTEN:8080,reuseaddr,fork system:"echo 'server1'; cat" >/dev/null 2>&1 &
+$hexec l3ep2  socat -v -T2 SCTP6-LISTEN:8080,reuseaddr,fork system:"echo 'server2'; cat" >/dev/null 2>&1 &
+$hexec l3ep3  socat -v -T2 SCTP6-LISTEN:8080,reuseaddr,fork system:"echo 'server3'; cat" >/dev/null 2>&1 &
+
+sleep 5
+code=0
+servArr=( "server1" "server2" "server3" )
+ep=( "4ffe::1" "5ffe::1" "6ffe::1" )
+j=0
+waitCount=0
+while [ $j -le 2 ]
+do
+    svr=${ep[j]}
+    res=$($hexec l3h1 bash -c "socat -T10 - SCTP:[${ep[j]}]:8080")
+    if [[ $res == *"${servArr[j]}"* ]]
+    then
+        echo "${servArr[j]} UP"
+        j=$(( $j + 1 ))
+    else
+        echo "Waiting for ${servArr[j]}(${ep[j]})"
+        waitCount=$(( $waitCount + 1 ))
+        if [[ $waitCount == 10 ]];
+        then
+            echo "All Servers are not UP"
+            echo nat66sctp [FAILED]
+            sudo pkill socat
+            exit 1
+        fi
+    fi
+    sleep 1
+done
+
+nid=0
+for i in {1..4}
+do
+for j in {0..2}
+do
+    res=$($hexec l3h1 bash -c "socat -T10 - SCTP:[2001::1]:2020")
+    echo $res
+    ids=`echo "${res//[!0-9]/}"`
+    if [[ $res == *"server"* ]]; then
+      ids=`echo "${res//[!0-9]/}"`
+      if [[ $nid == 0 ]];then
+        nid=$((($ids + 1)%4))
+        if [[ $nid == 0 ]];then
+          nid=1
+        fi
+      elif [[ $nid != $((ids)) ]]; then
+        echo "Expected server$nid got server$((ids))"
+        code=1
+      fi
+      nid=$((($ids + 1)%4))
+      if [[ $nid == 0 ]];then
+        nid=1
+      fi
+    else
+      code=1
+    fi
+    sleep 1
+done
+done
+if [[ $code == 0 ]]
+then
+    echo nat66sctp [OK]
+else
+    echo nat66sctp [FAILED]
+fi
+sudo pkill socat
+exit $code
+