Merge branch 'loxilb-io:main' into main

nik-netlox · Feb 8, 2024 · cbe35ed · cbe35ed
2 parents 5b0d5f0 + a956622
commit cbe35ed
Show file tree

Hide file tree

Showing 20 changed files with 596 additions and 162 deletions.
diff --git a/.github/workflows/docker-multiarch.yml b/.github/workflows/docker-multiarch.yml
@@ -0,0 +1,53 @@
+name:  Docker-Multi-Arch
+
+on:
+  workflow_dispatch:
+    inputs:
+      tagName:
+        description: 'Tag Name'     
+        required: true
+        default: 'latest'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: build for amd64/arm64
+    steps:
+    - uses: actions/checkout@v2
+      with:
+          submodules: recursive
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    # Setup hardware emulator using QEMU
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+
+    # Setup Docker Buildx for multi-arch images
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Build Check
+      if: |
+          github.repository != 'loxilb-io/loxilb'
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        platforms: linux/amd64, linux/arm64
+        push: false
+        tags: ghcr.io/loxilb-io/loxilb:${{ github.event.inputs.tagName }} 
+
+    - name: Build and push
+      if: |
+          github.repository == 'loxilb-io/loxilb'
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        platforms: linux/amd64, linux/arm64
+        push: true
+        tags: ghcr.io/loxilb-io/loxilb:${{ github.event.inputs.tagName }} 
diff --git a/cicd/common.sh b/cicd/common.sh
@@ -13,6 +13,7 @@ hns="sudo ip netns "
 hexist="$vrn$hn"
 lxdocker="ghcr.io/loxilb-io/loxilb:latest"
 cluster_opts=""
+ka_opts=""
 var=$(lsb_release -r | cut -f2)
 if [[ $var == *"22.04"* ]];then
   lxdocker="ghcr.io/loxilb-io/loxilb:latestu22"
@@ -104,24 +105,9 @@ spawn_docker_host() {
     fi
     if [[ ! -z ${ka+x} ]]; then
       sudo mkdir -p /etc/shared/$dname/
-      if [[ "$ka" == "in" ]];then
-        ka_opts="-k in"
-        if [[ ! -z "$kpath" ]]; then
-            ka_conf="-v $kpath:/etc/keepalived/" 
-        fi
-      fi
-      docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config $ka_conf --name $dname $lxdocker
+      docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config --name $dname $lxdocker
       get_llb_peerIP $dname
       docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $ka_opts
-
-      if [[ "$ka" == "out" ]];then
-        ka_opts="-k out"
-        if [[ ! -z "$kpath" ]]; then
-            ka_conf="-v $kpath:/container/service/keepalived/assets/" 
-        fi
-
-        docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit --network=container:$dname $ka_conf -v /etc/shared/$dname:/etc/shared --name ka_$dname osixia/keepalived:2.0.20
-      fi
     else
       docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log $loxilb_config --name $dname $lxdocker $bgp_opts
       docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts
@@ -170,6 +156,7 @@ get_llb_peerIP() {
         llb2IP="$A.$B.$C.$((D+1))"
       fi
       cluster_opts=" --cluster=$llb2IP --self=0"
+      ka_opts=" --ka=$llb2IP"
     elif [[ "$1" == "llb2" ]]; then
       llb2IP=$(docker inspect --format='{{.NetworkSettings.IPAddress}}' llb2)
       if [[ "lb$llb2IP" == "lb" ]];then
@@ -179,6 +166,7 @@ get_llb_peerIP() {
         llb1IP="$A.$B.$C.$((D-1))"
       fi
       cluster_opts=" --cluster=$llb1IP --self=1"
+      ka_opts=" --ka=$llb1IP"
     fi
 }
 

diff --git a/cicd/k3s-calico/config.sh b/cicd/k3s-calico/config.sh
@@ -159,10 +159,6 @@ sleep 10
 # Wait for cluster to be ready
 wait_cluster_ready_full
 
-# Start nginx pods and services for test
-kubectl $KUBECONFIG apply -f nginx.yml
-kubectl $KUBECONFIG apply -f nginx-svc-lb.yml
-
 sleep 5 
 
 # Start nginx pods and services for test(using kube-loxilb)
@@ -179,7 +175,7 @@ sleep 30
 kubectl $KUBECONFIG get svc
 
 # Route back to user
-sudo ip route add 1.1.1.1/32 via 12.12.12.1
+sudo ip route add 1.1.1.1/32 via 14.14.14.1
 
 # Wait for cluster to be ready
 wait_cluster_ready_full
diff --git a/cicd/k3s-calico/nginx-svc-lb1.yml b/cicd/k3s-calico/nginx-svc-lb1.yml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: Service
 metadata:
   name: nginx-lb1
+  annotations:
+    loxilb.io/staticIP : "11.11.11.11"
 spec:
   externalTrafficPolicy: Local
   loadBalancerClass: loxilb.io/loxilb

diff --git a/cicd/k3s-calico/validation.sh b/cicd/k3s-calico/validation.sh
@@ -11,7 +11,7 @@ IFS=' '
 
 for((i=0; i<120; i++))
 do
-  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb")
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb1")
   read -a strarr <<< "$extLB"
   len=${#strarr[*]}
   if [[ $((len)) -lt 6 ]]; then
@@ -60,14 +60,6 @@ print_debug_info() {
 code=0
 print_debug_info
 
-out=$($hexec user curl -s --connect-timeout 10 http://$extIP:80) 
-if [[ ${out} == *"Welcome to nginx"* ]]; then
-  echo "calico-k3s (ccm) [OK]"
-else
-  echo "calico-k3s (ccm) [FAILED]"
-  code=1
-fi
-
 out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) 
 
 if [[ ${out} == *"Welcome to nginx"* ]]; then
@@ -77,6 +69,24 @@ else
   code=1
 fi
 
+for((i=0; i<120; i++))
+do
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "udp-lb1")
+  read -a strarr <<< "$extLB"
+  len=${#strarr[*]}
+  if [[ $((len)) -lt 6 ]]; then
+    echo "Can't find nginx-lb service"
+    sleep 1
+    continue
+  fi 
+  if [[ ${strarr[3]} != *"none"* ]]; then
+    extIP="$(cut -d'-' -f2 <<<${strarr[3]})"
+    break
+  fi
+  echo "No external LB allocated"
+  sleep 1
+done
+
 out=$($hexec user timeout 30 ../common/udp_client $extIP 55003)
 if [[ ${out} == *"Client"* ]]; then
   echo "calico-k3s (kube-loxillb) udp [OK]"
@@ -85,6 +95,24 @@ else
   code=1
 fi
 
+for((i=0; i<120; i++))
+do
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "sctp-lb1")
+  read -a strarr <<< "$extLB"
+  len=${#strarr[*]}
+  if [[ $((len)) -lt 6 ]]; then
+    echo "Can't find nginx-lb service"
+    sleep 1
+    continue
+  fi
+  if [[ ${strarr[3]} != *"none"* ]]; then
+    extIP="$(cut -d'-' -f2 <<<${strarr[3]})"
+    break
+  fi
+  echo "No external LB allocated"
+  sleep 1
+done
+
 out=$($hexec user timeout 30 ../common/sctp_socat_client 1.1.1.1 41291 $extIP 55004)
 if [[ ${out} == *"server1"* ]]; then
   echo "calico-k3s (kube-loxillb) sctp [OK]"

diff --git a/cicd/k3s-flannel-incluster-l2/host.sh b/cicd/k3s-flannel-incluster-l2/host.sh
@@ -1,3 +1,3 @@
-sudo apt-get install lksctp-tools
+sudo apt-get install -y lksctp-tools
 sudo ip route add 123.123.123.0/24 via 192.168.90.10
 echo "Host is up"
diff --git a/cicd/k3s-flannel/config.sh b/cicd/k3s-flannel/config.sh
@@ -7,8 +7,8 @@ echo "#########################################"
 echo "Spawning all hosts"
 echo "#########################################"
 
-spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --bgp-config $(pwd)/llb1_gobgp_config --with-ka in --ka-config $(pwd)/keepalived_config1
-spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --bgp-config $(pwd)/llb2_gobgp_config --with-ka in --ka-config $(pwd)/keepalived_config2
+spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --bgp-config $(pwd)/llb1_gobgp_config --with-ka in
+spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --bgp-config $(pwd)/llb2_gobgp_config --with-ka in
 spawn_docker_host --dock-type host --dock-name ep1
 spawn_docker_host --dock-type host --dock-name ep2
 spawn_docker_host --dock-type host --dock-name ep3
@@ -98,6 +98,10 @@ $hexec r1 ip route add 20.20.20.1/32 via 11.11.11.11
 #add_route llb1 1.1.1.0/24 11.11.11.254
 #add_route llb2 1.1.1.0/24 11.11.11.254
 
+## host network
+sudo ip route add 11.11.11.11/32 via 14.14.14.1
+sudo ip route add 123.123.123.1/32 via 14.14.14.1
+
 sleep 1
 ##Create LB rule
 create_lb_rule llb1 20.20.20.1 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1 --mode=fullnat --bgp
@@ -157,8 +161,8 @@ wait_cluster_ready_full
 
 sleep 10
 # Start nginx pods and services for test
-kubectl $KUBECONFIG apply -f nginx.yml
-kubectl $KUBECONFIG apply -f nginx-svc-lb.yml
+#kubectl $KUBECONFIG apply -f nginx.yml
+#kubectl $KUBECONFIG apply -f nginx-svc-lb.yml
 
 sleep 5 
 

diff --git a/cicd/k3s-flannel/nginx-svc-lb1.yml b/cicd/k3s-flannel/nginx-svc-lb1.yml
@@ -2,6 +2,9 @@ apiVersion: v1
 kind: Service
 metadata:
   name: nginx-lb1
+  annotations:
+    loxilb.io/staticIP : "11.11.11.11"
+    loxilb.io/lbmode: "fullnat"
 spec:
   externalTrafficPolicy: Local
   loadBalancerClass: loxilb.io/loxilb

diff --git a/cicd/k3s-flannel/rmconfig.sh b/cicd/k3s-flannel/rmconfig.sh
@@ -10,6 +10,9 @@ fi
 
 source ../common.sh
 
+sudo ip route del 11.11.11.11/32 via 14.14.14.1
+sudo ip route del 123.123.123.1/32 via 14.14.14.1
+
 sudo kubectl $KUBECONFIG delete -f nginx-svc-lb1.yml >> /dev/null 2>&1
 sudo kubectl $KUBECONFIG delete -f nginx-svc-lb.yml >> /dev/null 2>&1
 sudo kubectl $KUBECONFIG delete -f nginx.yml >> /dev/null 2>&1

diff --git a/cicd/k3s-flannel/validation.sh b/cicd/k3s-flannel/validation.sh
@@ -11,7 +11,7 @@ IFS=' '
 
 for((i=0; i<120; i++))
 do
-  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb")
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb1")
   read -a strarr <<< "$extLB"
   len=${#strarr[*]}
   if [[ $((len)) -lt 6 ]]; then
@@ -58,15 +58,7 @@ print_debug_info() {
 }
 
 code=0
-print_debug_info
-
-out=$($hexec user curl -s --connect-timeout 10 http://$extIP:80) 
-if [[ ${out} == *"Welcome to nginx"* ]]; then
-  echo "cluster-k3s (ccm) [OK]"
-else
-  echo "cluster-k3s (ccm) [FAILED]"
-  code=1
-fi
+#print_debug_info
 
 out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) 
 if [[ ${out} == *"Welcome to nginx"* ]]; then
@@ -76,6 +68,24 @@ else
   code=1
 fi
 
+for((i=0; i<120; i++))
+do
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "udp-lb1")
+  read -a strarr <<< "$extLB"
+  len=${#strarr[*]}
+  if [[ $((len)) -lt 6 ]]; then
+    echo "Can't find udp-lb service"
+    sleep 1
+    continue
+  fi
+  if [[ ${strarr[3]} != *"none"* ]]; then
+    extIP="$(cut -d'-' -f2 <<<${strarr[3]})"
+    break
+  fi
+  echo "No external LB allocated"
+  sleep 1
+done
+
 out=$($hexec user timeout 30 ../common/udp_client $extIP 55003)
 if [[ ${out} == *"Client"* ]]; then
   echo "cluster-k3s (kube-loxilb) udp [OK]"
@@ -85,6 +95,7 @@ else
 fi
 
 if [[ $code -eq 1 ]]; then
+  print_debug_info
   echo "cluster-k3s failed"
   exit 1
 fi

diff --git a/cicd/k3s-sctpmh/config.sh b/cicd/k3s-sctpmh/config.sh
@@ -7,8 +7,8 @@ echo "#########################################"
 echo "Spawning all hosts"
 echo "#########################################"
 
-spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --bgp-config $(pwd)/llb1_gobgp_config --with-ka in --ka-config $(pwd)/keepalived_config1
-spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --bgp-config $(pwd)/llb2_gobgp_config --with-ka in --ka-config $(pwd)/keepalived_config2
+spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --bgp-config $(pwd)/llb1_gobgp_config --with-ka in
+spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --bgp-config $(pwd)/llb2_gobgp_config --with-ka in
 spawn_docker_host --dock-type host --dock-name ep1
 spawn_docker_host --dock-type host --dock-name ep2
 spawn_docker_host --dock-type host --dock-name ep3
@@ -99,7 +99,8 @@ $hexec r1 ip route add 20.20.20.1/32 via 11.11.11.11
 #add_route llb2 1.1.1.0/24 11.11.11.254
 
 # Route back to user
-sudo ip route add 11.11.11.0/24 via 12.12.12.1
+sudo ip route add 11.11.11.0/24 via 14.14.14.1
+sudo ip route add 123.123.123.0/24 via 14.14.14.1
 
 # Change default route in llb1
 $hexec llb1 ip route del default 
@@ -195,11 +196,9 @@ $dexec llb1 loxicmd get lb -o wide
 echo "llb1: loxicmd get ep -o wide"
 echo "****************************"
 $dexec llb1 loxicmd get ep -o wide
-$dexec llb1 cat /etc/shared/keepalive.state
 echo "llb2: loxicmd get lb -o wide"
 echo "****************************"
 $dexec llb2 loxicmd get lb -o wide
 echo "llb2: loxicmd get ep -o wide"
 echo "****************************"
 $dexec llb2 loxicmd get ep -o wide
-$dexec llb2 cat /etc/shared/keepalive.state
diff --git a/cicd/k3s-sctpmh/nginx-svc-lb1.yml b/cicd/k3s-sctpmh/nginx-svc-lb1.yml
@@ -2,6 +2,9 @@ apiVersion: v1
 kind: Service
 metadata:
   name: nginx-lb1
+  annotations:
+    loxilb.io/lbmode: "fullnat"
+    loxilb.io/staticIP : "11.11.11.11"
 spec:
   externalTrafficPolicy: Local
   loadBalancerClass: loxilb.io/loxilb

diff --git a/cicd/k3s-sctpmh/nginx.yml b/cicd/k3s-sctpmh/nginx.yml
@@ -2,6 +2,9 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: nginx
+  annotations:
+    loxilb.io/lbmode: "fullnat"
+    loxilb.io/staticIP : "11.11.11.11"
   labels:
     app.kubernetes.io/name: proxy
 spec: