From eb87af1d4d56a563dcad21a3d886f7d6a6a03219 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Fri, 8 Mar 2024 11:21:55 +0900 Subject: [PATCH 01/12] chore: gofmt and unwanted code cleanup --- api/restapi/handler/cluster.go | 2 +- loxinet/cluster.go | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/api/restapi/handler/cluster.go b/api/restapi/handler/cluster.go index b431d701c..ab8ef3a1d 100644 --- a/api/restapi/handler/cluster.go +++ b/api/restapi/handler/cluster.go @@ -130,4 +130,4 @@ func ConfigDeleteBFDSession(params operations.DeleteConfigBfdRemoteIPRemoteIPPar return &ResultResponse{Result: err.Error()} } return &ResultResponse{Result: "Success"} -} \ No newline at end of file +} diff --git a/loxinet/cluster.go b/loxinet/cluster.go index f6de07b3f..caad3d312 100644 --- a/loxinet/cluster.go +++ b/loxinet/cluster.go @@ -226,9 +226,7 @@ func (h *CIStateH) CIStateUpdate(cm cmn.HASMod) (int, error) { ci.StateStr = cm.State ci.State = h.StateMap[cm.State] ci.Vip = cm.Vip - if h.SpawnKa && (cm.State == "FAULT" || cm.State == "STOP") { - RunCommand("pkill keepalived", false) - } + if mh.bgp != nil { mh.bgp.UpdateCIState(cm.Instance, ci.State, ci.Vip) } @@ -291,21 +289,21 @@ func (h *CIStateH) CIBFDSessionAdd(bm cmn.BFDMod) (int, error) { if !h.SpawnKa { tk.LogIt(tk.LogInfo, "[CLUSTER] Cluster Instance %s starting BFD..\n", bm.Instance) h.SpawnKa = true - - h.RemoteIP = bm.RemoteIP - h.SourceIP = bm.SourceIP - h.Interval = int64(bm.Interval) + + h.RemoteIP = bm.RemoteIP + h.SourceIP = bm.SourceIP + h.Interval = int64(bm.Interval) bfdSessConfigArgs := bfd.ConfigArgs{RemoteIP: bm.RemoteIP.String(), SourceIP: bm.SourceIP.String(), - Port: cmn.BFDPort, Interval: uint32(bm.Interval), + Port: cmn.BFDPort, Interval: uint32(bm.Interval), Multi: bm.RetryCount, Instance: bm.Instance} go h.startBFDProto(bfdSessConfigArgs) } else { - bfdSessConfigArgs := bfd.ConfigArgs{RemoteIP: h.RemoteIP.String(), SourceIP: h.SourceIP.String(), - Port: cmn.BFDPort, Interval: uint32(bm.Interval), + bfdSessConfigArgs := bfd.ConfigArgs{RemoteIP: h.RemoteIP.String(), SourceIP: h.SourceIP.String(), + Port: cmn.BFDPort, Interval: uint32(bm.Interval), Multi: bm.RetryCount, Instance: bm.Instance} err := h.Bs.BFDAddRemote(bfdSessConfigArgs, h) if err != nil { - tk.LogIt(tk.LogCritical, "KA - Cant add BFD remote: %s\n",err.Error()) + tk.LogIt(tk.LogCritical, "KA - Cant add BFD remote: %s\n", err.Error()) return -1, err } tk.LogIt(tk.LogInfo, "KA - BFD remote %s:%s:%vus Added\n", h.RemoteIP.String(), h.SourceIP.String(), bm.Interval) From 4f60d36f0aec950e64750f813d0e1ed36a2e3dbc Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Fri, 8 Mar 2024 12:02:04 +0900 Subject: [PATCH 02/12] Tweaks to one-arm source selection --- loxinet/rules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loxinet/rules.go b/loxinet/rules.go index d88345fbe..a034239ef 100644 --- a/loxinet/rules.go +++ b/loxinet/rules.go @@ -978,7 +978,7 @@ func (R *RuleH) electEPSrc(r *ruleEnt) bool { sip = np.rIP if na.mode == cmn.LBModeOneArm { mode = "onearm" - e, sip, _ = R.zone.L3.IfaSelectAny(np.xIP, false) + e, sip, _ = R.zone.L3.IfaSelectAny(np.xIP, true) if e != 0 { tk.LogIt(tk.LogDebug, "Failed to find suitable source for %s\n", np.xIP.String()) addrRslv = true From b716725882ba83c826349afdac2eb3ca9f201535 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Fri, 8 Mar 2024 13:57:03 +0900 Subject: [PATCH 03/12] cicd: single node k3s loxilb docker with cilium --- cicd/docker-k3s-cilium/Vagrantfile | 29 ++ cicd/docker-k3s-cilium/common.sh | 555 +++++++++++++++++++++++++ cicd/docker-k3s-cilium/config.sh | 3 + cicd/docker-k3s-cilium/k3s_common.sh | 36 ++ cicd/docker-k3s-cilium/kube-loxilb.yml | 129 ++++++ cicd/docker-k3s-cilium/loxilb.sh | 76 ++++ cicd/docker-k3s-cilium/rmconfig.sh | 2 + cicd/docker-k3s-cilium/tcp-svc-lb.yml | 30 ++ cicd/docker-k3s-cilium/validation.sh | 49 +++ 9 files changed, 909 insertions(+) create mode 100644 cicd/docker-k3s-cilium/Vagrantfile create mode 100644 cicd/docker-k3s-cilium/common.sh create mode 100755 cicd/docker-k3s-cilium/config.sh create mode 100644 cicd/docker-k3s-cilium/k3s_common.sh create mode 100644 cicd/docker-k3s-cilium/kube-loxilb.yml create mode 100644 cicd/docker-k3s-cilium/loxilb.sh create mode 100755 cicd/docker-k3s-cilium/rmconfig.sh create mode 100644 cicd/docker-k3s-cilium/tcp-svc-lb.yml create mode 100755 cicd/docker-k3s-cilium/validation.sh diff --git a/cicd/docker-k3s-cilium/Vagrantfile b/cicd/docker-k3s-cilium/Vagrantfile new file mode 100644 index 000000000..c992d21d0 --- /dev/null +++ b/cicd/docker-k3s-cilium/Vagrantfile @@ -0,0 +1,29 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +loxilbs = (ENV['LOXILBS'] || "1").to_i +eps = (ENV['LOXILBS'] || "1").to_i +box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s") +box_version = "0.7.1" +Vagrant.configure("2") do |config| + config.vm.box = "#{box_name}" + config.vm.box_version = "#{box_version}" + + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + + (1..loxilbs).each do |node_number| + config.vm.define "llb#{node_number}" do |loxilb| + loxilb.vm.hostname = "llb#{node_number}" + ip = node_number + 245 + loxilb.vm.network :private_network, ip: "192.168.163.#{ip}", :netmask => "255.255.255.0" + loxilb.vm.provision :shell, :path => "loxilb.sh" + loxilb.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 16000] + vbox.customize ["modifyvm", :id, "--cpus", 8] + vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + end + end + end +end diff --git a/cicd/docker-k3s-cilium/common.sh b/cicd/docker-k3s-cilium/common.sh new file mode 100644 index 000000000..b82f6e0e4 --- /dev/null +++ b/cicd/docker-k3s-cilium/common.sh @@ -0,0 +1,555 @@ +#!/bin/bash + +if [[ "$1" == "init" ]]; then + pull_dockers +fi + +hn="netns" +pid="" +vrn="/var/run/" +hexec="sudo ip netns exec " +dexec="sudo docker exec -i " +hns="sudo ip netns " +hexist="$vrn$hn" +lxdocker="ghcr.io/loxilb-io/loxilb:latest" +var=$(lsb_release -r | cut -f2) +if [[ $var == *"22.04"* ]];then + lxdocker="ghcr.io/loxilb-io/loxilb:latestU22" +fi + +loxilbs=() + +## Given a docker name(arg1), return its pid +get_docker_pid() { + id=`docker ps -f name=$1| grep -w $1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + pid=`docker inspect -f '{{.State.Pid}}' $id` +} + +## Pull all necessary dockers for testbed +pull_dockers() { + ## loxilb docker + docker pull $lxdocker + ## Host docker + docker pull eyes852/ubuntu-iperf-test:0.5 + ## BGP host docker + docker pull ewindisch/quagga + ## Keepalive docker + docker pull osixia/keepalived:2.0.20 +} + +## Creates a docker host +## arg1 - "loxilb"|"host" +## arg2 - instance-name +spawn_docker_host() { + POSITIONAL_ARGS=() + local bpath + local kpath + local ka + local bgp + while [[ $# -gt 0 ]]; do + case "$1" in + -t | --dock-type ) + dtype="$2" + shift 2 + ;; + -d | --dock-name ) + dname="$2" + shift 2 + ;; + -b | --with-bgp ) + if [[ "$2" == "yes" ]]; then + bgp=$2 + fi + shift 2 + ;; + -c | --bgp-config ) + bpath="$2" + bgp="yes" + shift 2 + ;; + -k | --with-ka ) + ka="in" + if [[ "$2" == "out" ]]; then + ka=$2 + fi + shift 2 + ;; + -d | --ka-config ) + kpath="$2" + if [[ -z ${ka+x} ]]; then + ka="in" + fi + shift 2 + ;; + -*|--*) + echo "Unknown option $1" + exit + ;; + esac + done + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + echo "Spawning $dname($dtype)" >&2 + if [[ "$dtype" == "loxilb" ]]; then + loxilbs+=("$dname") + if [[ "$pick_config" == "yes" ]]; then + echo "$dname will pick config from $(pwd)/${dname}_config" + loxilb_config="-v $(pwd)/${dname}_config:/etc/loxilb/" + fi + if [[ "$bgp" == "yes" ]]; then + bgp_opts="-b" + if [[ ! -z "$bpath" ]]; then + bgp_conf="-v $bpath:/etc/gobgp/" + fi + fi + if [[ "$dname" == "llb1" ]]; then + cluster_opts=" --cluster=172.17.0.3 --self=0" + elif [[ "$dname" == "llb2" ]]; then + cluster_opts=" --cluster=172.17.0.2 --self=1" + fi + + if [[ ! -z ${ka+x} ]]; then + sudo mkdir -p /etc/shared/$dname/ + if [[ "$ka" == "in" ]];then + ka_opts="-k in" + if [[ ! -z "$kpath" ]]; then + ka_conf="-v $kpath:/etc/keepalived/" + fi + fi + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config $ka_conf --name $dname $lxdocker + docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $ka_opts + + if [[ "$ka" == "out" ]];then + ka_opts="-k out" + if [[ ! -z "$kpath" ]]; then + ka_conf="-v $kpath:/container/service/keepalived/assets/" + fi + + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit --network=container:$dname $ka_conf -v /etc/shared/$dname:/etc/shared --name ka_$dname osixia/keepalived:2.0.20 + fi + else + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log $loxilb_config --name $dname $lxdocker $bgp_opts + docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts + fi + elif [[ "$dtype" == "host" ]]; then + if [[ ! -z "$bpath" ]]; then + bgp_conf="--volume $bpath:/etc/quagga" + fi + if [[ "$bgp" == "yes" || ! -z "$bpath" ]]; then + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit $bgp_conf --name $dname ewindisch/quagga + else + docker run -u root --cap-add SYS_ADMIN -dit --name $dname eyes852/ubuntu-iperf-test:0.5 + fi + fi + + pid="" + + sleep 2 + get_docker_pid $dname + echo $pid + if [ ! -f "$hexist/$dname" -a "$pid" != "" ]; then + sudo mkdir -p /var/run/netns + sudo touch /var/run/netns/$dname + #echo "sudo mount -o bind /proc/$pid/ns/net /var/run/netns/$2" + sudo mount -o bind /proc/$pid/ns/net /var/run/netns/$dname + fi + + $hexec $dname ifconfig lo up + $hexec $dname sysctl net.ipv6.conf.all.disable_ipv6=1 2>&1 >> /dev/null + #$hexec $dname sysctl net.ipv4.conf.all.arp_accept=1 2>&1 >> /dev/null + $hexec $dname sysctl net.ipv4.conf.eth0.arp_ignore=2 2>&1 >> /dev/null +} + +## Deletes a docker host +## arg1 - hostname +delete_docker_host() { + id=`docker ps -f name=$1| grep -w $1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + if [ "$id" != "" ]; then + docker stop $1 2>&1 >> /dev/null + hd="true" + ka=`docker ps -f name=ka_$1| grep -w ka_$1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + loxilbs=( "${loxilbs[@]/$1}" ) + if [ "$ka" != "" ]; then + docker stop ka_$1 2>&1 >> /dev/null + docker rm ka_$1 2>&1 >> /dev/null + fi + fi + if [ -f "$hexist/$1" ]; then + $hns del $1 + sudo rm -fr "$hexist/$1" 2>&1 >> /dev/null + fi + if [ "$id" != "" ]; then + docker rm $1 2>&1 >> /dev/null + fi +} + +## Connects two docker hosts +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - mtu +connect_docker_hosts() { + link1=e$1$2 + link2=e$2$1 + + mtu="9000" + if [[ $# -gt 2 ]]; then + mtu=$3 + fi + + #echo $link1 $link2 + sudo ip -n $1 link add $link1 type veth peer name $link2 netns $2 + sudo ip -n $1 link set $link1 mtu $mtu up + sudo ip -n $2 link set $link2 mtu $mtu up +} + +## arg1 - hostname1 +## arg2 - hostname2 +disconnect_docker_hosts() { + link1=e$1$2 + link2=e$2$1 + # echo $link1 $link2 + if [ -f "$hexist/$1" ]; then + ifexist1=`sudo ip -n $1 link show $link1 | grep -w $link1` + if [ "chk$ifexist1" != "chk" ]; then + sudo ip -n $1 link set $link1 down 2>&1 >> /dev/null + sudo ip -n $1 link del $link1 2>&1 >> /dev/null + fi + fi + + if [ -f "$hexist/$2" ]; then + ifexist2=`sudo ip -n $2 link show | grep -w $link2` + if [ "chk$ifexist2" != "chk" ]; then + sudo ip -n $2 link set $link2 down 2>&1 >> /dev/null + sudo ip -n $2 link del $link2 2>&1 >> /dev/null + fi + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - ip_addr +## arg4 - gw +config_docker_host() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --ptype) + local ptype="$2" + shift + shift + ;; + --id) + local xid="$2" + shift + shift + ;; + --addr) + local addr="$2" + shift + shift + ;; + --gw) + local gw="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + + link1=e$h1$h2 + link2=e$h2$h1 + #echo "$h1:$link1->$h2:$link2" + + #if [[ -n "${loxilbs[$h1]}" && "$pick_config" == "yes" ]]; then + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + if [[ "$ptype" == "phy" ]]; then + sudo ip -n $h1 addr add $addr dev $link1 + elif [[ "$ptype" == "vlan" ]]; then + sudo ip -n $h1 addr add $addr dev vlan$xid + elif [[ "$ptype" == "vxlan" ]]; then + sudo ip -n $h1 addr add $addr dev vxlan$xid + elif [[ "$ptype" == "trunk" ]]; then + trunk="bond$xid" + sudo ip -n $h1 link set $link1 down + sudo ip -n $h1 link add $trunk type bond + sudo ip -n $h1 link set $link1 master $trunk + sudo ip -n $h1 link set $link1 up + sudo ip -n $h1 link set $trunk up + + sudo ip -n $h2 link set $link2 down + sudo ip -n $h2 link add $trunk type bond + sudo ip -n $h2 link set $link2 master $trunk + sudo ip -n $h2 link set $link2 up + sudo ip -n $h2 link set $trunk up + + sudo ip -n $h1 addr add $addr dev bond$xid + if [[ "$gw" != "" ]]; then + sudo ip -n $h2 addr add $gw/24 dev bond$xid + sudo ip -n $h1 route add default via $gw proto static + fi + else + echo "Check port-type" + fi + + if [[ "$gw" != "" ]]; then + sudo ip -n $h1 route del default 2>&1 >> /dev/null + sudo ip -n $h1 route add default via $gw + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - vlan +## arg4 - tagged/untagged +create_docker_host_vlan() { + local addr="" + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --ptype) + local ptype="$2" + shift + shift + ;; + --id) + local vid="$2" + shift + shift + ;; + --addr) + addr="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + if [[ "$ptype" == "tagged" ]]; then + brport="$link1.$vid" + sudo ip -n $h1 link add link $link1 name $brport type vlan id $vid + sudo ip -n $h1 link set $brport up + else + brport=$link1 + fi + + sudo ip -n $h1 link add vlan$vid type bridge 2>&1 | true + sudo ip -n $h1 link set $brport master vlan$vid + sudo ip -n $h1 link set vlan$vid up + if [[ "$addr" != "" ]]; then + sudo ip -n $h1 addr add $addr dev vlan$vid + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - vxlan-id +## arg4 - phy/vlan +## arg5 - local ip if arg4 is phy/vlan-id if arg4 is vlan +## arg6 - local ip if arg4 is vlan +create_docker_host_vxlan() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --uif) + local uifType="$2" + shift + shift + ;; + --vid) + local vid="$2" + shift + shift + ;; + --pvid) + local pvid="$2" + shift + shift + ;; + --id) + local vxid="$2" + shift + shift + ;; + --ep) + local ep="$2" + shift + shift + ;; + --lip) + local lip="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + if [[ "$uifType" == "phy" ]]; then + sudo ip -n $h1 link add vxlan$vxid type vxlan id $vxid local $lip dev $link1 dstport 4789 + sudo ip -n $h1 link set vxlan$vxid up + elif [[ "$uifType" == "vlan" ]]; then + sudo ip -n $h1 link add vxlan$vxid type vxlan id $vxid local $lip dev vlan$vid dstport 4789 + sudo ip -n $h1 link set vxlan$vxid up + fi + + if [[ "$pvid" != "" ]]; then + sudo ip -n $h1 link add vlan$pvid type bridge 2>&1 | true + sudo ip -n $h1 link set vxlan$vxid master vlan$pvid + sudo ip -n $h1 link set vlan$pvid up + fi + + if [[ "$ep" != "" ]]; then + sudo bridge -n $h1 fdb append 00:00:00:00:00:00 dst $ep dev vxlan$vxid + fi + +} + +## arg1 - hostname1 +## arg2 - hostname2 +create_docker_host_cnbridge() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + brport=$link1 + + sudo ip -n $h1 link add br$h1 type bridge 2>&1 | true + sudo ip -n $h1 link set $brport master br$h1 + sudo ip -n $h1 link set br$h1 up +} + +#Arg1: host name +#Arg2: --:: +#Arg3: --endpoints::,.. +function create_lb_rule() { + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + args=( "$@" ) + args=( "${args[@]/$1}" ) + echo "$1: loxicmd create lb ${args[*]}" + $dexec $1 loxicmd create lb ${args[*]} + + hook=$($dexec llb1 ntc filter show dev eth0 ingress | grep tc_packet_hook) + if [[ $hook != *"tc_packet_hook"* ]]; then + echo "ERROR : No hook point found"; + exit 1 + fi +} + +#Arg1: host name +#Arg2: +#Arg3: +function add_route() { + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + echo "$1: ip route add $2 via $3 proto static" + $hexec $1 ip route add $2 via $3 proto static +} diff --git a/cicd/docker-k3s-cilium/config.sh b/cicd/docker-k3s-cilium/config.sh new file mode 100755 index 000000000..6b8ee48ef --- /dev/null +++ b/cicd/docker-k3s-cilium/config.sh @@ -0,0 +1,3 @@ +#!/bin/bash +vagrant global-status | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f +vagrant up diff --git a/cicd/docker-k3s-cilium/k3s_common.sh b/cicd/docker-k3s-cilium/k3s_common.sh new file mode 100644 index 000000000..5a0dfe998 --- /dev/null +++ b/cicd/docker-k3s-cilium/k3s_common.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +function wait_cluster_ready { + Res=$(sudo kubectl $KUBECONFIG get pods -A | + while IFS= read -r line; do + if [[ "$line" != *"Running"* && "$line" != *"READY"* ]]; then + echo "not ready" + return + fi + done) + if [[ $Res == *"not ready"* ]]; then + return 1 + fi + return 0 +} + +function wait_cluster_ready_full { + i=1 + nr=0 + for ((;;)) do + wait_cluster_ready + nr=$? + if [[ $nr == 0 ]]; then + echo "Cluster is ready" + break + fi + i=$(( $i + 1 )) + if [[ $i -ge 40 ]]; then + echo "Cluster is not ready.Giving up" + exit 1 + fi + echo "Cluster is not ready...." + sleep 10 + done +} + diff --git a/cicd/docker-k3s-cilium/kube-loxilb.yml b/cicd/docker-k3s-cilium/kube-loxilb.yml new file mode 100644 index 000000000..05ea21ab2 --- /dev/null +++ b/cicd/docker-k3s-cilium/kube-loxilb.yml @@ -0,0 +1,129 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-loxilb + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - endpoints + - services + - services/status + verbs: + - get + - watch + - list + - patch + - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - watch + - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-loxilb +subjects: + - kind: ServiceAccount + name: kube-loxilb + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-loxilb + namespace: kube-system + labels: + app: loxilb +spec: + replicas: 1 + selector: + matchLabels: + app: loxilb + template: + metadata: + labels: + app: loxilb + spec: + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + terminationGracePeriodSeconds: 0 + containers: + - name: kube-loxilb + image: ghcr.io/loxilb-io/kube-loxilb:latest + imagePullPolicy: Always + command: + - /bin/kube-loxilb + args: + - --loxiURL=http://172.17.0.2:11111 + - --externalCIDR=192.168.163.247/32 + #- --monitor + #- --setBGP + #- --setLBMode=1 + #- --config=/opt/loxilb/agent/kube-loxilb.conf + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + capabilities: + add: ["NET_ADMIN", "NET_RAW"] diff --git a/cicd/docker-k3s-cilium/loxilb.sh b/cicd/docker-k3s-cilium/loxilb.sh new file mode 100644 index 000000000..8c4fc5e50 --- /dev/null +++ b/cicd/docker-k3s-cilium/loxilb.sh @@ -0,0 +1,76 @@ +source /vagrant/common.sh +source /vagrant/k3s_common.sh + +export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/') + +## Set promisc mode for mac-vlan to work +sudo ifconfig eth1 promisc + +apt-get update +apt-get install -y software-properties-common ethtool +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get install -y docker-ce + +echo "Start loxilb installation" +sudo docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged --entrypoint /root/loxilb-io/loxilb/loxilb -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest + +#docker exec -i loxilb apt-get update +#docker exec -i loxilb apt-get -y install clang-10 llvm libelf-dev gcc-multilib libpcap-dev linux-tools-$(uname -r) elfutils dwarves git libbsd-dev bridge-utils unzip build-essential bison flex iperf iproute2 nodejs socat ethtool + +# Create mac-vlan on top of underlying eth1 interface +docker network create -d macvlan -o parent=eth1 --subnet 192.168.163.0/24 --gateway 192.168.163.1 --aux-address 'host=192.168.163.252' llbnet + +# Assign mac-vlan to loxilb docker with specified IP (which will be used as LB VIP) +docker network connect llbnet loxilb --ip=192.168.163.247 + +# Start a docker to simulate e2 sctp endpoint +docker run -u root --cap-add SYS_ADMIN -dit --privileged --name e2 eyes852/ubuntu-iperf-test:0.5 +docker exec -i e2 apt-get update +docker exec -i e2 apt-get -y install lksctp-tools + +# Add iptables rule to allow traffic from source IP(192.168.163.1) to loxilb +sudo iptables -A DOCKER -s 192.168.163.1 -j ACCEPT + +# Start application to simulate sctp end-point +#docker exec -i e2 nohup sctp_darn -H 172.17.0.3 -P 5003 -l 2>&1 & + +echo "Start K3s installation" + +curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --disable "metrics-server" --kubelet-arg cloud-provider=external --flannel-backend=none --cluster-cidr=10.42.0.0/16" K3S_KUBECONFIG_MODE="644" sh - + +sleep 10 + +# Install Cilium +echo "Start cilium installation" +CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt) +CLI_ARCH=amd64 +if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi + curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} +sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum +sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin +rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + +mkdir ~/.kube/ +cat /etc/rancher/k3s/k3s.yaml > ~/.kube/config +cilium install + +sleep 60 + +# Check kubectl works +kubectl $KUBECONFIG get pods -A + +# Remove taints in k3s if any (usually happens if started without cloud-manager) +kubectl $KUBECONFIG taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule- + +echo "End K3s installation" +sleep 60 + +kubectl apply -f /vagrant/kube-loxilb.yml +sleep 60 +kubectl apply -f /vagrant/tcp-svc-lb.yml + +# Wait for cluster to be ready +wait_cluster_ready_full + +echo $LOXILB_IP > /vagrant/loxilb-$(hostname) diff --git a/cicd/docker-k3s-cilium/rmconfig.sh b/cicd/docker-k3s-cilium/rmconfig.sh new file mode 100755 index 000000000..e3e171229 --- /dev/null +++ b/cicd/docker-k3s-cilium/rmconfig.sh @@ -0,0 +1,2 @@ +#!/bin/bash +vagrant destroy -f llb1 diff --git a/cicd/docker-k3s-cilium/tcp-svc-lb.yml b/cicd/docker-k3s-cilium/tcp-svc-lb.yml new file mode 100644 index 000000000..ce5614866 --- /dev/null +++ b/cicd/docker-k3s-cilium/tcp-svc-lb.yml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + name: tcp-lb-onearm + annotations: + loxilb.io/liveness: "yes" + loxilb.io/lbmode: "onearm" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: tcp-onearm-test + ports: + - port: 56002 + targetPort: 80 + nodePort: 30001 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: tcp-onearm-test + labels: + what: tcp-onearm-test +spec: + containers: + - name: tcp-onearm-test + image: ghcr.io/loxilb-io/nginx:stable + ports: + - containerPort: 80 diff --git a/cicd/docker-k3s-cilium/validation.sh b/cicd/docker-k3s-cilium/validation.sh new file mode 100755 index 000000000..b358f971d --- /dev/null +++ b/cicd/docker-k3s-cilium/validation.sh @@ -0,0 +1,49 @@ +#!/bin/bash +source ../common.sh +echo docker-k3s-lb + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +sleep 30 +extIP="192.168.163.247" +echo $extIP + +echo "Service Info" +vagrant ssh llb1 -c 'sudo kubectl get svc' +echo "LB Info" +vagrant ssh llb1 -c 'sudo docker exec -i loxilb loxicmd get lb -o wide' +echo "EP Info" +vagrant ssh llb1 -c 'sudo docker exec -i loxilb loxicmd get ep -o wide' + +print_debug_info() { + echo "llb1 route-info" + vagrant ssh llb1 -c 'ip route' + vagrant ssh llb1 -c 'sudo kubectl get pods -A' + vagrant ssh llb1 -c 'sudo kubectl get svc' + vagrant ssh llb1 -c 'sudo kubectl get nodes' +} + +sctp_darn -H 192.168.163.1 -h 192.168.163.247 -p 55003 -s < input > output +sleep 5 +exp="New connection, peer addresses +192.168.163.247:55003" + +res=`cat output | grep -A 1 "New connection, peer addresses"` +echo "Result" +echo $res +echo "Expected" +echo $exp +sudo rm -rf output +if [[ "$res" == "$exp" ]]; then + echo $res + echo "docker-k3s-lb SCTP service sctp-lb (loxilb) [OK]" +else + echo "docker-k3s-lb SCTP service sctp-lb (loxilb) [NOK]" + print_debug_info + exit 1 +fi From fe365bf50385c0116e94d1ccbfac9a370926b5eb Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Sat, 9 Mar 2024 22:30:53 +0900 Subject: [PATCH 04/12] gh-516 - support for multi-arch docker --- Dockerfile | 7 ++++--- Makefile | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f39607136..373d5753e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,9 @@ LABEL description="loxilb official docker image" # Disable Prompt During Packages Installation ARG DEBIAN_FRONTEND=noninteractive -# Env for golang +# Env variables ENV PATH="${PATH}:/usr/local/go/bin" +ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib64/" # Install loxilb related packages RUN mkdir -p /opt/loxilb && \ @@ -43,8 +44,8 @@ RUN mkdir -p /opt/loxilb && \ /usr/local/sbin/loxicmd completion bash > /etc/bash_completion.d/loxi_completion && \ # Install loxilb git clone --recurse-submodules https://github.com/loxilb-io/loxilb /root/loxilb-io/loxilb/ && \ - cd /root/loxilb-io/loxilb/ && go get . && make && \ - cp loxilb-ebpf/utils/mkllb_bpffs.sh /usr/local/sbin/mkllb_bpffs && \ + cd /root/loxilb-io/loxilb/ && go get . && if [ "$arch" = "arm64" ] ; then DOCKER_BUILDX_ARM64=true make; \ + else make ;fi && cp loxilb-ebpf/utils/mkllb_bpffs.sh /usr/local/sbin/mkllb_bpffs && \ cp api/certification/* /opt/loxilb/cert/ && cd - && \ cp /root/loxilb-io/loxilb/loxilb-ebpf/kernel/loxilb_dp_debug /usr/local/sbin/loxilb_dp_debug && \ cp /root/loxilb-io/loxilb/loxilb /usr/local/sbin/loxilb && \ diff --git a/Makefile b/Makefile index 793592c25..60cc940db 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ docker: docker build -t ghcr.io/loxilb-io/loxilb:latest . docker-arm64: - docker buildx build --platform linux/arm64 -t ghcr.io/loxilb-io/loxilb:latest-arm64 . + docker buildx build --platform linux/arm64 --load -t ghcr.io/loxilb-io/loxilb:latest-arm64 . lint: golangci-lint run --enable-all From 7b9af2276c96eb81b6ebe83e9ce7e9011443377d Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Sun, 10 Mar 2024 13:06:05 +0900 Subject: [PATCH 05/12] gh-516 - gh-actions support for multi-arch docker --- .github/workflows/basic-sanity-ubuntu-22.yml | 2 +- .github/workflows/docker-image.yml | 17 +++++++++-------- .github/workflows/docker-multiarch.yml | 2 ++ .github/workflows/k3s-base-sanity.yml | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/basic-sanity-ubuntu-22.yml b/.github/workflows/basic-sanity-ubuntu-22.yml index a2eb30f43..9eb590587 100644 --- a/.github/workflows/basic-sanity-ubuntu-22.yml +++ b/.github/workflows/basic-sanity-ubuntu-22.yml @@ -52,7 +52,7 @@ jobs: - run: sudo -E env "PATH=$PATH" make - run: sudo -E env "PATH=$PATH" make test - run: docker pull ghcr.io/loxilb-io/loxilb:latest - - run: docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest + - run: docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest-amd64 - run: pwd && ls && sudo -E env "PATH=$PATH" make docker-cp-ebpf - run: docker exec -dit loxilb mkllb_bpffs - run: id=`docker ps -f name=loxilb | cut -d " " -f 1 | grep -iv "CONTAINER"` && docker commit $id ghcr.io/loxilb-io/loxilb:latest diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index c8563bdf9..14b0d11e9 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -8,7 +8,7 @@ on: tagName: description: 'Tag Name' required: true - default: 'latest' + default: 'latest-amd64' jobs: @@ -31,17 +31,18 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build the latest loxilb Docker image + - name: Build and run the latest loxilb Docker image if: github.event.inputs.tagName == '' - run: docker build . --tag ghcr.io/loxilb-io/loxilb:latest + run: | + docker build . --tag ghcr.io/loxilb-io/loxilb:latest + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest - - name: Build the loxilb Docker image with given tag + - name: Build and run the loxilb Docker image with given tag if: github.event.inputs.tagName != '' - run: docker build . --tag ghcr.io/loxilb-io/loxilb:${{ github.event.inputs.tagName }} + run: | + docker build . --tag ghcr.io/loxilb-io/loxilb:${{ github.event.inputs.tagName }} + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:${{ github.event.inputs.tagName }} - - name: Run the loxilb docker image to check - run: docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest - - name: Publish the latest loxilb Docker image if: | github.repository == 'loxilb-io/loxilb' diff --git a/.github/workflows/docker-multiarch.yml b/.github/workflows/docker-multiarch.yml index 073d03d70..9c30020dd 100644 --- a/.github/workflows/docker-multiarch.yml +++ b/.github/workflows/docker-multiarch.yml @@ -1,6 +1,8 @@ name: Docker-Multi-Arch on: + push: + branches: [ "main" ] workflow_dispatch: inputs: tagName: diff --git a/.github/workflows/k3s-base-sanity.yml b/.github/workflows/k3s-base-sanity.yml index a492243af..5070c9f68 100644 --- a/.github/workflows/k3s-base-sanity.yml +++ b/.github/workflows/k3s-base-sanity.yml @@ -10,7 +10,7 @@ on: required: true default: 'k3s-base' workflow_run: - workflows: ["Build-CI"] + workflows: ["Docker-Multi-Arch"] types: - completed jobs: From 645688c6f8bbd470ba609540f876cdc41b31b036 Mon Sep 17 00:00:00 2001 From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com> Date: Sun, 10 Mar 2024 13:11:03 +0900 Subject: [PATCH 06/12] Update docker-multiarch.yml --- .github/workflows/docker-multiarch.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-multiarch.yml b/.github/workflows/docker-multiarch.yml index 9c30020dd..1678b6056 100644 --- a/.github/workflows/docker-multiarch.yml +++ b/.github/workflows/docker-multiarch.yml @@ -4,11 +4,6 @@ on: push: branches: [ "main" ] workflow_dispatch: - inputs: - tagName: - description: 'Tag Name' - required: true - default: 'latest' jobs: build: @@ -42,7 +37,7 @@ jobs: context: . platforms: linux/amd64, linux/arm64 push: false - tags: ghcr.io/loxilb-io/loxilb:${{ github.event.inputs.tagName }} + tags: ghcr.io/loxilb-io/loxilb:latest - name: Build and push if: | @@ -52,4 +47,4 @@ jobs: context: . platforms: linux/amd64, linux/arm64 push: true - tags: ghcr.io/loxilb-io/loxilb:${{ github.event.inputs.tagName }} + tags: ghcr.io/loxilb-io/loxilb:latest From 039eb42f6814d01b70425afe2d191f02b4cdadd3 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Sun, 10 Mar 2024 13:13:08 +0900 Subject: [PATCH 07/12] gh-516 - gh-actions support for multi-arch docker --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 14b0d11e9..88dec538c 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -47,7 +47,7 @@ jobs: if: | github.repository == 'loxilb-io/loxilb' && github.event.inputs.tagName == '' - run: docker push ghcr.io/loxilb-io/loxilb:latest + run: docker push ghcr.io/loxilb-io/loxilb:latest-amd64 - name: Publish the loxilb Docker image with given tag if: | From 532351731553eed1573fb233c77e5ab1ae452da7 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Sun, 10 Mar 2024 14:51:10 +0900 Subject: [PATCH 08/12] gh-516 - gh-actions support for multi-arch docker --- .github/workflows/docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 88dec538c..c69d246df 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -34,8 +34,8 @@ jobs: - name: Build and run the latest loxilb Docker image if: github.event.inputs.tagName == '' run: | - docker build . --tag ghcr.io/loxilb-io/loxilb:latest - docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest + docker build . --tag ghcr.io/loxilb-io/loxilb:latest-amd64 + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest-amd64 - name: Build and run the loxilb Docker image with given tag if: github.event.inputs.tagName != '' From a30beacfe999fb0ec645bd229178c3d031bfdefa Mon Sep 17 00:00:00 2001 From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com> Date: Mon, 11 Mar 2024 09:52:59 +0900 Subject: [PATCH 09/12] Update docker-multiarch.yml --- .github/workflows/docker-multiarch.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-multiarch.yml b/.github/workflows/docker-multiarch.yml index 1678b6056..24157f3c0 100644 --- a/.github/workflows/docker-multiarch.yml +++ b/.github/workflows/docker-multiarch.yml @@ -4,6 +4,11 @@ on: push: branches: [ "main" ] workflow_dispatch: + inputs: + tagName: + description: 'Tag Name' + required: true + default: 'latest' jobs: build: @@ -39,12 +44,25 @@ jobs: push: false tags: ghcr.io/loxilb-io/loxilb:latest - - name: Build and push + - name: Build and push to latest if: | - github.repository == 'loxilb-io/loxilb' + github.repository == 'loxilb-io/loxilb' + && github.event.inputs.tagName == '' uses: docker/build-push-action@v4 with: context: . platforms: linux/amd64, linux/arm64 push: true tags: ghcr.io/loxilb-io/loxilb:latest + + - name: Build and push to given tag + if: | + github.repository == 'loxilb-io/loxilb' + && github.event.inputs.tagName != '' + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64, linux/arm64 + push: true + tags: ghcr.io/loxilb-io/loxilb:${{ github.event.inputs.tagName }} + From 8e2f8306edc701f3facb8ad0828d09b9bc166c60 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Mon, 11 Mar 2024 15:23:57 +0900 Subject: [PATCH 10/12] cicd: host docker changed to netshoot for multi-arch --- cicd/common.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cicd/common.sh b/cicd/common.sh index 974d57bce..e8725ea42 100644 --- a/cicd/common.sh +++ b/cicd/common.sh @@ -12,12 +12,13 @@ dexec="sudo docker exec -i " hns="sudo ip netns " hexist="$vrn$hn" lxdocker="ghcr.io/loxilb-io/loxilb:latest" +hostdocker="ghcr.io/nicolaka/netshoot:latest" cluster_opts="" ka_opts="" var=$(lsb_release -r | cut -f2) -if [[ $var == *"22.04"* ]];then - lxdocker="ghcr.io/loxilb-io/loxilb:latestu22" -fi +#if [[ $var == *"22.04"* ]];then +# lxdocker="ghcr.io/loxilb-io/loxilb:latestu22" +#fi loxilbs=() @@ -32,11 +33,9 @@ pull_dockers() { ## loxilb docker docker pull $lxdocker ## Host docker - docker pull eyes852/ubuntu-iperf-test:0.5 + docker pull docker pull $hostdocker ## BGP host docker docker pull ewindisch/quagga - ## Keepalive docker - docker pull osixia/keepalived:2.0.20 } ## Creates a docker host @@ -119,7 +118,7 @@ spawn_docker_host() { if [[ "$bgp" == "yes" || ! -z "$bpath" ]]; then docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit $bgp_conf --name $dname ewindisch/quagga else - docker run -u root --cap-add SYS_ADMIN -dit --name $dname eyes852/ubuntu-iperf-test:0.5 + docker run -u root --cap-add SYS_ADMIN -dit --name $dname $hostdocker fi elif [[ "$dtype" == "seahost" ]]; then docker run -u root --cap-add SYS_ADMIN -i -t --rm --detach --entrypoint /bin/bash --name $dname ghcr.io/loxilb-io/seagull:ubuntu1804 @@ -173,13 +172,13 @@ get_llb_peerIP() { ## Deletes a docker host ## arg1 - hostname delete_docker_host() { - if docker stop $1 2>&1 >> /dev/null + if docker kill $1 2>&1 >> /dev/null then hd="true" ka=`docker ps -f name=ka_$1| grep -w ka_$1 | cut -d " " -f 1 | grep -iv "CONTAINER"` loxilbs=( "${loxilbs[@]/$1}" ) if [ "$ka" != "" ]; then - docker stop ka_$1 2>&1 >> /dev/null + docker kill ka_$1 2>&1 >> /dev/null docker rm ka_$1 2>&1 >> /dev/null fi fi From a2fb20017b04f870e77cf8d0e672eada62dfa5de Mon Sep 17 00:00:00 2001 From: TrekkieCoder <111065900+TrekkieCoder@users.noreply.github.com> Date: Mon, 11 Mar 2024 15:38:13 +0900 Subject: [PATCH 11/12] Update nat66-sanity-ubuntu-22.yml --- .github/workflows/nat66-sanity-ubuntu-22.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nat66-sanity-ubuntu-22.yml b/.github/workflows/nat66-sanity-ubuntu-22.yml index 870dd3aaf..3abcd4c99 100644 --- a/.github/workflows/nat66-sanity-ubuntu-22.yml +++ b/.github/workflows/nat66-sanity-ubuntu-22.yml @@ -43,7 +43,7 @@ jobs: - run: sudo -E env "PATH=$PATH" make test - run: docker pull ghcr.io/loxilb-io/loxilb:latest - run: docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest - - run: pwd && ls && sudo -E env "PATH=$PATH" make docker-cp + - run: pwd && ls && sudo -E env "PATH=$PATH" make docker-cp-ebpf - run: docker exec -dit loxilb mkllb_bpffs - run: id=`docker ps -f name=loxilb | cut -d " " -f 1 | grep -iv "CONTAINER"` && docker commit $id ghcr.io/loxilb-io/loxilb:latest - run: docker stop loxilb && docker rm loxilb From f0d57296c407e8e73e800b97c94c6bd7b7e4633c Mon Sep 17 00:00:00 2001 From: UltraInstinct14 Date: Mon, 11 Mar 2024 21:19:50 +0900 Subject: [PATCH 12/12] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 5d5be0a50..9c54f9e79 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,11 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S - [How-To : Standalone configuration](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/cmd.md) - [How-To : debug](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/debugging.md) +## Getting started with different K8s distributions & tools + +- [K3s : loxilb with default flannel](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s_quick_start_flannel.md) +- [K3s : loxilb with cilium](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/quick_start_with_cilium.md) + ## Knowledge-Base - [What is eBPF](ebpf.md) - [What is k8s service - load-balancer](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/lb.md)