External EP cicd added

cicd/k3s-ext-ep/Vagrantfile

@@ -0,0 +1,36 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+workers = (ENV['WORKERS'] || "2").to_i
+#box_name = (ENV['VAGRANT_BOX'] || "ubuntu/focal64")
+box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s")
+box_version = "0.7.1"
+Vagrant.configure("2") do |config|
+  config.vm.box = "#{box_name}"
+  config.vm.box_version = "#{box_version}"
+
+  if Vagrant.has_plugin?("vagrant-vbguest")
+    config.vbguest.auto_update = false
+  end
+
+  config.vm.define "host" do |host|
+    host.vm.hostname = 'host'
+    host.vm.network :private_network, ip: "192.168.82.2", :netmask => "255.255.255.0"
+    host.vm.provision :shell, :path => "host.sh"
+    host.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 2048]
+        vbox.customize ["modifyvm", :id, "--cpus", 2]
+    end
+  end
+
+  config.vm.define "master" do |master|
+    master.vm.hostname = 'master'
+    master.vm.network :private_network, ip: "192.168.82.128", :netmask => "255.255.255.0"
+    master.vm.provision :shell, :path => "master.sh"
+    master.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 8192]
+        vbox.customize ["modifyvm", :id, "--cpus", 8]
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+    end
+  end
+end

cicd/k3s-ext-ep/config.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+vagrant global-status  | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f
+vagrant up

cicd/k3s-ext-ep/ext-tcp.yml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: ext-tcp-lb
+spec:
+ loadBalancerClass: loxilb.io/loxilb
+ type: LoadBalancer 
+ ports:
+    - protocol: TCP
+      port: 8000
+      targetPort: 80
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: ext-tcp-lb
+subsets:
+  - addresses:
+    - ip: 192.168.82.2
+    ports:
+    - port: 80

cicd/k3s-ext-ep/host.sh

@@ -0,0 +1,10 @@
+echo "20.20.20.1 k8s-svc" >> /etc/hosts
+apt-get update
+apt-get install -y software-properties-common lksctp-tools
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+apt-get update
+apt-get install -y docker-ce
+docker run --cap-add SYS_ADMIN -dit --net=host --name tcp_ep ghcr.io/loxilb-io/nginx:stable
+sudo ip route add 20.20.20.1 via 192.168.82.100
+echo "Host is up"

cicd/k3s-ext-ep/k3s.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkakNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTVRFME16SXpNRGN3SGhjTk1qUXdNekkyTURVMU1UUTNXaGNOTXpRd016STBNRFUxTVRRMwpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTVRFME16SXpNRGN3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTK296V2p3aEJRcHZrSDVCaXJFSVdnRnhNRGR0OFZsaXZLM1Y5Ym56T1EKQmdnUXVkQWxlQ0FYUmk4RjdhWU9ISXUwaGZPbkthRi84cnJTZnlvdWJrSkdvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXYyT0xtZFo5TDFublNTTjExbUhCCmxQWmxjbGt3Q2dZSUtvWkl6ajBFQXdJRFJ3QXdSQUlnU05IbXNMakh5bkdMRTRFY04yVjVtZFNJaWRNdHNVUTcKSHIwbi9BdUg1NUVDSUVjSGExY3c1RGdOeEZvaDRCWWpQRGdycDIwam9DTVZ6d2ZXRGNzdVhLTUQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+    server: https://192.168.82.128:6443
+  name: default
+contexts:
+- context:
+    cluster: default
+    user: default
+  name: default
+current-context: default
+kind: Config
+preferences: {}
+users:
+- name: default
+  user:
+    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRlZ0F3SUJBZ0lJV3BvZGdxci9Bb1V3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOekV4TkRNeU16QTNNQjRYRFRJME1ETXlOakExTlRFME4xb1hEVEkxTURNeQpOakExTlRFME4xb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJKU1gyRTdwbHNmM0RlM0cKWHZZclJNYVlKaWJmNGpoTlhYSXRYbWEwN01ZV3VXLzB3N1BVTlcyRW9KSVJKdUd0MHVuK0dmbmtvaUd3aG5udwpUWHBJNVlXalNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCU1RiQ0l6U0ZGaGdNTklnZitrcHNJMU5Ma2JlekFLQmdncWhrak9QUVFEQWdOSkFEQkcKQWlFQTJBMjlDNktDTDBYazVMZFBXTjBJLzA3cWZjNEpFd3k0OFJrd0QvYTBER29DSVFDeEYvQkMyTjYrTVVWQQowUFNyZTloeFA4WkZpa3Mzd3pCbnFxU0VTbUIrMGc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZGpDQ0FSMmdBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwClpXNTBMV05oUURFM01URTBNekl6TURjd0hoY05NalF3TXpJMk1EVTFNVFEzV2hjTk16UXdNekkwTURVMU1UUTMKV2pBak1TRXdId1lEVlFRRERCaHJNM010WTJ4cFpXNTBMV05oUURFM01URTBNekl6TURjd1dUQVRCZ2NxaGtqTwpQUUlCQmdncWhrak9QUU1CQndOQ0FBUlVDY0t3TzJIUEJSNXNBcWUvamx1VC9rc25iSWVVQUtsMWpmbDZLbThjCnBwbm93LzNXNkdGbVhBekErRHpCVmtaODBYU0IzN3J0UUl0cHB3alNrV0hvbzBJd1FEQU9CZ05WSFE4QkFmOEUKQkFNQ0FxUXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVrMndpTTBoUllZRERTSUgvcEtiQwpOVFM1RzNzd0NnWUlLb1pJemowRUF3SURSd0F3UkFJZ0djeTQ3dmhJT3lqdHgvMjkva2VTV2p5OFUrd3pmN1ROCkhraWhER21aS3pBQ0lGdEp0YnlGTmU0enRrSHZQRDdBKzcrUDYwaWZ3eGhDUGJlQW81Ny9UYXJnCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUxkOVk0WUl3bjhyZzZ4NFAxb1ZZWE83dEZqRm8ydGRqZGp2bVhSMkhlL0VvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFbEpmWVR1bVd4L2NON2NaZTlpdEV4cGdtSnQvaU9FMWRjaTFlWnJUc3hoYTViL1REczlRMQpiWVNna2hFbTRhM1M2ZjRaK2VTaUliQ0dlZkJOZWtqbGhRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=

cicd/k3s-ext-ep/kube-loxilb.yml

@@ -0,0 +1,134 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - services
+      - services/status
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+      - update
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-loxilb
+subjects:
+  - kind: ServiceAccount
+    name: kube-loxilb
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+  labels:
+    app: loxilb
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: loxilb
+  template:
+    metadata:
+      labels:
+        app: loxilb
+    spec:
+      hostNetwork: true
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      priorityClassName: system-node-critical
+      serviceAccountName: kube-loxilb
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: kube-loxilb
+        image: ghcr.io/loxilb-io/kube-loxilb:latest
+        imagePullPolicy: Always
+        command:
+        - /bin/kube-loxilb
+        args:
+        - --loxiURL=http://172.17.0.2:11111
+        - --externalCIDR=20.20.20.1/32
+        #- --externalSecondaryCIDRs=124.124.124.1/24,125.125.125.1/24
+        #- --monitor
+        #- --setBGP=64511
+        #- --extBGPPeers=50.50.50.1:65101,51.51.51.1:65102
+        #- --setRoles=0.0.0.0
+        #- --monitor
+        #- --setBGP
+        - --setLBMode=2
+        #- --config=/opt/loxilb/agent/kube-loxilb.conf
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["NET_ADMIN", "NET_RAW"]

cicd/k3s-ext-ep/master.sh

@@ -0,0 +1,54 @@
+export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.82' | awk '{print $2}' | cut -f1 -d '/')
+
+apt-get update
+apt-get install -y software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+apt-get update
+apt-get install -y docker-ce
+
+## Set promisc mode for mac-vlan to work
+sudo ifconfig eth1 promisc
+
+sudo docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged --entrypoint /root/loxilb-io/loxilb/loxilb -dit -v /dev/log:/dev/log  --name loxilb ghcr.io/loxilb-io/loxilb:latest
+
+# Create mac-vlan on top of underlying eth1 interface
+docker network create -d macvlan -o parent=eth1 --subnet 192.168.82.0/24   --gateway 192.168.82.1 --aux-address 'host=192.168.82.252' llbnet
+
+# Assign mac-vlan to loxilb docker with specified IP (which will be used as LB VIP)
+docker network connect llbnet loxilb --ip=192.168.82.100
+
+# Add iptables rule to allow traffic from source IP(192.168.82.1) to loxilb
+sudo iptables -A DOCKER -s 192.168.82.1 -j ACCEPT
+
+
+#K3s installation
+curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable traefik --disable servicelb --disable-cloud-controller  \
+--flannel-backend=none \
+--disable-network-policy" sh -
+
+#Install Cilium
+CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
+CLI_ARCH=amd64
+if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
+curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
+sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
+sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
+rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
+mkdir -p ~/.kube/
+sudo cat /etc/rancher/k3s/k3s.yaml > ~/.kube/config
+cilium install
+
+echo $MASTER_IP > /vagrant/master-ip
+sudo cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token
+sudo cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml
+sudo sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /vagrant/k3s.yaml
+\
+#Add route for service IP towards loxilb
+sudo ip route add 20.20.20.1/32 via 172.17.0.2
+
+/vagrant/wait_ready.sh
+sudo kubectl apply -f /vagrant/kube-loxilb.yml
+sudo kubectl apply -f /vagrant/nginx.yml
+sudo kubectl apply -f /vagrant/ext-tcp.yml
+/vagrant/wait_ready.sh

cicd/k3s-ext-ep/nginx.yml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-lb1
+spec:
+  externalTrafficPolicy: Local
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    what: nginx-test
+  ports:
+    - port: 55002
+      targetPort: 80 
+  type: LoadBalancer
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-test
+  labels:
+    what: nginx-test
+spec:
+  containers:
+    - name: nginx-test
+      image: nginx:stable
+      ports:
+        - containerPort: 80

cicd/k3s-ext-ep/rmconfig.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+vagrant destroy -f master
+vagrant destroy -f host
+rm master-ip node-token extIP
+

cicd/k3s-ext-ep/validation.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+source ../common.sh
+echo k3s-ext-ip
+
+if [ "$1" ]; then
+  KUBECONFIG="$1"
+fi
+
+set -eo pipefail
+# Set space as the delimiter
+IFS=' '
+
+for((i=0; i<120; i++))
+do
+  extLB=$(vagrant ssh master -c 'sudo kubectl get svc' 2> /dev/null | grep "nginx")
+  read -a strarr <<< "$extLB"
+  len=${#strarr[*]}
+  if [[ $((len)) -lt 6 ]]; then
+    echo "Can't find tcp-lb service"
+    sleep 1
+    continue
+  fi 
+  if [[ ${strarr[3]} != *"none"* ]]; then
+    extIP="$(cut -d'-' -f2 <<<${strarr[3]})"
+    break
+  fi
+  echo "No external LB allocated"
+  sleep 1
+done
+
+## Any routing updates  ??
+#sleep 30
+
+echo Service IP : $extIP
+echo $extIP > extIP
+
+echo -e "\nEnd Points List"
+echo "******************************************************************************"
+vagrant ssh master -c 'sudo kubectl get endpoints -A' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\nSVC List"
+echo "******************************************************************************"
+vagrant ssh master -c 'sudo kubectl get svc' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\nPod List"
+echo "******************************************************************************"
+vagrant ssh master -c 'sudo kubectl get pods -A' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\nLB List"
+echo "******************************************************************************"
+vagrant ssh master -c 'sudo sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null
+echo "******************************************************************************"
+echo -e "\nEP List"
+echo "******************************************************************************"
+vagrant ssh master -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null
+echo "******************************************************************************"
+
+echo -e "\nTEST RESULTS"
+echo "******************************************************************************"
+
+echo -e "\n\nCommand: curl --connect-time 10 http://20.20.20.1:55002'\n\n"
+vagrant ssh host -c 'curl --connect-time 10 http://20.20.20.1:55002' 2> /dev/null
+echo -e "\n\n\nConnecting external EP service from the pod\n\n"
+echo "sudo kubectl exec -it nginx-test -- curl 20.20.20.1:8000\n"
+vagrant ssh master -c 'sudo kubectl exec -it nginx-test -- curl 20.20.20.1:8000' 2> /dev/null