Merge pull request #510 from TrekkieCoder/main

PR - cicd : scenario for k3s single-node incluster with calico
loxilb-io · Jan 27, 2024 · bb6b2f8 · bb6b2f8
2 parents 12fe978 + 2f9bb43
commit bb6b2f8
Show file tree

Hide file tree

Showing 17 changed files with 495 additions and 0 deletions.
diff --git a/cicd/k3s-calico-single-node-incluster/Vagrantfile b/cicd/k3s-calico-single-node-incluster/Vagrantfile
@@ -0,0 +1,35 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+workers = (ENV['WORKERS'] || "2").to_i
+#box_name = (ENV['VAGRANT_BOX'] || "ubuntu/focal64")
+box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s")
+box_version = "0.7.1"
+Vagrant.configure("2") do |config|
+  config.vm.box = "#{box_name}"
+  config.vm.box_version = "#{box_version}"
+
+  if Vagrant.has_plugin?("vagrant-vbguest")
+    config.vbguest.auto_update = false
+  end
+
+  config.vm.define "host" do |host|
+    host.vm.hostname = 'host1'
+    host.vm.network :private_network, ip: "192.168.82.2", :netmask => "255.255.255.0"
+    host.vm.provision :shell, :path => "host.sh"
+    host.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 2048]
+        vbox.customize ["modifyvm", :id, "--cpus", 2]
+    end
+  end
+
+  config.vm.define "master1" do |master|
+    master.vm.hostname = 'master1'
+    master.vm.network :private_network, ip: "192.168.82.128", :netmask => "255.255.255.0"
+    master.vm.provision :shell, :path => "master1.sh"
+    master.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 8192]
+        vbox.customize ["modifyvm", :id, "--cpus", 8]
+    end
+  end
+end
diff --git a/cicd/k3s-calico-single-node-incluster/client.yaml b/cicd/k3s-calico-single-node-incluster/client.yaml
@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: netshoot-2
+  labels:
+    app: MyApp1
+    kubernetes.io/hostname: master
+spec:
+  containers:
+  - name: netshoot
+    image: nicolaka/netshoot
+    imagePullPolicy: IfNotPresent
+    command: [ "sleep" ]
+    args: [ "infinity" ]
+    ports:
+    - name: svr
+      containerPort: 32345
+      protocol: TCP
+    securityContext:
+      capabilities:
+        add:
+          - all
diff --git a/cicd/k3s-calico-single-node-incluster/config.sh b/cicd/k3s-calico-single-node-incluster/config.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+vagrant global-status  | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f
+vagrant up
diff --git a/cicd/k3s-calico-single-node-incluster/host.sh b/cicd/k3s-calico-single-node-incluster/host.sh
@@ -0,0 +1,5 @@
+sudo su
+echo "123.123.123.1 k8s-svc" >> /etc/hosts
+ifconfig eth2 mtu 1450
+ip route add 123.123.123.0/24 via 192.168.90.10
+echo "Host is up"
diff --git a/cicd/k3s-calico-single-node-incluster/kube-loxilb.yml b/cicd/k3s-calico-single-node-incluster/kube-loxilb.yml
@@ -0,0 +1,134 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - services
+      - services/status
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+      - update
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-loxilb
+subjects:
+  - kind: ServiceAccount
+    name: kube-loxilb
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+  labels:
+    app: kube-loxilb-app
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kube-loxilb-app
+  template:
+    metadata:
+      labels:
+        app: kube-loxilb-app
+    spec:
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      priorityClassName: system-node-critical
+      serviceAccountName: kube-loxilb
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: kube-loxilb
+        image: ghcr.io/loxilb-io/kube-loxilb:latest
+        imagePullPolicy: Always
+        command:
+        - /bin/kube-loxilb
+        args:
+        - --loxiURL=http://192.168.82.128:11111
+        - --externalCIDR=192.168.80.5/32
+        #- --externalSecondaryCIDRs=124.124.124.1/24,125.125.125.1/24
+        #- --setBGP=64512
+        #- --listenBGPPort=1791
+        - --setRoles=0.0.0.0
+        #- --monitor
+        #- --extBGPPeers=50.50.50.1:65101,51.51.51.1:65102
+        #- --setLBMode=1
+        #- --config=/opt/loxilb/agent/kube-loxilb.conf
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["NET_ADMIN", "NET_RAW"]
diff --git a/cicd/k3s-calico-single-node-incluster/loxilb.yml b/cicd/k3s-calico-single-node-incluster/loxilb.yml
@@ -0,0 +1,70 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: loxilb-lb
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: loxilb-app
+  template:
+    metadata:
+      name: loxilb-lb
+      labels:
+        app: loxilb-app
+    spec:
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      tolerations:
+      - key: "node-role.kubernetes.io/master"
+        operator: Exists
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: Exists
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: "node-role.kubernetes.io/master"
+                operator: Exists
+              - key: "node-role.kubernetes.io/control-plane"
+                operator: Exists
+      containers:
+      - name: loxilb-app
+        image: "ghcr.io/loxilb-io/loxilb:debug"
+        imagePullPolicy: Always
+        command: [ "/root/loxilb-io/loxilb/loxilb", "--egr-hooks", "--blacklist=cali.|tunl.|vxlan[.]calico|veth.|cni[0-9a-z]" ]
+        #command: [ "sleep" ]
+        #args: [ "infinity" ]
+        ports:
+        - containerPort: 11111
+        - containerPort: 1791
+        - containerPort: 50051
+        securityContext:
+          privileged: true
+          capabilities:
+            add:
+              - SYS_ADMIN
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: loxilb-lb-service
+  namespace: kube-system
+spec:
+  clusterIP: None
+  selector:
+    app: loxilb-app
+  ports:
+  - name: loxilb-app
+    port: 11111
+    targetPort: 11111
+    protocol: TCP
+  - name: loxilb-app-bgp
+    port: 1791
+    targetPort: 1791
+    protocol: TCP
+  - name: loxilb-app-gobgp
+    port: 50051
+    targetPort: 50051
+    protocol: TCP
diff --git a/cicd/k3s-calico-single-node-incluster/master1.sh b/cicd/k3s-calico-single-node-incluster/master1.sh
@@ -0,0 +1,13 @@
+sudo su
+export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.90' | grep '192.168.82' | awk '{print $2}' | cut -f1 -d '/')
+curl -fL https://get.k3s.io | sh -s - server --node-ip=192.168.82.128 --disable servicelb --disable traefik --cluster-init external-hostname=192.168.82.128 --node-external-ip=192.168.82.128 --disable-cloud-controller  --kubelet-arg cloud-provider=external --flannel-backend=none --disable-network-policy --cluster-cidr=172.16.219.0/24
+sleep 60
+echo $MASTER_IP > /vagrant/master-ip
+cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token
+sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /etc/rancher/k3s/k3s.yaml
+cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml
+#sudo kubectl apply -f /vagrant/loxilb.yml
+#sudo kubectl apply -f /vagrant/kube-loxilb.yml
+sudo kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml
+sudo kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml
+/vagrant/wait_ready.sh
diff --git a/cicd/k3s-calico-single-node-incluster/master2.sh b/cicd/k3s-calico-single-node-incluster/master2.sh
@@ -0,0 +1,8 @@
+sudo su
+export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
+export MASTER_ADDR=$(cat /vagrant/master-ip)
+export NODE_TOKEN=$(cat /vagrant/node-token)
+
+curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 -t ${NODE_TOKEN}
+
+/vagrant/wait_ready.sh
diff --git a/cicd/k3s-calico-single-node-incluster/nginx.yml b/cicd/k3s-calico-single-node-incluster/nginx.yml
@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-lb1
+  annotations:
+    loxilb.io/lbmode: "fullnat"
+spec:
+  externalTrafficPolicy: Local
+  loadBalancerClass: loxilb.io/loxilb
+  selector:
+    what: nginx-test
+  ports:
+    - port: 55002
+      targetPort: 80 
+  type: LoadBalancer
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-test
+  labels:
+    what: nginx-test
+spec:
+  affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: "node-role.kubernetes.io/master"
+                operator: Exists
+  #            - key: "node-role.kubernetes.io/control-plane"
+  #              operator: DoesNotExist
+  containers:
+    - name: nginx-test
+      image: ghcr.io/loxilb-io/nginx:stable
+      ports:
+        - containerPort: 80
diff --git a/cicd/k3s-calico-single-node-incluster/rmconfig.sh b/cicd/k3s-calico-single-node-incluster/rmconfig.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+vagrant destroy -f master2
+vagrant destroy -f host
diff --git a/cicd/k3s-calico-single-node-incluster/sctp.yml b/cicd/k3s-calico-single-node-incluster/sctp.yml
@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sctp-lb1
+  annotations:
+   loxilb.io/liveness: "yes"
+   loxilb.io/lbmode: "fullnat"
+spec:
+  loadBalancerClass: loxilb.io/loxilb
+  externalTrafficPolicy: Local
+  selector:
+    what: sctp-test
+  ports:
+    - port: 55004
+      protocol: SCTP
+      targetPort: 9999
+  type: LoadBalancer
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sctp-test
+  labels:
+    what: sctp-test
+spec:
+  containers:
+    - name: sctp-test
+      image: ghcr.io/loxilb-io/alpine-socat:latest
+      command: [ "sh", "-c"]
+      args:
+      - while true; do
+          socat -v -T2 sctp-l:9999,reuseaddr,fork system:"echo 'server1'; cat";
+          sleep 20;
+        done;
+      ports:
+        - containerPort: 9999
+      env:
+        - name: MY_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
diff --git a/cicd/k3s-calico-single-node-incluster/sctp_client b/cicd/k3s-calico-single-node-incluster/sctp_client