adding end to end testing

Signed-off-by: John O'Loughlin <[email protected]>

.github/workflows/e2e.yml

@@ -0,0 +1,29 @@
+name: E2E
+on: [push, pull_request]
+jobs:
+
+  e2e:
+    name: E2E
+    runs-on: hugepage-runner
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        with:
+          go-version: 1.20.1
+      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: install_go_kubectl_kind
+        run: source ./ci/ci.sh && install_go_kubectl_kind
+      - name: create_kind_cluster
+        run: source ./ci/ci.sh && create_kind_cluster
+      - name: deploy_multus
+        run: source ./ci/ci.sh && deploy_multus
+      - name: vpp_e2e_test
+        run: source ./ci/ci.sh && vpp_e2e_test
+      - name: build_ovs_container
+        run: source ./ci/ci.sh && build_ovs_container
+      - name: build_test-pmd_container
+        run: source ./ci/ci.sh && build_testpmd_container
+      - name: ovs_e2e_test
+        run: source ./ci/ci.sh && ovs_e2e_test
+
+

.github/workflows/scorecard.yml

@@ -69,4 +69,4 @@ jobs:
       - name: "Upload to code-scanning"
         uses: github/codeql-action/upload-sarif@0ba4244466797eb048eb91a6cd43d5c03ca8bd05 # v2.21.2
         with:
-          sarif_file: results.sarif
+          sarif_file: results.sarif

ci/ci.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+set -e
+CI_DIR="/runner/_work/userspace-cni-network-plugin/userspace-cni-network-plugin/ci/"
+
+install_go_kubectl_kind(){
+wget -qO- https://golang.org/dl/go1.20.1.linux-amd64.tar.gz |tar -C /home/runner -xz
+export PATH="${PATH}:/home/runner/go/bin"
+echo "export PATH=\"${PATH}:/home/runner/go/bin/:home/runner/.local/bin/\"" >>~/.bashrc
+go install sigs.k8s.io/[email protected]
+
+wget -q https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl -O /home/runner/go/bin/kubectl 
+chmod +x /home/runner/go/bin/kubectl
+}
+
+
+create_kind_cluster(){
+kind create cluster
+kubectl get all --all-namespaces
+
+#docker run -itd --device=/dev/hugepages:/dev/hugepages --privileged -v "$(pwd)/docker/vpp-centos-userspace-cni/:/etc/vpp/" --name vpp ligato/vpp-base
+sleep 10
+cd /runner/_work/userspace-cni-network-plugin/userspace-cni-network-plugin/
+
+docker build . -f ./docker/userspacecni/Dockerfile -t userspacecni:latest
+kind load docker-image userspacecni
+docker exec -i kind-control-plane bash -c "mkdir -p '/runner/_work/userspace-cni-network-plugin/'"
+docker cp "/runner/_work/userspace-cni-network-plugin/userspace-cni-network-plugin/" "kind-control-plane:/runner/_work/userspace-cni-network-plugin/"
+}
+
+deploy_multus(){
+## Multus main branch has major bugs so we fix version
+cd $CI_DIR
+MULTUS_VERSION="v4.0.2"
+wget https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/$MULTUS_VERSION/deployments/multus-daemonset.yml
+sed -i "s/snapshot-thick/v4.0.2/g" multus-daemonset.yml
+kubectl apply -f ./multus-daemonset.yml
+}
+
+vpp_e2e_test(){
+cd $CI_DIR/vpp_test_setup/
+echo "Setting up vpp host"
+./vpp_host.sh
+sleep 20
+echo "Setting up vpp pods"
+./vpp_pod.sh
+
+sleep 20
+kubectl get all -A
+
+kubectl exec -n vpp vpp-app1-kind-control-plane -- ./vpp_pod_setup_memif.sh
+kubectl exec -n vpp vpp-app2-kind-control-plane -- ./vpp_pod_setup_memif.sh
+kubectl exec -n vpp vpp-app1-kind-control-plane -- vppctl "sh int address"
+kubectl exec -n vpp vpp-app2-kind-control-plane -- vppctl "sh int address"
+
+kubectl exec -n vpp vpp-app1-kind-control-plane -- vppctl "ping  192.168.1.4"
+if kubectl exec -n vpp vpp-app1-kind-control-plane -- vppctl "ping  192.168.1.4" |grep -q "bytes"; then
+	echo "VPP ping test pass"
+else 
+	echo VPP ping test failed
+	exit 1
+fi
+}
+
+build_ovs_container(){
+cd $CI_DIR/ovs_test_setup
+docker build . -t ovs
+kind load docker-image ovs
+# set alias for ovs-vsctl and ovs-ofctl in kind container, would be better if we could just install its bin
+docker exec -i kind-control-plane bash -c "echo '#!/bin/bash' > /usr/bin/ovs-vsctl"
+docker exec -i kind-control-plane bash -c "echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /usr/bin/ovs-vsctl"
+docker exec -i kind-control-plane bash -c "echo 'kubectl exec -n ovs ovs-kind-control-plane -- ovs-vsctl \"\$@\"' >> /usr/bin/ovs-vsctl"
+docker exec -i kind-control-plane bash -c "chmod +x  /usr/bin/ovs-vsctl"
+
+docker exec -i kind-control-plane bash -c "echo '#!/bin/bash' > /usr/bin/ovs-ofctl"
+docker exec -i kind-control-plane bash -c "echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /usr/bin/ovs-ofctl"
+docker exec -i kind-control-plane bash -c "echo 'kubectl exec -n ovs ovs-kind-control-plane -- ovs-ofctl \"\$@\"' >> /usr/bin/ovs-ofctl"
+docker exec -i kind-control-plane bash -c "chmod +x  /usr/bin/ovs-ofctl"
+}
+
+build_testpmd_container(){
+cd $CI_DIR/ovs_test_setup/testpmd_image
+docker build -t testpmd .
+kind load docker-image testpmd
+}
+
+ovs_e2e_test(){
+cd $CI_DIR/ovs_test_setup
+./ovs_host.sh
+sleep 20
+
+# workaround, cant create in dockerfile
+kubectl exec -n ovs pod/ovs-kind-control-plane -- bash -c "mkdir -p /dev/net/"
+kubectl exec -n ovs pod/ovs-kind-control-plane -- bash -c "mknod /dev/net/tun c 10 200"
+kubectl exec -n ovs pod/ovs-kind-control-plane -- bash -c 'ovs-vsctl set Open_vSwitch . "other_config:dpdk-init=true"'
+
+./testpmd_pod.sh
+
+sleep 30
+kubectl get all -A
+kubectl logs -n ovs pod/ovs-kind-control-plane
+kubectl describe -n ovs pod/ovs-app1-kind-control-plane
+kubectl logs -n ovs pod/ovs-app1-kind-control-plane |tail -11
+kubectl logs -n ovs pod/ovs-app2-kind-control-plane |tail -11
+pps="$(kubectl logs -n ovs pod/ovs-app2-kind-control-plane |tail -11 | grep 'RX-packets'|sed 's/ * / /g' |cut -d ' ' -f 3)"
+echo "RX Packets: $pps"
+
+if [ "$pps" -eq "0" ] || [ -z "${pps}" ]; then
+   echo "Test Failed: no traffic";
+   exit 1;
+else
+   echo "OVS Test Pass";
+fi
+}
+
+
+run_all(){
+# theese steps are triggered by the ci by sourcing this script and running the following separately
+# it gives much better logging breakdown on github
+# the run_all function is only used for manual deployment
+install_go_kubectl_kind
+create_kind_cluster
+deploy_multus
+vpp_e2e_test
+build_ovs_container
+build_testpmd_container
+ovs_e2e_test
+}
+#run_all

ci/ovs_test_setup/Dockerfile

@@ -0,0 +1,6 @@
+FROM ubuntu:22.04
+RUN apt-get -q update && apt-get install -y openvswitch-switch-dpdk
+RUN update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk
+##RUN sed -i "/rmmod bridge/d" /usr/share/openvswitch/scripts/ovs-kmod-ctl
+RUN apt install -y linux-headers-$(uname -r)
+CMD /usr/share/openvswitch/scripts/ovs-ctl start && sleep inf

ci/ovs_test_setup/ovs_host.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+kubectl delete ns ovs
+kubectl create ns ovs
+
+worker="kind-control-plane"
+cat << EOF | kubectl apply -f -
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ovs-$worker
+  labels:
+    name: ovs
+  namespace: ovs
+spec:
+  hostNetwork: false
+  nodeSelector:
+    kubernetes.io/hostname: $worker
+  hostname: ovs-$worker
+  subdomain: ovs
+  containers:
+  - image: ovs
+    imagePullPolicy: IfNotPresent
+    name: ovs-$worker
+    volumeMounts:
+    - name: vpp-api
+      mountPath: /run/openvswitch
+    - name: modules
+      mountPath: /lib/modules
+    - name: vpp-run
+      mountPath: /var/run/openvswitch
+    - name: hugepage
+      mountPath: /hugepages
+    resources:
+      requests:
+        hugepages-2Mi: 1Gi
+        memory: "1Gi"
+        cpu: "10"
+      limits:
+        hugepages-2Mi: 1Gi
+        memory: "1Gi"
+        cpu: "10"
+    securityContext:
+      capabilities:
+        add: ["NET_ADMIN", "SYS_TIME"]
+  restartPolicy: Always
+  volumes:
+    - name: vpp-run
+      hostPath:
+        path: /var/run/openvswitch/
+    - name: modules
+      hostPath:
+        path: /lib/modules
+    - name: vpp-api
+      hostPath:
+        path: /run/openvswitch/
+    - name: userspace-api
+      hostPath:
+        path: /var/lib/cni/usrspcni/
+    - name: hugepage
+      emptyDir:
+        medium: HugePages
+EOF
+

ci/ovs_test_setup/ovs_network_attachment_definition.yaml

@@ -0,0 +1,45 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: userspace-ovs-net-1
+  namespace: ovs
+spec:
+  config: '{
+        "cniVersion": "0.3.1",
+        "type": "userspace",
+        "name": "userspace-ovs-net-1",
+        "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
+        "logFile": "/var/log/userspace-ovs-net-1-cni.log",
+        "logLevel": "debug",
+        "host": {
+                "engine": "ovs-dpdk",
+                "iftype": "vhostuser",
+                "netType": "bridge",
+                "vhost": {
+                        "mode": "client"
+                },
+                "bridge": {
+                        "bridgeName": "br-4"
+                }
+        },
+        "container": {
+                "engine": "ovs-dpdk",
+                "iftype": "vhostuser",
+                "netType": "interface",
+                "vhost": {
+                        "mode": "server"
+                }
+        },
+        "ipam": {
+                "type": "host-local",
+                "subnet": "10.56.217.0/24",
+                "rangeStart": "10.56.217.131",
+                "rangeEnd": "10.56.217.190",
+                "routes": [
+                        {
+                                "dst": "0.0.0.0/0"
+                        }
+                ],
+                "gateway": "10.56.217.1"
+        }
+    }'

ci/ovs_test_setup/testpmd_image/Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:22.04
+RUN apt-get -q update && apt-get install -y git vim meson python3-pyelftools libnuma-dev
+RUN git clone https://github.com/DPDK/dpdk.git
+WORKDIR /dpdk/
+RUN meson build
+RUN cd build && ninja
+WORKDIR /dpdk/build/app
+COPY ./testpmd.sh testpmd.sh
+CMD ./testpmd.sh

ci/ovs_test_setup/testpmd_image/testpmd.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+cd /dpdk || exit 1
+#set container id  as env
+LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/x86_64-linux-gnu/
+while ! [ -s /etc/podinfo/annotations ]; do
+    echo "annotations not found yet"
+    sleep 1 # throttle the check
+done
+cat /etc/podinfo/annotations
+sleep 10
+#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
+##containerid=$(grep containerID /etc/podinfo/annotations |cut -d = -f 2 | sed 's/"//g')
+containerid=$(grep containerId /etc/podinfo/annotations |cut -d '"' -f 5 |sed 's/\\//')
+echo "${containerid:0:12}"
+#./build/app/dpdk-testpmd -l 10,14,12 --vdev net_vhost0,iface=/var/lib/cni/usrspcni/${containerid:0:12}-net1 -- -i
+cpu=$(cat /sys/fs/cgroup/cpuset.cpus)
+#--vdev "net_virtio_user0,path=/run/openvswitch/host/host-net1,server=1,queue_size=2048" --in-memory --single-file-segments --no-pci
+
+if grep -q app1 /etc/podinfo/labels; then
+        fordwardmode="txonly"
+	cpu="2,3,4,5"
+else 
+	fordwardmode="rxonly"
+	cpu="6,7,8,9"
+fi
+
+#--stats-period 1 is needed to avoid testpmd exiting
+commands="./build/app/dpdk-testpmd -l $cpu --vdev net_virtio_user0,path=/var/lib/cni/usrspcni/${containerid:0:12}-net1,server=1,queue_size=2048 --in-memory --single-file-segments -- --tx-ip 192.168.1.1,192.168.1.2 --tx-udp=4000,4000 --forward-mode=$fordwardmode --stats-period 1" #-i
+echo "$commands"
+$commands