[Release-1.25] Add E2E testing in Drone (#7375)

* Add E2E to Drone * Build e2e test image * Add ci flag to secretsencryption * Fix vagrant log on secretsencryption * Add cron conformance pipeline Signed-off-by: Derek Nola <[email protected]> * Add string output for nodes * Switch snapshot restore for upgrade cluster Signed-off-by: Derek Nola <[email protected]> * Added IPv6 check and agent restart on e2e test utils Signed-off-by: Roberto Bonafiglia <[email protected]> * Cleanup leftover VMs in E2E pipeline * Dont run most pipelines on nightly cron Signed-off-by: Derek Nola <[email protected]> * Don't default to local K3s for startup test (#6950) Signed-off-by: Derek Nola <[email protected]> * Added multiClusterCIDR E2E test Signed-off-by: Roberto Bonafiglia <[email protected]> * fix_get_sha_url (#7187) Signed-off-by: ShylajaDevadiga <[email protected]> * Improve RunCmdOnNode error Signed-off-by: Derek Nola <[email protected]> * Pin upgradecluster to v1.25 Signed-off-by: Derek Nola <[email protected]> --------- Signed-off-by: Derek Nola <[email protected]> Signed-off-by: Roberto Bonafiglia <[email protected]> Signed-off-by: ShylajaDevadiga <[email protected]> Co-authored-by: Roberto Bonafiglia <[email protected]> Co-authored-by: ShylajaDevadiga <[email protected]>
k3s-io · May 1, 2023 · 71e53ae · 71e53ae
1 parent 7502ce6
commit 71e53ae
Show file tree

Hide file tree

Showing 17 changed files with 800 additions and 68 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -6,6 +6,11 @@ platform:
   os: linux
   arch: amd64
 
+trigger:
+  event:
+    exclude:
+    - cron
+
 steps:
 - name: build
   image: rancher/dapper:v0.5.0
@@ -128,6 +133,48 @@ volumes:
   host:
     path: /var/run/docker.sock
 
+---
+kind: pipeline
+name: conformance
+
+platform:
+  os: linux
+  arch: amd64
+
+trigger:
+  event:
+  - cron
+  cron:
+  - nightly
+
+steps:
+- name: build
+  image: rancher/dapper:v0.5.0
+  commands:
+  - dapper ci
+  - echo "${DRONE_TAG}-amd64" | sed -e 's/+/-/g' >.tags
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+
+- name: test
+  image: rancher/dapper:v0.5.0
+  environment:
+    ENABLE_REGISTRY: 'true'
+  commands:
+  - docker build --target test-k3s -t k3s:test-${DRONE_STAGE_ARCH}-${DRONE_COMMIT} -f Dockerfile.test .
+  - >
+    docker run -i -e REPO -e TAG -e DRONE_TAG -e DRONE_BUILD_EVENT -e IMAGE_NAME -e SONOBUOY_VERSION -e ENABLE_REGISTRY
+    -v /var/run/docker.sock:/var/run/docker.sock --privileged --network host -v /tmp:/tmp k3s:test-${DRONE_STAGE_ARCH}-${DRONE_COMMIT}
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+
+volumes:
+- name: docker
+  host:
+    path: /var/run/docker.sock
+
 ---
 kind: pipeline
 name: arm64
@@ -136,6 +183,11 @@ platform:
   os: linux
   arch: arm64
 
+trigger:
+  event:
+    exclude:
+    - cron
+
 steps:
 - name: build
   image: rancher/dapper:v0.5.0
@@ -222,6 +274,11 @@ platform:
   os: linux
   arch: arm
 
+trigger:
+  event:
+    exclude:
+    - cron
+
 steps:
 - name: build
   image: rancher/dapper:v0.5.0
@@ -312,6 +369,11 @@ platform:
 node:
   arch: s390x
 
+trigger:
+  event:
+    exclude:
+    - cron
+
 clone:
   disable: true
 
@@ -412,6 +474,11 @@ platform:
   os: linux
   arch: amd64
 
+trigger:
+  event:
+    exclude:
+    - cron
+
 steps:
   - name: validate_go_mods
     image: rancher/dapper:v0.5.0
@@ -459,7 +526,10 @@ trigger:
   - refs/head/master
   - refs/tags/*
   event:
-  - tag
+    include:
+    - tag
+    exclude:
+      - cron
 
 depends_on:
 - amd64
@@ -499,3 +569,76 @@ trigger:
 
 depends_on:
 - manifest
+
+---
+kind: pipeline
+name: e2e
+type: docker
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: build-e2e-image
+  image: rancher/dapper:v0.5.0
+  commands:
+    - DOCKER_BUILDKIT=1 docker build --target test-e2e -t test-e2e -f Dockerfile.test .
+    - SKIP_VALIDATE=true SKIP_AIRGAP=true dapper ci
+    - cp dist/artifacts/* /tmp/artifacts/
+  volumes:
+  - name: cache
+    path: /tmp/artifacts
+  - name: docker
+    path: /var/run/docker.sock
+
+- name: test-e2e
+  image: test-e2e
+  pull: never
+  resources:
+    cpu: 6000
+    memory: 10Gi
+  environment:
+    E2E_REGISTRY: 'true'
+  commands:
+  - mkdir -p dist/artifacts
+  - cp /tmp/artifacts/* dist/artifacts/
+  - docker stop registry && docker rm registry
+  # Cleanup any VMs running, happens if a previous test panics
+  - |
+    VMS=$(virsh list --name | grep '_server-\|_agent-' || true)
+    if [ -n "$VMS" ]; then
+      for vm in $VMS
+      do
+        virsh destroy $vm
+        virsh undefine $vm --remove-all-storage
+      done
+    fi 
+  - docker run -d -p 5000:5000 -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io --name registry registry:2
+  - cd tests/e2e/validatecluster
+  - vagrant destroy -f
+  - go test -v -timeout=45m ./validatecluster_test.go -ci -local
+  - cd ../secretsencryption
+  - vagrant destroy -f
+  - go test -v -timeout=30m ./secretsencryption_test.go -ci -local
+  - cd ../upgradecluster
+  - E2E_RELEASE_CHANNEL="v1.25" go test -v -timeout=45m ./upgradecluster_test.go  -ci -local
+  - docker stop registry && docker rm registry
+
+  volumes:
+  - name: libvirt
+    path: /var/run/libvirt/
+  - name: docker
+    path: /var/run/docker.sock
+  - name: cache
+    path: /tmp/artifacts
+
+volumes:
+- name: docker
+  host:
+    path: /var/run/docker.sock
+- name: libvirt
+  host:
+    path: /var/run/libvirt/
+- name: cache
+  temp: {}
diff --git a/Dockerfile.test b/Dockerfile.test
@@ -35,3 +35,21 @@ ENV TEST_CLEANUP true
 
 ENTRYPOINT ["./scripts/entry.sh"]
 CMD ["test"]
+
+
+FROM vagrantlibvirt/vagrant-libvirt:0.10.7 AS test-e2e
+
+RUN apt-get update && apt-get install -y docker.io
+RUN vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp
+RUN vagrant box add generic/ubuntu2004 --provider libvirt --force
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \
+    chmod +x ./kubectl; \
+    mv ./kubectl /usr/local/bin/kubectl
+ENV GO_VERSION 1.19.2
+RUN curl -O -L "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz"; \
+    rm -rf /usr/local/go; \
+    tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz;
+
+ENV PATH="${PATH}:/usr/local/go/bin"
+
+
diff --git a/pkg/agent/loadbalancer/loadbalancer.go b/pkg/agent/loadbalancer/loadbalancer.go
@@ -56,7 +56,7 @@ var (
 	ETCDServerServiceName = version.Program + "-etcd-server-load-balancer"
 )
 
-// New contstructs a new LoadBalancer instance. The default server URL, and
+// New constructs a new LoadBalancer instance. The default server URL, and
 // currently active servers, are stored in a file within the dataDir.
 func New(ctx context.Context, dataDir, serviceName, serverURL string, lbServerPort int, isIPv6 bool) (_lb *LoadBalancer, _err error) {
 	config := net.ListenConfig{Control: reusePort}

diff --git a/scripts/test b/scripts/test
@@ -43,43 +43,53 @@ echo "Did test-run-lazypull $?"
 [ "$ARCH" != 'amd64' ] && \
   early-exit "Skipping remaining tests, images not available for $ARCH."
 
-E2E_OUTPUT=$artifacts test-run-sonobuoy serial
-echo "Did test-run-sonobuoy serial $?"
-
 # ---
 
 if [ "$DRONE_BUILD_EVENT" = 'tag' ]; then
+  E2E_OUTPUT=$artifacts test-run-sonobuoy serial
+  echo "Did test-run-sonobuoy serial $?"
   E2E_OUTPUT=$artifacts test-run-sonobuoy parallel
   echo "Did test-run-sonobuoy parallel $?"
   early-exit 'Skipping remaining tests on tag.'
 fi
 # ---
 
-test-run-sonobuoy etcd serial
-echo "Did test-run-sonobuoy-etcd serial $?"
-test-run-sonobuoy mysql serial
-echo "Did test-run-sonobuoy-mysqk serial $?"
-test-run-sonobuoy postgres serial
-echo "Did test-run-sonobuoy-postgres serial $?"
+if [ "$DRONE_BUILD_EVENT" = 'cron' ]; then
+  E2E_OUTPUT=$artifacts test-run-sonobuoy serial
+  echo "Did test-run-sonobuoy serial $?"
+  test-run-sonobuoy etcd serial
+  echo "Did test-run-sonobuoy-etcd serial $?"
+  test-run-sonobuoy mysql serial
+  echo "Did test-run-sonobuoy-mysqk serial $?"
+  test-run-sonobuoy postgres serial
+  echo "Did test-run-sonobuoy-postgres serial $?"
+
+  # Wait until all serial tests have finished
+  delay=15
+  (
+  set +x
+  while [ $(count-running-tests) -ge 1 ]; do
+      sleep $delay
+  done
+  )
+
+  E2E_OUTPUT=$artifacts test-run-sonobuoy parallel
+  echo "Did test-run-sonobuoy parallel $?"
+  test-run-sonobuoy etcd parallel
+  echo "Did test-run-sonobuoy-etcd parallel $?"
+  test-run-sonobuoy mysql parallel
+  echo "Did test-run-sonobuoy-mysql parallel $?"
+  test-run-sonobuoy postgres parallel
+  echo "Did test-run-sonobuoy-postgres parallel $?"
+fi
 
-# Wait until all serial tests have finished
+# Wait until all tests have finished
 delay=15
 (
 set +x
 while [ $(count-running-tests) -ge 1 ]; do
     sleep $delay
 done
 )
-E2E_OUTPUT=$artifacts test-run-sonobuoy parallel
-echo "Did test-run-sonobuoy parallel $?"
-test-run-sonobuoy etcd parallel
-echo "Did test-run-sonobuoy-etcd parallel $?"
-test-run-sonobuoy mysql parallel
-echo "Did test-run-sonobuoy-mysql parallel $?"
-test-run-sonobuoy postgres parallel
-echo "Did test-run-sonobuoy-postgres parallel $?"
-
-
-
 
 exit 0
diff --git a/tests/e2e/amd64_resource_files/cluster-cidr-ipv6.yaml b/tests/e2e/amd64_resource_files/cluster-cidr-ipv6.yaml
@@ -0,0 +1,14 @@
+apiVersion: networking.k8s.io/v1alpha1
+kind: ClusterCIDR
+metadata:
+  name: new-cidr
+spec:
+  nodeSelector:
+    nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          -  "agent-0"
+  perNodeHostBits: 64
+  ipv6: 2001:cafe:248::/56
diff --git a/tests/e2e/amd64_resource_files/cluster-cidr.yaml b/tests/e2e/amd64_resource_files/cluster-cidr.yaml
@@ -0,0 +1,14 @@
+apiVersion: networking.k8s.io/v1alpha1
+kind: ClusterCIDR
+metadata:
+  name: new-cidr
+spec:
+  nodeSelector:
+    nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          -  "agent-0"
+  perNodeHostBits: 8
+  ipv4: 10.248.0.0/16