Support MixedOS E2E local testing (rancher#6137)

* Consolidate CreateWindowsCluster test function Signed-off-by: Derek Nola <[email protected]> * Support local windows builds in mixedOS e2e test Signed-off-by: Derek Nola <[email protected]> * Add mixedos to E2E CI Signed-off-by: Derek Nola <[email protected]> * Simplify local cluster provisioning with libvirt native parallel bringup Signed-off-by: Derek Nola <[email protected]> * Don't run mixedOS on E2E Signed-off-by: Derek Nola <[email protected]> --------- Signed-off-by: Derek Nola <[email protected]>
iamsarthakk · Aug 19, 2024 · bf453e2 · bf453e2
1 parent 709f08e
commit bf453e2
Show file tree

Hide file tree

Showing 16 changed files with 166 additions and 132 deletions.
diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml
@@ -42,7 +42,7 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
     - name: Install OS Packages
-      run: sudo apt-get install -y libarchive-tools
+      run: sudo apt-get install -y libarchive-tools g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64
     # Can only upload from a single path, so we need to copy the binary to the image directory
     # Additionally, we have a rke2-runtime.tar and a rke2-images.linux-amd64.tar.zst which are the same thing
     # just compressed. We remove the rke2-runtime.tar as its not used by the install script.
@@ -51,7 +51,7 @@ jobs:
         GOCOVER=true make package-bundle
         make package-image-runtime
         cp ./bin/rke2 ./build/images/rke2
-        cp ./dist/artifacts/rke2.linux-amd64.tar.gz ./build/images/rke2.linux-amd64.tar.gz
+        cp ./dist/artifacts/rke2.*-amd64.tar.gz ./build/images/
         rm ./build/images/rke2-runtime.tar
     - name: Upload RKE2 Binary and Runtime Image
       uses: actions/upload-artifact@v4
@@ -161,8 +161,9 @@ jobs:
       run: |
         mkdir -p ./dist/artifacts
         mkdir -p ./build/images
+        mkdir -p ./bin
         mv ./artifacts/rke2-* ./build/images/
-        mv ./artifacts/rke2.linux-amd64.tar.gz ./dist/artifacts/rke2.linux-amd64.tar.gz
+        mv ./artifacts/rke2.*-amd64.tar.gz ./dist/artifacts
         sha256sum ./dist/artifacts/rke2.linux-amd64.tar.gz > ./dist/artifacts/sha256sum-amd64.txt
     - name: Run ${{ matrix.etest }} Test
       run: | 

diff --git a/scripts/build-image-runtime b/scripts/build-image-runtime
@@ -29,6 +29,11 @@ if [ "${GOARCH}" != "s390x" ] && [ "${GOARCH}" != "arm64" ]; then
       --target windows-runtime \
       --file Dockerfile.windows \
       .
+  # Only ever used in its compressed form for e2e tests
+  mkdir -p build/images
+  docker image save \
+    ${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}-windows-${GOARCH} | \
+    zstd -T0 -16 -f --long=25 --no-progress - -o build/images/${PROG}-images.windows-${GOARCH}.tar.zst    
 fi
 mkdir -p build/images
 docker image save \

diff --git a/tests/e2e/ciliumnokp/Vagrantfile b/tests/e2e/ciliumnokp/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2310', 'generic/ubuntu2310'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")

diff --git a/tests/e2e/dnscache/Vagrantfile b/tests/e2e/dnscache/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")

diff --git a/tests/e2e/dualstack/Vagrantfile b/tests/e2e/dualstack/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-0", "server-1", "server-2", "agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||

diff --git a/tests/e2e/mixedos/Vagrantfile b/tests/e2e/mixedos/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
 ["server-0", "linux-agent-0", "windows-agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
@@ -22,24 +22,19 @@ def provision(vm, role, role_num, node_num)
   load vagrant_defaults if File.exist?(vagrant_defaults)
 
   defaultOSConfigure(vm)
-
+ 
   if role.include?("windows")
     vm.provision "shell", path: "../scripts/install-bgp.ps1"
-    if !RELEASE_VERSION.empty?
-      install_type = "Version=#{RELEASE_VERSION}"
+    if RELEASE_VERSION == "skip"
+      install_type = "-ArtifactPath 'C:\tmp'" 
+    elsif !RELEASE_VERSION.empty?
+      install_type = "-Version #{RELEASE_VERSION}"
     else 
       vm.provision "shell", path: "../scripts/latest_commit.ps1", args: [GITHUB_BRANCH, "./rke2_commits.txt"]
-      install_type = "Commit=(Get-Content -TotalCount 1 ./rke2_commits.txt)"
+      install_type = "-Commit (Get-Content -TotalCount 1 ./rke2_commits.txt)"
     end
   else
-    if !RELEASE_VERSION.empty?
-      install_type = "INSTALL_RKE2_VERSION=#{RELEASE_VERSION}"
-    else 
-      # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
-      vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/tmp/rke2_commits"]
-      install_type = "INSTALL_RKE2_COMMIT=$(head\ -n\ 1\ /tmp/rke2_commits)"
-    end
-    vm.provision "shell", inline: "ping -c 2 rke2.io"
+    install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
     vm.provision "Create Calico Manifest", type: "shell", path: "../scripts/calico_manifest.sh", args: [ "#{NETWORK_PREFIX}.1" ]
   end
 

diff --git a/tests/e2e/mixedos/mixedos_test.go b/tests/e2e/mixedos/mixedos_test.go
@@ -4,7 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"strconv"
 	"strings"
 	"testing"
 
@@ -16,11 +15,10 @@ import (
 // Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "operating system for linux nodes")
 var serverCount = flag.Int("serverCount", 1, "number of server nodes")
-var linuxAgentCount = flag.Int("linuxAgentCount", 1, "number of linux agent nodes")
+var linuxAgentCount = flag.Int("linuxAgentCount", 0, "number of linux agent nodes")
 var windowsAgentCount = flag.Int("windowsAgentCount", 1, "number of windows agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")
-
-const defaultWindowsOS = "jborean93/WindowsServer2022"
+var local = flag.Bool("local", false, "deploy a locally built RKE2")
 
 func Test_E2EMixedOSValidation(t *testing.T) {
 	flag.Parse()
@@ -29,41 +27,6 @@ func Test_E2EMixedOSValidation(t *testing.T) {
 	RunSpecs(t, "Validate MixedOS Test Suite", suiteConfig, reporterConfig)
 }
 
-func createMixedCluster(nodeOS string, serverCount, linuxAgentCount, windowsAgentCount int) ([]string, []string, []string, error) {
-	serverNodeNames := []string{}
-	for i := 0; i < serverCount; i++ {
-		serverNodeNames = append(serverNodeNames, "server-"+strconv.Itoa(i))
-	}
-	linuxAgentNames := []string{}
-	for i := 0; i < linuxAgentCount; i++ {
-		linuxAgentNames = append(linuxAgentNames, "linux-agent-"+strconv.Itoa(i))
-	}
-	windowsAgentNames := []string{}
-	for i := 0; i < linuxAgentCount; i++ {
-		windowsAgentNames = append(windowsAgentNames, "windows-agent-"+strconv.Itoa(i))
-	}
-	nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(linuxAgentNames, " ") + " " + strings.Join(windowsAgentNames, " ")
-	nodeRoles = strings.TrimSpace(nodeRoles)
-	nodeBoxes := strings.Repeat(nodeOS+" ", serverCount+linuxAgentCount)
-	nodeBoxes += strings.Repeat(defaultWindowsOS+" ", windowsAgentCount)
-	nodeBoxes = strings.TrimSpace(nodeBoxes)
-
-	var testOptions string
-	for _, env := range os.Environ() {
-		if strings.HasPrefix(env, "E2E_") {
-			testOptions += " " + env
-		}
-	}
-
-	cmd := fmt.Sprintf("NODE_ROLES=\"%s\" NODE_BOXES=\"%s\" %s vagrant up &> vagrant.log", nodeRoles, nodeBoxes, testOptions)
-	fmt.Println(cmd)
-	if _, err := e2e.RunCommand(cmd); err != nil {
-		fmt.Println("Error Creating Cluster", err)
-		return nil, nil, nil, err
-	}
-	return serverNodeNames, linuxAgentNames, windowsAgentNames, nil
-}
-
 var (
 	kubeConfigFile    string
 	serverNodeNames   []string
@@ -76,7 +39,11 @@ var _ = Describe("Verify Basic Cluster Creation", Ordered, func() {
 
 	It("Starts up with no issues", func() {
 		var err error
-		serverNodeNames, linuxAgentNames, windowsAgentNames, err = createMixedCluster(*nodeOS, *serverCount, *linuxAgentCount, *windowsAgentCount)
+		if *local {
+			serverNodeNames, linuxAgentNames, windowsAgentNames, err = e2e.CreateLocalMixedCluster(*nodeOS, *serverCount, *linuxAgentCount, *windowsAgentCount)
+		} else {
+			serverNodeNames, linuxAgentNames, windowsAgentNames, err = e2e.CreateMixedCluster(*nodeOS, *serverCount, *linuxAgentCount, *windowsAgentCount)
+		}
 		Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 		fmt.Println("CLUSTER CONFIG")
 		fmt.Println("OS:", *nodeOS)

diff --git a/tests/e2e/mixedosbgp/Vagrantfile b/tests/e2e/mixedosbgp/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
 ["server-0", "linux-agent-0", "windows-agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||

diff --git a/tests/e2e/mixedosbgp/mixedosbgp_test.go b/tests/e2e/mixedosbgp/mixedosbgp_test.go
@@ -4,7 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"strconv"
 	"strings"
 	"testing"
 
@@ -20,50 +19,13 @@ var linuxAgentCount = flag.Int("linuxAgentCount", 1, "number of linux agent node
 var windowsAgentCount = flag.Int("windowsAgentCount", 1, "number of windows agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")
 
-const defaultWindowsOS = "jborean93/WindowsServer2022"
-
 func Test_E2EMixedOSBGPValidation(t *testing.T) {
 	flag.Parse()
 	RegisterFailHandler(Fail)
 	suiteConfig, reporterConfig := GinkgoConfiguration()
 	RunSpecs(t, "Validate MixedOS BGP Test Suite", suiteConfig, reporterConfig)
 }
 
-func createMixedCluster(nodeOS string, serverCount, linuxAgentCount, windowsAgentCount int) ([]string, []string, []string, error) {
-	serverNodeNames := []string{}
-	for i := 0; i < serverCount; i++ {
-		serverNodeNames = append(serverNodeNames, "server-"+strconv.Itoa(i))
-	}
-	linuxAgentNames := []string{}
-	for i := 0; i < linuxAgentCount; i++ {
-		linuxAgentNames = append(linuxAgentNames, "linux-agent-"+strconv.Itoa(i))
-	}
-	windowsAgentNames := []string{}
-	for i := 0; i < linuxAgentCount; i++ {
-		windowsAgentNames = append(windowsAgentNames, "windows-agent-"+strconv.Itoa(i))
-	}
-	nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(linuxAgentNames, " ") + " " + strings.Join(windowsAgentNames, " ")
-	nodeRoles = strings.TrimSpace(nodeRoles)
-	nodeBoxes := strings.Repeat(nodeOS+" ", serverCount+linuxAgentCount)
-	nodeBoxes += strings.Repeat(defaultWindowsOS+" ", windowsAgentCount)
-	nodeBoxes = strings.TrimSpace(nodeBoxes)
-
-	var testOptions string
-	for _, env := range os.Environ() {
-		if strings.HasPrefix(env, "E2E_") {
-			testOptions += " " + env
-		}
-	}
-
-	cmd := fmt.Sprintf("NODE_ROLES=\"%s\" NODE_BOXES=\"%s\" %s vagrant up &> vagrant.log", nodeRoles, nodeBoxes, testOptions)
-	fmt.Println(cmd)
-	if _, err := e2e.RunCommand(cmd); err != nil {
-		fmt.Println("Error Creating Cluster", err)
-		return nil, nil, nil, err
-	}
-	return serverNodeNames, linuxAgentNames, windowsAgentNames, nil
-}
-
 var (
 	kubeConfigFile    string
 	serverNodeNames   []string
@@ -76,7 +38,7 @@ var _ = Describe("Verify Basic Cluster Creation", Ordered, func() {
 
 	It("Starts up with no issues", func() {
 		var err error
-		serverNodeNames, linuxAgentNames, windowsAgentNames, err = createMixedCluster(*nodeOS, *serverCount, *linuxAgentCount, *windowsAgentCount)
+		serverNodeNames, linuxAgentNames, windowsAgentNames, err = e2e.CreateMixedCluster(*nodeOS, *serverCount, *linuxAgentCount, *windowsAgentCount)
 		Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 		fmt.Println("CLUSTER CONFIG")
 		fmt.Println("OS:", *nodeOS)
@@ -91,7 +53,7 @@ var _ = Describe("Verify Basic Cluster Creation", Ordered, func() {
 		Eventually(func(g Gomega) {
 			nodes, err := e2e.ParseNodes(kubeConfigFile, false)
 			g.Expect(err).NotTo(HaveOccurred())
-			g.Expect(len(serverNodeNames)+len(linuxAgentNames)+len(windowsAgentNames)).Should(Equal(len(nodes)))
+			g.Expect(len(serverNodeNames) + len(linuxAgentNames) + len(windowsAgentNames)).Should(Equal(len(nodes)))
 			for _, node := range nodes {
 				g.Expect(node.Status).Should(Equal("Ready"))
 			}
@@ -160,7 +122,6 @@ var _ = Describe("Verify Basic Cluster Creation", Ordered, func() {
 		Eventually(func() (string, error) {
 			return e2e.RunCommand(cmd)
 		}, "20s", "3s").Should(ContainSubstring("Welcome to nginx!"), "failed cmd: "+cmd)
-
 	})
 	It("Runs the mixed os sonobuoy plugin", func() {
 		cmd := "sonobuoy run --kubeconfig=/etc/rancher/rke2/rke2.yaml --plugin my-sonobuoy-plugins/mixed-workload-e2e/mixed-workload-e2e.yaml --aggregator-node-selector kubernetes.io/os:linux --wait"

diff --git a/tests/e2e/multus/Vagrantfile b/tests/e2e/multus/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")

diff --git a/tests/e2e/secretsencryption/Vagrantfile b/tests/e2e/secretsencryption/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||  ["server-0", "server-1", "server-2"])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||  ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")

diff --git a/tests/e2e/secretsencryption_old/Vagrantfile b/tests/e2e/secretsencryption_old/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||  ["server-0", "server-1", "server-2"])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||  ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")

diff --git a/tests/e2e/splitserver/Vagrantfile b/tests/e2e/splitserver/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-etcd-0", "server-cp-0", "server-cp-1", "agent-0"])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||