Fix cleanup + set kubeconfig

Signed-off-by: Derek Nola <[email protected]>

tests/e2e/startup/startup_test.go

@@ -71,6 +71,11 @@ func KillK3sCluster(nodes []string) error {
 		if _, err := e2e.RunCmdOnNode("sudo k3s-killall.sh", node); err != nil {
 			return err
 		}
+		if strings.Contains(node, "server") {
+			if _, err := e2e.RunCmdOnNode("sudo rm -rf /var/lib/rancher/k3s/server/db", node); err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }
@@ -100,6 +105,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
 			fmt.Println("Agent Nodes:", agentNodeNames)
 			kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
+			Expect(e2e.SetKubeConfig(kubeConfigFile)).To(Succeed())
 		})
 
 		It("Checks node and pod status", func() {
@@ -144,6 +150,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
 			fmt.Println("Agent Nodes:", agentNodeNames)
 			kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
+			Expect(e2e.SetKubeConfig(kubeConfigFile)).To(Succeed())
 		})
 
 		It("Checks node and pod status", func() {
@@ -188,6 +195,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
 			fmt.Println("Agent Nodes:", agentNodeNames)
 			kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
+			Expect(e2e.SetKubeConfig(kubeConfigFile)).To(Succeed())
 		})
 
 		It("Checks node and pod status", func() {
@@ -219,14 +227,14 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
 		It("Returns pod metrics", func() {
 			cmd := "kubectl top pod -A"
 			Eventually(func() error {
-				_, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
+				_, err := e2e.RunCommand(cmd)
 				return err
 			}, "600s", "5s").Should(Succeed())
 		})
 
 		It("Returns node metrics", func() {
 			cmd := "kubectl top node"
-			_, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
+			_, err := e2e.RunCommand(cmd)
 			Expect(err).NotTo(HaveOccurred())
 		})
 
@@ -239,7 +247,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
 		It("Collects logs from a pod", func() {
 			cmd := "kubectl logs -n kube-system -l app.kubernetes.io/name=traefik -c traefik"
 			Eventually(func() error {
-				_, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
+				_, err := e2e.RunCommand(cmd)
 				return err
 			}, "360s", "5s").Should(Succeed())
 		})

tests/e2e/testutils.go

@@ -306,6 +306,12 @@ func GenKubeConfigFile(serverName string) (string, error) {
 	return kubeConfigFile, nil
 }
 
+// SetKubeConfig sets the E2E_KUBECONFIG environment variable
+// for use with RunCommand function
+func SetKubeConfig(kubeconfig string) error {
+	return os.Setenv("E2E_KUBECONFIG", kubeconfig)
+}
+
 func GenReport(specReport ginkgo.SpecReport) {
 	state := struct {
 		State string        `json:"state"`
@@ -437,16 +443,19 @@ func RestartClusterAgent(nodeNames []string) error {
 
 // RunCmdOnNode executes a command from within the given node
 func RunCmdOnNode(cmd string, nodename string) (string, error) {
-	runcmd := "vagrant ssh -c \"" + cmd + "\" " + nodename
+	runcmd := "vagrant ssh " + nodename + " -c \"" + cmd + "\""
 	out, err := RunCommand(runcmd)
 	if err != nil {
 		return out, fmt.Errorf("failed to run command: %s on node %s: %s, %v", cmd, nodename, out, err)
 	}
 	return out, nil
 }
 
-// RunCommand executes a command on the host
 func RunCommand(cmd string) (string, error) {
+	// we don't use c.env because we embed the command inside a bash shell
+	if k, b := os.LookupEnv("E2E_KUBECONFIG"); b {
+		cmd = "KUBECONFIG=" + k + " " + cmd
+	}
 	c := exec.Command("bash", "-c", cmd)
 	out, err := c.CombinedOutput()
 	return string(out), err