From 33726228606ca07cc790fb08e25f9939c092da98 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 1 May 2023 10:05:14 -0700 Subject: [PATCH 1/5] Add beforesuite to startup Signed-off-by: Derek Nola --- tests/e2e/startup/startup_test.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index f33fedb02605..5d4a7cebeadc 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -77,17 +77,18 @@ func KillK3sCluster(nodes []string) error { var _ = ReportAfterEach(e2e.GenReport) +var _ = BeforeSuite(func() { + var err error + if *local { + serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, 1, 1) + } else { + serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, 1, 1) + } + Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) +}) + var _ = Describe("Various Startup Configurations", Ordered, func() { Context("Verify CRI-Dockerd :", func() { - It("Stands up the nodes", func() { - var err error - if *local { - serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, 1, 1) - } else { - serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, 1, 1) - } - Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) - }) It("Starts K3s with no issues", func() { dockerYAML := "docker: true" err := StartK3sCluster(append(serverNodeNames, agentNodeNames...), dockerYAML, dockerYAML) @@ -175,7 +176,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) }) - Context("Verify disable-agent and egress-selector-mode flags", func() { + FContext("Verify disable-agent and egress-selector-mode flags", func() { It("Starts K3s with no issues", func() { disableAgentYAML := "disable-agent: true\negress-selector-mode: cluster" err := StartK3sCluster(append(serverNodeNames, agentNodeNames...), disableAgentYAML, "") @@ -237,8 +238,8 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { It("Collects logs from a pod", func() { cmd := "kubectl logs -n kube-system -l app.kubernetes.io/name=traefik -c traefik" - _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) - Expect(err).NotTo(HaveOccurred()) + res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred(), res) }) It("Kills the cluster", func() { From dfe050fc190f35d9ef39998042b6a8552fb59f43 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 1 May 2023 10:05:56 -0700 Subject: [PATCH 2/5] Reduce timeouts for startup Signed-off-by: Derek Nola --- .drone.yml | 1 + tests/e2e/startup/startup_test.go | 22 ++++++++++++---------- tests/e2e/testutils.go | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.drone.yml b/.drone.yml index 73ef977f02db..1395d72b2416 100644 --- a/.drone.yml +++ b/.drone.yml @@ -752,6 +752,7 @@ steps: - | if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then cd ../upgradecluster + vagrant destroy -f E2E_RELEASE_CHANNEL="latest" go test -v -timeout=45m ./upgradecluster_test.go -ci -local fi - docker stop registry && docker rm registry diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index 5d4a7cebeadc..d247e1a21529 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -110,7 +110,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { for _, node := range nodes { g.Expect(node.Status).Should(Equal("Ready")) } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParseNodes(kubeConfigFile, true) fmt.Printf("\nFetching pods status\n") @@ -124,7 +124,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { g.Expect(pod.Status).Should(Equal("Running"), pod.Name) } } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParsePods(kubeConfigFile, true) }) It("Kills the cluster", func() { @@ -154,7 +154,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { for _, node := range nodes { g.Expect(node.Status).Should(Equal("Ready")) } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParseNodes(kubeConfigFile, true) fmt.Printf("\nFetching pods status\n") @@ -168,7 +168,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { g.Expect(pod.Status).Should(Equal("Running"), pod.Name) } } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParsePods(kubeConfigFile, true) }) It("Kills the cluster", func() { @@ -176,7 +176,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) }) - FContext("Verify disable-agent and egress-selector-mode flags", func() { + Context("Verify disable-agent and egress-selector-mode flags", func() { It("Starts K3s with no issues", func() { disableAgentYAML := "disable-agent: true\negress-selector-mode: cluster" err := StartK3sCluster(append(serverNodeNames, agentNodeNames...), disableAgentYAML, "") @@ -198,7 +198,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { for _, node := range nodes { g.Expect(node.Status).Should(Equal("Ready")) } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParseNodes(kubeConfigFile, true) fmt.Printf("\nFetching pods status\n") @@ -212,7 +212,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { g.Expect(pod.Status).Should(Equal("Running"), pod.Name) } } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParsePods(kubeConfigFile, true) }) @@ -221,7 +221,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { Eventually(func() error { _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) return err - }, "620s", "5s").Should(Succeed()) + }, "600s", "5s").Should(Succeed()) }) It("Returns node metrics", func() { @@ -238,8 +238,10 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { It("Collects logs from a pod", func() { cmd := "kubectl logs -n kube-system -l app.kubernetes.io/name=traefik -c traefik" - res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) - Expect(err).NotTo(HaveOccurred(), res) + Eventually(func() error { + _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + return err + }, "360s", "5s").Should(Succeed()) }) It("Kills the cluster", func() { diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 484b06ad7fe8..69e7865a6549 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -440,7 +440,7 @@ func RunCmdOnNode(cmd string, nodename string) (string, error) { runcmd := "vagrant ssh -c \"" + cmd + "\" " + nodename out, err := RunCommand(runcmd) if err != nil { - return out, fmt.Errorf("failed to run command %s on node %s: %v", cmd, nodename, err) + return out, fmt.Errorf("failed to run command: %s on node %s: %s, %v", cmd, nodename, out, err) } return out, nil } From f490e369360fa4950be88cf8bdd0e4e7ff6c5db1 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Wed, 3 May 2023 08:45:02 -0700 Subject: [PATCH 3/5] Fix cleanup + set kubeconfig Signed-off-by: Derek Nola --- tests/e2e/startup/startup_test.go | 14 +++++++++++--- tests/e2e/testutils.go | 13 +++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index d247e1a21529..5bcebf091947 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -71,6 +71,11 @@ func KillK3sCluster(nodes []string) error { if _, err := e2e.RunCmdOnNode("sudo k3s-killall.sh", node); err != nil { return err } + if strings.Contains(node, "server") { + if _, err := e2e.RunCmdOnNode("sudo rm -rf /var/lib/rancher/k3s/server/db", node); err != nil { + return err + } + } } return nil } @@ -100,6 +105,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { fmt.Println("Agent Nodes:", agentNodeNames) kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) + Expect(e2e.SetKubeConfig(kubeConfigFile)).To(Succeed()) }) It("Checks node and pod status", func() { @@ -144,6 +150,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { fmt.Println("Agent Nodes:", agentNodeNames) kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) + Expect(e2e.SetKubeConfig(kubeConfigFile)).To(Succeed()) }) It("Checks node and pod status", func() { @@ -188,6 +195,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { fmt.Println("Agent Nodes:", agentNodeNames) kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) + Expect(e2e.SetKubeConfig(kubeConfigFile)).To(Succeed()) }) It("Checks node and pod status", func() { @@ -219,14 +227,14 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { It("Returns pod metrics", func() { cmd := "kubectl top pod -A" Eventually(func() error { - _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + _, err := e2e.RunCommand(cmd) return err }, "600s", "5s").Should(Succeed()) }) It("Returns node metrics", func() { cmd := "kubectl top node" - _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + _, err := e2e.RunCommand(cmd) Expect(err).NotTo(HaveOccurred()) }) @@ -239,7 +247,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { It("Collects logs from a pod", func() { cmd := "kubectl logs -n kube-system -l app.kubernetes.io/name=traefik -c traefik" Eventually(func() error { - _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + _, err := e2e.RunCommand(cmd) return err }, "360s", "5s").Should(Succeed()) }) diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 69e7865a6549..14ff9ce31bd0 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -306,6 +306,12 @@ func GenKubeConfigFile(serverName string) (string, error) { return kubeConfigFile, nil } +// SetKubeConfig sets the E2E_KUBECONFIG environment variable +// for use with RunCommand function +func SetKubeConfig(kubeconfig string) error { + return os.Setenv("E2E_KUBECONFIG", kubeconfig) +} + func GenReport(specReport ginkgo.SpecReport) { state := struct { State string `json:"state"` @@ -437,7 +443,7 @@ func RestartClusterAgent(nodeNames []string) error { // RunCmdOnNode executes a command from within the given node func RunCmdOnNode(cmd string, nodename string) (string, error) { - runcmd := "vagrant ssh -c \"" + cmd + "\" " + nodename + runcmd := "vagrant ssh " + nodename + " -c \"" + cmd + "\"" out, err := RunCommand(runcmd) if err != nil { return out, fmt.Errorf("failed to run command: %s on node %s: %s, %v", cmd, nodename, out, err) @@ -445,8 +451,11 @@ func RunCmdOnNode(cmd string, nodename string) (string, error) { return out, nil } -// RunCommand executes a command on the host func RunCommand(cmd string) (string, error) { + // we don't use c.env because we embed the command inside a bash shell + if k, b := os.LookupEnv("E2E_KUBECONFIG"); b { + cmd = "KUBECONFIG=" + k + " " + cmd + } c := exec.Command("bash", "-c", cmd) out, err := c.CombinedOutput() return string(out), err From 297783b7d08ba7837192eb7f7c4145b984e5cf06 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 4 May 2023 12:50:54 -0700 Subject: [PATCH 4/5] Revert eventually Signed-off-by: Derek Nola --- tests/e2e/startup/startup_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index 5bcebf091947..0b370964bdaf 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -246,10 +246,8 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { It("Collects logs from a pod", func() { cmd := "kubectl logs -n kube-system -l app.kubernetes.io/name=traefik -c traefik" - Eventually(func() error { - _, err := e2e.RunCommand(cmd) - return err - }, "360s", "5s").Should(Succeed()) + _, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred()) }) It("Kills the cluster", func() { From 5c24d6d7bc9718a275c045b78592e69453942780 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 4 May 2023 15:46:13 -0700 Subject: [PATCH 5/5] Simplify utility function Signed-off-by: Derek Nola --- tests/e2e/startup/startup_test.go | 3 --- tests/e2e/testutils.go | 16 ++++++---------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index 0b370964bdaf..6c3858f19577 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -105,7 +105,6 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { fmt.Println("Agent Nodes:", agentNodeNames) kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) - Expect(e2e.SetKubeConfig(kubeConfigFile)).To(Succeed()) }) It("Checks node and pod status", func() { @@ -150,7 +149,6 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { fmt.Println("Agent Nodes:", agentNodeNames) kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) - Expect(e2e.SetKubeConfig(kubeConfigFile)).To(Succeed()) }) It("Checks node and pod status", func() { @@ -195,7 +193,6 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { fmt.Println("Agent Nodes:", agentNodeNames) kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) - Expect(e2e.SetKubeConfig(kubeConfigFile)).To(Succeed()) }) It("Checks node and pod status", func() { diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 14ff9ce31bd0..16325bac8367 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -303,15 +303,12 @@ func GenKubeConfigFile(serverName string) (string, error) { if err := os.WriteFile(kubeConfigFile, []byte(kubeConfig), 0644); err != nil { return "", err } + if err := os.Setenv("E2E_KUBECONFIG", kubeConfigFile); err != nil { + return "", err + } return kubeConfigFile, nil } -// SetKubeConfig sets the E2E_KUBECONFIG environment variable -// for use with RunCommand function -func SetKubeConfig(kubeconfig string) error { - return os.Setenv("E2E_KUBECONFIG", kubeconfig) -} - func GenReport(specReport ginkgo.SpecReport) { state := struct { State string `json:"state"` @@ -452,11 +449,10 @@ func RunCmdOnNode(cmd string, nodename string) (string, error) { } func RunCommand(cmd string) (string, error) { - // we don't use c.env because we embed the command inside a bash shell - if k, b := os.LookupEnv("E2E_KUBECONFIG"); b { - cmd = "KUBECONFIG=" + k + " " + cmd - } c := exec.Command("bash", "-c", cmd) + if kc, ok := os.LookupEnv("E2E_KUBECONFIG"); ok { + c.Env = append(os.Environ(), "KUBECONFIG="+kc) + } out, err := c.CombinedOutput() return string(out), err }