test: Add a test for k3s.

gitpod-io · Mar 9, 2022 · 93d4db5 · 93d4db5
1 parent 3931eae
commit 93d4db5
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 26 deletions.
diff --git a/test/tests/workspace/cgroup_v2_test.go b/test/tests/workspace/cgroup_v2_test.go
@@ -85,11 +85,11 @@ func TestCgroupV2(t *testing.T) {
 				},
 			}, &respCheckControllers)
 			if err != nil {
-				t.Fatalf("cgroup v2 controllers check failed: %v\n%s\n%s", err, respCheckControllers.Stdout, respNewCgroup.Stderr)
+				t.Fatalf("cgroup v2 controllers check failed: %v\n%s\n%s", err, respCheckControllers.Stdout, respCheckControllers.Stderr)
 			}
 
 			if respCheckControllers.ExitCode != 0 {
-				t.Fatalf("cgroup v2 controllers check failed: %s\n%s", respCheckControllers.Stdout, respNewCgroup.Stderr)
+				t.Fatalf("cgroup v2 controllers check failed: %s\n%s", respCheckControllers.Stdout, respCheckControllers.Stderr)
 			}
 
 			expect := "cpuset cpu io memory hugetlb pids rdma"

diff --git a/test/tests/workspace/k3s_test.go b/test/tests/workspace/k3s_test.go
@@ -6,8 +6,6 @@ package workspace
 
 import (
 	"context"
-	"fmt"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -19,12 +17,6 @@ import (
 	"github.com/gitpod-io/gitpod/test/tests/workspace/common"
 )
 
-// curl -oL https://github.com/k3s-io/k3s/releases/download/v1.23.4%2Bk3s1/k3s && chmod +x k3s && sudo ./k3s server -d /workspace/data &
-
-// export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
-// ./k3s kubectl get node
-// ./k3s kubectl get pods -A
-
 func TestK3s(t *testing.T) {
 	f := features.New("k3s").
 		WithLabel("component", "workspace").
@@ -64,44 +56,71 @@ func TestK3s(t *testing.T) {
 				t.Skip("This test only works for cgroup v2")
 			}
 
-			cgroupBase := "/sys/fs/cgroup/test"
-			var respNewCgroup agent.ExecResponse
+			go func() {
+				var respReadyForK3s agent.ExecResponse
+				err = rsa.Call("WorkspaceAgent.Exec", &agent.ExecRequest{
+					Dir:     "/",
+					Command: "bash",
+					Args: []string{
+						"-c",
+						"curl -L https://github.com/k3s-io/k3s/releases/download/v1.23.4%2Bk3s1/k3s -o /workspace/k3s && sudo chmod +x /workspace/k3s && sudo /workspace/k3s server -d /workspace/data --flannel-backend=host-gw > /dev/null 2>&1",
+					},
+				}, &respReadyForK3s)
+			}()
+
+			// It is possible to use kubectl from k3s, but it conflicts with k3s server (Text too busy), so kubectl is installed separately.
+			var respGetKubectl agent.ExecResponse
 			err = rsa.Call("WorkspaceAgent.Exec", &agent.ExecRequest{
 				Dir:     "/",
 				Command: "bash",
 				Args: []string{
 					"-c",
-					fmt.Sprintf("sudo mkdir %s", cgroupBase),
+					"curl -L https://dl.k8s.io/release/v1.23.4/bin/linux/amd64/kubectl -o /workspace/kubectl && chmod +x /workspace/kubectl",
 				},
-			}, &respNewCgroup)
+			}, &respGetKubectl)
 			if err != nil {
-				t.Fatalf("new cgroup create failed: %v\n%s\n%s", err, respNewCgroup.Stdout, respNewCgroup.Stderr)
+				t.Fatalf("failed to get kubectl: %v\n%s\n%s", err, respGetKubectl.Stdout, respGetKubectl.Stderr)
 			}
 
-			if respNewCgroup.ExitCode != 0 {
-				t.Fatalf("new cgroup create failed: %s\n%s", respNewCgroup.Stdout, respNewCgroup.Stderr)
+			if respGetKubectl.ExitCode != 0 {
+				t.Fatalf("failed to get kubectl: %s\n%s", respGetKubectl.Stdout, respGetKubectl.Stderr)
 			}
 
-			var respCheckControllers agent.ExecResponse
+			var respWaitForK3s agent.ExecResponse
 			err = rsa.Call("WorkspaceAgent.Exec", &agent.ExecRequest{
 				Dir:     "/",
 				Command: "bash",
 				Args: []string{
 					"-c",
-					fmt.Sprintf("cat %s", filepath.Join(cgroupBase, "cgroup.controllers")),
+					"timeout 30s bash -c 'while [ ! -e /etc/rancher/k3s/k3s.yaml ]; do sleep 1; done' && sudo chmod +r /etc/rancher/k3s/k3s.yaml && export KUBECONFIG=/etc/rancher/k3s/k3s.yaml && timeout 30s bash -c 'until /workspace/kubectl wait --for=condition=Ready nodes -l node-role.kubernetes.io/master=true --timeout 30s; do sleep 1; done'",
 				},
-			}, &respCheckControllers)
+			}, &respWaitForK3s)
 			if err != nil {
-				t.Fatalf("cgroup v2 controllers check failed: %v\n%s\n%s", err, respCheckControllers.Stdout, respNewCgroup.Stderr)
+				t.Fatalf("failed to wait for starting k3s: %v\n%s\n%s", err, respWaitForK3s.Stdout, respWaitForK3s.Stderr)
 			}
 
-			if respCheckControllers.ExitCode != 0 {
-				t.Fatalf("cgroup v2 controllers check failed: %s\n%s", respCheckControllers.Stdout, respNewCgroup.Stderr)
+			if respWaitForK3s.ExitCode != 0 {
+				t.Fatalf("failed to wait for starting k3s: %s\n%s", respWaitForK3s.Stdout, respWaitForK3s.Stderr)
+			}
+
+			var respGetPods agent.ExecResponse
+			err = rsa.Call("WorkspaceAgent.Exec", &agent.ExecRequest{
+				Dir:     "/",
+				Command: "bash",
+				Env: []string{
+					"KUBECONFIG=/etc/rancher/k3s/k3s.yaml",
+				},
+				Args: []string{
+					"-c",
+					"workspace/kubectl get nodes",
+				},
+			}, &respGetPods)
+			if err != nil {
+				t.Fatalf("failed to get nodes: %v\n%s\n%s", err, respGetPods.Stdout, respGetPods.Stderr)
 			}
 
-			expect := "cpuset cpu io memory hugetlb pids rdma"
-			if respCheckControllers.Stdout == expect {
-				t.Fatalf("cgroup v2 controllers check failed:\nexpect %s\nact%s", expect, respCheckControllers.Stdout)
+			if respGetPods.ExitCode != 0 {
+				t.Fatalf("failed to get nodes: %s\n%s", respGetPods.Stdout, respGetPods.Stderr)
 			}
 
 			return ctx