From 93d4db534d0bce6cda05e9376917a91106e83ec0 Mon Sep 17 00:00:00 2001 From: utam0k Date: Tue, 8 Mar 2022 05:26:46 +0000 Subject: [PATCH] test: Add a test for k3s. --- test/tests/workspace/cgroup_v2_test.go | 4 +- test/tests/workspace/k3s_test.go | 67 +++++++++++++++++--------- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/test/tests/workspace/cgroup_v2_test.go b/test/tests/workspace/cgroup_v2_test.go index 0243b72f09641a..976b81c9598713 100644 --- a/test/tests/workspace/cgroup_v2_test.go +++ b/test/tests/workspace/cgroup_v2_test.go @@ -85,11 +85,11 @@ func TestCgroupV2(t *testing.T) { }, }, &respCheckControllers) if err != nil { - t.Fatalf("cgroup v2 controllers check failed: %v\n%s\n%s", err, respCheckControllers.Stdout, respNewCgroup.Stderr) + t.Fatalf("cgroup v2 controllers check failed: %v\n%s\n%s", err, respCheckControllers.Stdout, respCheckControllers.Stderr) } if respCheckControllers.ExitCode != 0 { - t.Fatalf("cgroup v2 controllers check failed: %s\n%s", respCheckControllers.Stdout, respNewCgroup.Stderr) + t.Fatalf("cgroup v2 controllers check failed: %s\n%s", respCheckControllers.Stdout, respCheckControllers.Stderr) } expect := "cpuset cpu io memory hugetlb pids rdma" diff --git a/test/tests/workspace/k3s_test.go b/test/tests/workspace/k3s_test.go index 06b43807b813ad..838a072a53cbe6 100644 --- a/test/tests/workspace/k3s_test.go +++ b/test/tests/workspace/k3s_test.go @@ -6,8 +6,6 @@ package workspace import ( "context" - "fmt" - "path/filepath" "testing" "time" @@ -19,12 +17,6 @@ import ( "github.com/gitpod-io/gitpod/test/tests/workspace/common" ) -// curl -oL https://github.com/k3s-io/k3s/releases/download/v1.23.4%2Bk3s1/k3s && chmod +x k3s && sudo ./k3s server -d /workspace/data & - -// export KUBECONFIG=/etc/rancher/k3s/k3s.yaml -// ./k3s kubectl get node -// ./k3s kubectl get pods -A - func TestK3s(t *testing.T) { f := features.New("k3s"). WithLabel("component", "workspace"). @@ -64,44 +56,71 @@ func TestK3s(t *testing.T) { t.Skip("This test only works for cgroup v2") } - cgroupBase := "/sys/fs/cgroup/test" - var respNewCgroup agent.ExecResponse + go func() { + var respReadyForK3s agent.ExecResponse + err = rsa.Call("WorkspaceAgent.Exec", &agent.ExecRequest{ + Dir: "/", + Command: "bash", + Args: []string{ + "-c", + "curl -L https://github.com/k3s-io/k3s/releases/download/v1.23.4%2Bk3s1/k3s -o /workspace/k3s && sudo chmod +x /workspace/k3s && sudo /workspace/k3s server -d /workspace/data --flannel-backend=host-gw > /dev/null 2>&1", + }, + }, &respReadyForK3s) + }() + + // It is possible to use kubectl from k3s, but it conflicts with k3s server (Text too busy), so kubectl is installed separately. + var respGetKubectl agent.ExecResponse err = rsa.Call("WorkspaceAgent.Exec", &agent.ExecRequest{ Dir: "/", Command: "bash", Args: []string{ "-c", - fmt.Sprintf("sudo mkdir %s", cgroupBase), + "curl -L https://dl.k8s.io/release/v1.23.4/bin/linux/amd64/kubectl -o /workspace/kubectl && chmod +x /workspace/kubectl", }, - }, &respNewCgroup) + }, &respGetKubectl) if err != nil { - t.Fatalf("new cgroup create failed: %v\n%s\n%s", err, respNewCgroup.Stdout, respNewCgroup.Stderr) + t.Fatalf("failed to get kubectl: %v\n%s\n%s", err, respGetKubectl.Stdout, respGetKubectl.Stderr) } - if respNewCgroup.ExitCode != 0 { - t.Fatalf("new cgroup create failed: %s\n%s", respNewCgroup.Stdout, respNewCgroup.Stderr) + if respGetKubectl.ExitCode != 0 { + t.Fatalf("failed to get kubectl: %s\n%s", respGetKubectl.Stdout, respGetKubectl.Stderr) } - var respCheckControllers agent.ExecResponse + var respWaitForK3s agent.ExecResponse err = rsa.Call("WorkspaceAgent.Exec", &agent.ExecRequest{ Dir: "/", Command: "bash", Args: []string{ "-c", - fmt.Sprintf("cat %s", filepath.Join(cgroupBase, "cgroup.controllers")), + "timeout 30s bash -c 'while [ ! -e /etc/rancher/k3s/k3s.yaml ]; do sleep 1; done' && sudo chmod +r /etc/rancher/k3s/k3s.yaml && export KUBECONFIG=/etc/rancher/k3s/k3s.yaml && timeout 30s bash -c 'until /workspace/kubectl wait --for=condition=Ready nodes -l node-role.kubernetes.io/master=true --timeout 30s; do sleep 1; done'", }, - }, &respCheckControllers) + }, &respWaitForK3s) if err != nil { - t.Fatalf("cgroup v2 controllers check failed: %v\n%s\n%s", err, respCheckControllers.Stdout, respNewCgroup.Stderr) + t.Fatalf("failed to wait for starting k3s: %v\n%s\n%s", err, respWaitForK3s.Stdout, respWaitForK3s.Stderr) } - if respCheckControllers.ExitCode != 0 { - t.Fatalf("cgroup v2 controllers check failed: %s\n%s", respCheckControllers.Stdout, respNewCgroup.Stderr) + if respWaitForK3s.ExitCode != 0 { + t.Fatalf("failed to wait for starting k3s: %s\n%s", respWaitForK3s.Stdout, respWaitForK3s.Stderr) + } + + var respGetPods agent.ExecResponse + err = rsa.Call("WorkspaceAgent.Exec", &agent.ExecRequest{ + Dir: "/", + Command: "bash", + Env: []string{ + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", + }, + Args: []string{ + "-c", + "workspace/kubectl get nodes", + }, + }, &respGetPods) + if err != nil { + t.Fatalf("failed to get nodes: %v\n%s\n%s", err, respGetPods.Stdout, respGetPods.Stderr) } - expect := "cpuset cpu io memory hugetlb pids rdma" - if respCheckControllers.Stdout == expect { - t.Fatalf("cgroup v2 controllers check failed:\nexpect %s\nact%s", expect, respCheckControllers.Stdout) + if respGetPods.ExitCode != 0 { + t.Fatalf("failed to get nodes: %s\n%s", respGetPods.Stdout, respGetPods.Stderr) } return ctx