Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

E2E: Capture coverage of K3s subcommands #7686

Merged
merged 3 commits into from
Jun 8, 2023
Merged

E2E: Capture coverage of K3s subcommands #7686

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4964e4d
E2E: Use sudo for all RunCmdOnNode
dereknola Jun 7, 2023
6883200
E2E: Inject gocover ENV for k3s commands
dereknola Jun 7, 2023
29edc75
Run integration tests on E2E changes, ensures correct coverage values
dereknola Jun 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions tests/e2e/rotateca/rotateca_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,9 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() {

It("Generates New CA Certificates", func() {
cmds := []string{
"sudo mkdir -p /opt/rancher/k3s/server",
"sudo cp -r /var/lib/rancher/k3s/server/tls /opt/rancher/k3s/server",
"sudo DATA_DIR=/opt/rancher/k3s /tmp/generate-custom-ca-certs.sh",
"mkdir -p /opt/rancher/k3s/server",
"cp -r /var/lib/rancher/k3s/server/tls /opt/rancher/k3s/server",
"DATA_DIR=/opt/rancher/k3s /tmp/generate-custom-ca-certs.sh",
}
for _, cmd := range cmds {
_, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expand All @@ -89,7 +89,7 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() {
})

It("Rotates CA Certificates", func() {
cmd := "sudo k3s certificate rotate-ca --path=/opt/rancher/k3s/server"
cmd := "k3s certificate rotate-ca --path=/opt/rancher/k3s/server"
_, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
})
Expand Down
36 changes: 18 additions & 18 deletions tests/e2e/secretsencryption/secretsencryption_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
})

It("Verifies encryption start stage", func() {
cmd := "sudo k3s secrets-encrypt status"
cmd := "k3s secrets-encrypt status"
for _, nodeName := range serverNodeNames {
res, err := e2e.RunCmdOnNode(cmd, nodeName)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -96,11 +96,11 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
})

It("Prepares for Secrets-Encryption Rotation", func() {
cmd := "sudo k3s secrets-encrypt prepare"
cmd := "k3s secrets-encrypt prepare"
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expect(err).NotTo(HaveOccurred(), res)
for i, nodeName := range serverNodeNames {
cmd := "sudo k3s secrets-encrypt status"
cmd := "k3s secrets-encrypt status"
res, err := e2e.RunCmdOnNode(cmd, nodeName)
Expect(err).NotTo(HaveOccurred(), res)
Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
Expand Down Expand Up @@ -140,7 +140,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
})

It("Verifies encryption prepare stage", func() {
cmd := "sudo k3s secrets-encrypt status"
cmd := "k3s secrets-encrypt status"
for _, nodeName := range serverNodeNames {
Eventually(func(g Gomega) {
res, err := e2e.RunCmdOnNode(cmd, nodeName)
Expand All @@ -153,12 +153,12 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
})

It("Rotates the Secrets-Encryption Keys", func() {
cmd := "sudo k3s secrets-encrypt rotate"
cmd := "k3s secrets-encrypt rotate"
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expect(err).NotTo(HaveOccurred(), res)
for i, nodeName := range serverNodeNames {
Eventually(func(g Gomega) {
cmd := "sudo k3s secrets-encrypt status"
cmd := "k3s secrets-encrypt status"
res, err := e2e.RunCmdOnNode(cmd, nodeName)
g.Expect(err).NotTo(HaveOccurred(), res)
g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
Expand All @@ -176,7 +176,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
})

It("Verifies encryption rotate stage", func() {
cmd := "sudo k3s secrets-encrypt status"
cmd := "k3s secrets-encrypt status"
for _, nodeName := range serverNodeNames {
Eventually(func(g Gomega) {
res, err := e2e.RunCmdOnNode(cmd, nodeName)
Expand All @@ -189,11 +189,11 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
})

It("Reencrypts the Secrets-Encryption Keys", func() {
cmd := "sudo k3s secrets-encrypt reencrypt"
cmd := "k3s secrets-encrypt reencrypt"
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expect(err).NotTo(HaveOccurred(), res)

cmd = "sudo k3s secrets-encrypt status"
cmd = "k3s secrets-encrypt status"
Eventually(func() (string, error) {
return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
Expand All @@ -211,7 +211,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
})

It("Verifies Encryption Reencrypt Stage", func() {
cmd := "sudo k3s secrets-encrypt status"
cmd := "k3s secrets-encrypt status"
for _, nodeName := range serverNodeNames {
Eventually(func(g Gomega) {
res, err := e2e.RunCmdOnNode(cmd, nodeName)
Expand All @@ -226,15 +226,15 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {

Context("Disabling Secrets-Encryption", func() {
It("Disables encryption", func() {
cmd := "sudo k3s secrets-encrypt disable"
cmd := "k3s secrets-encrypt disable"
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expect(err).NotTo(HaveOccurred(), res)

cmd = "sudo k3s secrets-encrypt reencrypt -f --skip"
cmd = "k3s secrets-encrypt reencrypt -f --skip"
res, err = e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expect(err).NotTo(HaveOccurred(), res)

cmd = "sudo k3s secrets-encrypt status"
cmd = "k3s secrets-encrypt status"
Eventually(func() (string, error) {
return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
Expand All @@ -257,7 +257,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
})

It("Verifies encryption disabled on all nodes", func() {
cmd := "sudo k3s secrets-encrypt status"
cmd := "k3s secrets-encrypt status"
for _, nodeName := range serverNodeNames {
Eventually(func(g Gomega) {
g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Disabled"))
Expand All @@ -269,15 +269,15 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {

Context("Enabling Secrets-Encryption", func() {
It("Enables encryption", func() {
cmd := "sudo k3s secrets-encrypt enable"
cmd := "k3s secrets-encrypt enable"
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expect(err).NotTo(HaveOccurred(), res)

cmd = "sudo k3s secrets-encrypt reencrypt -f --skip"
cmd = "k3s secrets-encrypt reencrypt -f --skip"
res, err = e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expect(err).NotTo(HaveOccurred(), res)

cmd = "sudo k3s secrets-encrypt status"
cmd = "k3s secrets-encrypt status"
Eventually(func() (string, error) {
return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
Expand All @@ -288,7 +288,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
})

It("Verifies encryption enabled on all nodes", func() {
cmd := "sudo k3s secrets-encrypt status"
cmd := "k3s secrets-encrypt status"
for _, nodeName := range serverNodeNames {
Eventually(func(g Gomega) {
g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled"))
Expand Down
24 changes: 12 additions & 12 deletions tests/e2e/snapshotrestore/snapshotrestore_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,10 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {

It("Verifies Snapshot is created", func() {
Eventually(func(g Gomega) {
cmd := "sudo k3s etcd-snapshot save"
cmd := "k3s etcd-snapshot save"
_, err := e2e.RunCmdOnNode(cmd, "server-0")
g.Expect(err).NotTo(HaveOccurred())
cmd = "sudo ls /var/lib/rancher/k3s/server/db/snapshots/"
cmd = "ls /var/lib/rancher/k3s/server/db/snapshots/"
snapshotname, err = e2e.RunCmdOnNode(cmd, "server-0")
g.Expect(err).NotTo(HaveOccurred())
fmt.Println("Snapshot Name", snapshotname)
Expand All @@ -128,20 +128,20 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
Context("Cluster is reset normally", func() {
It("Resets the cluster", func() {
for _, nodeName := range serverNodeNames {
cmd := "sudo systemctl stop k3s"
cmd := "systemctl stop k3s"
Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
if nodeName != serverNodeNames[0] {
cmd = "k3s-killall.sh"
Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
}
}

cmd := "sudo k3s server --cluster-reset"
cmd := "k3s server --cluster-reset"
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
Expect(res).Should(ContainSubstring("Managed etcd cluster membership has been reset, restart without --cluster-reset flag now"))

cmd = "sudo systemctl start k3s"
cmd = "systemctl start k3s"
Expect(e2e.RunCmdOnNode(cmd, serverNodeNames[0])).Error().NotTo(HaveOccurred())
})

Expand All @@ -165,12 +165,12 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
// We must remove the db directory on the other servers before restarting k3s
// otherwise the nodes may join the old cluster
for _, nodeName := range serverNodeNames[1:] {
cmd := "sudo rm -rf /var/lib/rancher/k3s/server/db"
cmd := "rm -rf /var/lib/rancher/k3s/server/db"
Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
}

for _, nodeName := range serverNodeNames[1:] {
cmd := "sudo systemctl start k3s"
cmd := "systemctl start k3s"
Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
time.Sleep(20 * time.Second) //Stagger the restarts for etcd leaners
}
Expand Down Expand Up @@ -214,20 +214,20 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
It("Restores the snapshot", func() {
//Stop k3s on all nodes
for _, nodeName := range serverNodeNames {
cmd := "sudo systemctl stop k3s"
cmd := "systemctl stop k3s"
Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
if nodeName != serverNodeNames[0] {
cmd = "k3s-killall.sh"
Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
}
}
//Restores from snapshot on server-0
cmd := "sudo k3s server --cluster-init --cluster-reset --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/" + snapshotname
cmd := "k3s server --cluster-init --cluster-reset --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/" + snapshotname
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
Expect(res).Should(ContainSubstring("Managed etcd cluster membership has been reset, restart without --cluster-reset flag now"))

cmd = "sudo systemctl start k3s"
cmd = "systemctl start k3s"
Expect(e2e.RunCmdOnNode(cmd, serverNodeNames[0])).Error().NotTo(HaveOccurred())

})
Expand All @@ -252,12 +252,12 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
// We must remove the db directory on the other servers before restarting k3s
// otherwise the nodes may join the old cluster
for _, nodeName := range serverNodeNames[1:] {
cmd := "sudo rm -rf /var/lib/rancher/k3s/server/db"
cmd := "rm -rf /var/lib/rancher/k3s/server/db"
Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
}

for _, nodeName := range serverNodeNames[1:] {
cmd := "sudo systemctl start k3s"
cmd := "systemctl start k3s"
Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
}
})
Expand Down
16 changes: 8 additions & 8 deletions tests/e2e/startup/startup_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,13 @@ func StartK3sCluster(nodes []string, serverYAML string, agentYAML string) error
var resetCmd string
var startCmd string
if strings.Contains(node, "server") {
resetCmd = "sudo head -n 3 /etc/rancher/k3s/config.yaml > /tmp/config.yaml && sudo mv /tmp/config.yaml /etc/rancher/k3s/config.yaml"
yamlCmd = fmt.Sprintf("sudo echo '%s' >> /etc/rancher/k3s/config.yaml", serverYAML)
startCmd = "sudo systemctl start k3s"
resetCmd = "head -n 3 /etc/rancher/k3s/config.yaml > /tmp/config.yaml && sudo mv /tmp/config.yaml /etc/rancher/k3s/config.yaml"
yamlCmd = fmt.Sprintf("echo '%s' >> /etc/rancher/k3s/config.yaml", serverYAML)
startCmd = "systemctl start k3s"
} else {
resetCmd = "sudo head -n 4 /etc/rancher/k3s/config.yaml > /tmp/config.yaml && sudo mv /tmp/config.yaml /etc/rancher/k3s/config.yaml"
yamlCmd = fmt.Sprintf("sudo echo '%s' >> /etc/rancher/k3s/config.yaml", agentYAML)
startCmd = "sudo systemctl start k3s-agent"
resetCmd = "head -n 4 /etc/rancher/k3s/config.yaml > /tmp/config.yaml && sudo mv /tmp/config.yaml /etc/rancher/k3s/config.yaml"
yamlCmd = fmt.Sprintf("echo '%s' >> /etc/rancher/k3s/config.yaml", agentYAML)
startCmd = "systemctl start k3s-agent"
}
if _, err := e2e.RunCmdOnNode(resetCmd, node); err != nil {
return err
Expand All @@ -68,11 +68,11 @@ func StartK3sCluster(nodes []string, serverYAML string, agentYAML string) error

func KillK3sCluster(nodes []string) error {
for _, node := range nodes {
if _, err := e2e.RunCmdOnNode("sudo k3s-killall.sh", node); err != nil {
if _, err := e2e.RunCmdOnNode("k3s-killall.sh", node); err != nil {
return err
}
if strings.Contains(node, "server") {
if _, err := e2e.RunCmdOnNode("sudo rm -rf /var/lib/rancher/k3s/server/db", node); err != nil {
if _, err := e2e.RunCmdOnNode("rm -rf /var/lib/rancher/k3s/server/db", node); err != nil {
return err
}
}
Expand Down
32 changes: 25 additions & 7 deletions tests/e2e/testutils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ func scpK3sBinary(nodeNames []string) error {
if _, err := RunCommand(cmd); err != nil {
return fmt.Errorf("failed to scp k3s binary to %s: %v", node, err)
}
if _, err := RunCmdOnNode("sudo mv /tmp/k3s /usr/local/bin/", node); err != nil {
if _, err := RunCmdOnNode("mv /tmp/k3s /usr/local/bin/", node); err != nil {
return err
}
}
Expand Down Expand Up @@ -419,7 +419,7 @@ func ParsePods(kubeConfig string, print bool) ([]Pod, error) {
// RestartCluster restarts the k3s service on each node given
func RestartCluster(nodeNames []string) error {
for _, nodeName := range nodeNames {
cmd := "sudo systemctl restart k3s* --all"
cmd := "systemctl restart k3s* --all"
if _, err := RunCmdOnNode(cmd, nodeName); err != nil {
return err
}
Expand All @@ -430,7 +430,7 @@ func RestartCluster(nodeNames []string) error {
// StartCluster starts the k3s service on each node given
func StartCluster(nodeNames []string) error {
for _, nodeName := range nodeNames {
cmd := "sudo systemctl start k3s"
cmd := "systemctl start k3s"
if strings.Contains(nodeName, "agent") {
cmd += "-agent"
}
Expand All @@ -444,17 +444,21 @@ func StartCluster(nodeNames []string) error {
// StopCluster starts the k3s service on each node given
func StopCluster(nodeNames []string) error {
for _, nodeName := range nodeNames {
cmd := "sudo systemctl stop k3s*"
cmd := "systemctl stop k3s*"
if _, err := RunCmdOnNode(cmd, nodeName); err != nil {
return err
}
}
return nil
}

// RunCmdOnNode executes a command from within the given node
// RunCmdOnNode executes a command from within the given node as sudo
func RunCmdOnNode(cmd string, nodename string) (string, error) {
runcmd := "vagrant ssh " + nodename + " -c \"" + cmd + "\""
injectEnv := ""
if _, ok := os.LookupEnv("E2E_GOCOVER"); ok && strings.HasPrefix(cmd, "k3s") {
injectEnv = "GOCOVERDIR=/tmp/k3scov "
}
runcmd := "vagrant ssh " + nodename + " -c \"sudo " + injectEnv + cmd + "\""
out, err := RunCommand(runcmd)
if err != nil {
return out, fmt.Errorf("failed to run command: %s on node %s: %s, %v", cmd, nodename, out, err)
Expand Down Expand Up @@ -500,10 +504,24 @@ func GetCoverageReport(nodeNames []string) error {
return err
}
}
cmd := "go tool covdata textfmt -i " + strings.Join(covDirs, ",") + " -o coverage.out"
coverageFile := "coverage.out"
cmd := "go tool covdata textfmt -i " + strings.Join(covDirs, ",") + " -o " + coverageFile
if out, err := RunCommand(cmd); err != nil {
return fmt.Errorf("failed to generate coverage report: %s, %v", out, err)
}

f, err := os.ReadFile(coverageFile)
if err != nil {
return err
}
nf := strings.Replace(string(f),
"/go/src/github.com/k3s-io/k3s/cmd/server/main.go",
"github.com/k3s-io/k3s/cmd/server/main.go", -1)

if err = os.WriteFile(coverageFile, []byte(nf), os.ModePerm); err != nil {
return err
}

for _, covDir := range covDirs {
if err := os.RemoveAll(covDir); err != nil {
return err
Expand Down
8 changes: 4 additions & 4 deletions tests/e2e/validatecluster/validatecluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(errStop).NotTo(HaveOccurred(), "Cluster could not be stoped successfully")

for _, nodeName := range serverNodeNames {
cmd := "sudo k3s certificate rotate"
cmd := "k3s certificate rotate"
if _, err := e2e.RunCmdOnNode(cmd, nodeName); err != nil {
Expect(err).NotTo(HaveOccurred(), "Certificate could not be rotated successfully")
}
Expand All @@ -333,7 +333,7 @@ var _ = Describe("Verify Create", Ordered, func() {
It("Start normally", func() {
// Since we stopped all the server, we have to start 2 at once to get it back up
// If we only start one at a time, the first will hang waiting for the second to be up
_, err := e2e.RunCmdOnNode("sudo systemctl --no-block start k3s", serverNodeNames[0])
_, err := e2e.RunCmdOnNode("systemctl --no-block start k3s", serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
err = e2e.StartCluster(serverNodeNames[1:])
Expect(err).NotTo(HaveOccurred(), "Cluster could not be started successfully")
Expand All @@ -360,7 +360,7 @@ var _ = Describe("Verify Create", Ordered, func() {
}, "620s", "5s").Should(Succeed())
})
It("Validates certificates", func() {
const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls"
const grepCert = "ls -lt /var/lib/rancher/k3s/server/ | grep tls"
var expectResult = []string{
"client-ca.crt", "client-ca.key", "client-ca.nochain.crt",
"client-supervisor.crt", "client-supervisor.key",
Expand All @@ -381,7 +381,7 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(errGrep).NotTo(HaveOccurred(), "Certificate could not be created successfully")
re := regexp.MustCompile("tls-[0-9]+")
tls := re.FindAllString(grCert, -1)[0]
final := fmt.Sprintf("sudo diff -sr /var/lib/rancher/k3s/server/tls/ /var/lib/rancher/k3s/server/%s/"+
final := fmt.Sprintf("diff -sr /var/lib/rancher/k3s/server/tls/ /var/lib/rancher/k3s/server/%s/"+
"| grep -i identical | cut -f4 -d ' ' | xargs basename -a \n", tls)
finalResult, finalErr = e2e.RunCmdOnNode(final, nodeName)
Expect(finalErr).NotTo(HaveOccurred(), "Final Certification does not created successfully")
Expand Down
Loading