From 0bccd71ed099e57724c1d87d52e8e430ff747bab Mon Sep 17 00:00:00 2001 From: est-suse Date: Sun, 2 Apr 2023 19:35:49 -0600 Subject: [PATCH 1/7] Create test for certificate rotation, delete restartAgent Function and add functions for start and stop server Signed-off-by: est-suse --- tests/e2e/certrotation/certrotation_test.go | 162 ++++++++++++++++++ tests/e2e/certrotation/vagrantfile | 88 ++++++++++ .../multiclustercidr/multiclustercidr_test.go | 4 +- tests/e2e/testutils.go | 28 ++- 4 files changed, 277 insertions(+), 5 deletions(-) create mode 100644 tests/e2e/certrotation/certrotation_test.go create mode 100644 tests/e2e/certrotation/vagrantfile diff --git a/tests/e2e/certrotation/certrotation_test.go b/tests/e2e/certrotation/certrotation_test.go new file mode 100644 index 000000000000..c00adef2b40c --- /dev/null +++ b/tests/e2e/certrotation/certrotation_test.go @@ -0,0 +1,162 @@ +package secretsencryption + +import ( + "flag" + "fmt" + "os" + "regexp" + "strings" + "testing" + + "github.com/k3s-io/k3s/tests/e2e" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") +var serverCount = flag.Int("serverCount", 3, "number of server nodes") +var agentCount = flag.Int("agentCount", 1, "number of agent nodes") +var ci = flag.Bool("ci", false, "running on CI") + +// Environment Variables Info: +// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master + +func Test_E2ECustomCARotation(t *testing.T) { + RegisterFailHandler(Fail) + flag.Parse() + suiteConfig, reporterConfig := GinkgoConfiguration() + RunSpecs(t, "Custom Certificate Rotation Test Suite", suiteConfig, reporterConfig) +} + +var ( + kubeConfigFile string + agentNodeNames []string + serverNodeNames []string +) + +var _ = ReportAfterEach(e2e.GenReport) + +var _ = Describe("Verify Custom CA Rotation", Ordered, func() { + Context("Custom CA is rotated:", func() { + It("Starts up with no issues", func() { + var err error + serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) + Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) + fmt.Println("CLUSTER CONFIG") + fmt.Println("OS:", *nodeOS) + fmt.Println("Server Nodes:", serverNodeNames) + fmt.Println("Agent Nodes:", agentNodeNames) + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Verifies Certificate Rotation", func() { + const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls" + //const expectResult = "client-ca.crt,client-ca.key,client-ca.nochain.crt,dynamic-cert.json,peer-ca.crt,peer-ca.key,server-ca.crt,server-ca.key,request-header-ca.crt,request-header-ca.key,server-ca.crt,server-ca.key,server-ca.nochain.crt,service.current.key,service.key,apiserver-loopback-client__.crt,apiserver-loopback-client__.key," + + var expectResult = []string{"client-ca.crt", "client-ca.key", "client-ca.nochain.crt", "client-ca.pem", + "dynamic-cert.json", "peer-ca.crt", "peer-ca.key", "peer-ca.pem", "server-ca.crt", "server-ca.key", "server-ca.pem", + "intermediate-ca.crt", "intermediate-ca.key", + "intermediate-ca.pem", "request-header-ca.crt", "request-header-ca.key", + "request-header-ca.pem", "root-ca.crt", "root-ca.key", "root-ca.pem", "server-ca.crt", "server-ca.key", + "server-ca.nochain.crt", "server-ca.pem", "service.current.key", "service.key", "apiserver-loopback-client__.crt", + "apiserver-loopback-client__.key", "", + } + + var finalResult string + var finalErr error + errStop := e2e.StopCluster(serverNodeNames) + Expect(errStop).NotTo(HaveOccurred(), "Server not stop correctly") + errRotate := e2e.RotateCertificate(serverNodeNames) + Expect(errRotate).NotTo(HaveOccurred(), "Certificate not rotate correctly") + errStart := e2e.StartCluster(serverNodeNames) + Expect(errStart).NotTo(HaveOccurred(), "Server not start correctly") + + for _, nodeName := range serverNodeNames { + grCert, errGrep := e2e.RunCmdOnNode(grepCert, nodeName) + Expect(errGrep).NotTo(HaveOccurred(), "Certificate not created correctly") + re := regexp.MustCompile("tls-[0-9]+") + tls := re.FindAllString(grCert, -1)[0] + final := fmt.Sprintf("sudo diff -sr /var/lib/rancher/k3s/server/tls/ /var/lib/rancher/k3s/server/%s/ | grep -i identical | cut -f4 -d ' ' | xargs basename -a \n", tls) + finalResult, finalErr = e2e.RunCmdOnNode(final, nodeName) + Expect(finalErr).NotTo(HaveOccurred(), "Final Certification does not created correctly") + } + if len(agentNodeNames) > 0 { + errRestartAgent := e2e.RestartCluster(agentNodeNames) + Expect(errRestartAgent).NotTo(HaveOccurred(), "Restart Agent not happened correctly") + } + Eventually(func(g Gomega) { + finalCert := strings.Replace(finalResult, "\n", ",", -1) + finalCertArray := strings.Split(finalCert, ",") + fmt.Println("FINAL ESTEBAN ARRAY", finalCertArray) + fmt.Println("FINAL ESTEBAN expectResult", expectResult) + Expect((finalCertArray)).Should((Equal(expectResult)), "Final certification does not match the expected results") + }, "620s", "5s").Should(Succeed()) + + }) + + /*It("Generates New CA Certificates", func() { + cmds := []string{ + "sudo mkdir -p /opt/rancher/k3s/server", + "sudo cp -r /var/lib/rancher/k3s/server/tls /opt/rancher/k3s/server", + "sudo DATA_DIR=/opt/rancher/k3s /tmp/generate-custom-ca-certs.sh", + } + for _, cmd := range cmds { + _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + } + }) + + It("Rotates CA Certificates", func() { + cmd := "sudo k3s certificate rotate-ca --path=/opt/rancher/k3s/server" + _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Restarts K3s servers", func() { + Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) + }) + + It("Restarts K3s agents", func() { + Expect(e2e.RestartCluster(agentNodeNames)).To(Succeed()) + }) + + It("Checks node and pod status", func() { + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + }, "420s", "5s").Should(Succeed()) + + Eventually(func(g Gomega) { + pods, err := e2e.ParsePods(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) + } else { + g.Expect(pod.Status).Should(Equal("Running"), pod.Name) + } + } + }, "420s", "5s").Should(Succeed()) + _, _ = e2e.ParseNodes(kubeConfigFile, true) + })*/ + }) +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + +var _ = AfterSuite(func() { + if failed && !*ci { + fmt.Println("FAILED!") + } else { + Expect(e2e.DestroyCluster()).To(Succeed()) + Expect(os.Remove(kubeConfigFile)).To(Succeed()) + } +}) diff --git a/tests/e2e/certrotation/vagrantfile b/tests/e2e/certrotation/vagrantfile new file mode 100644 index 000000000000..bb50599ca58f --- /dev/null +++ b/tests/e2e/certrotation/vagrantfile @@ -0,0 +1,88 @@ +ENV['VAGRANT_NO_PARALLEL'] = 'no' +NODE_ROLES = (ENV['E2E_NODE_ROLES'] || + ["server-0", "server-1", "server-2", "agent-0"]) +NODE_BOXES = (ENV['E2E_NODE_BOXES'] || + ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204']) +GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") +RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") +NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i +NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i +# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks +NETWORK_PREFIX = "10.10.10" +install_type = "" + +def provision(vm, role, role_num, node_num) + vm.box = NODE_BOXES[node_num] + vm.hostname = role + # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 + vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0" + + vagrant_defaults = '../vagrantdefaults.rb' + load vagrant_defaults if File.exists?(vagrant_defaults) + + defaultOSConfigure(vm) + install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) + + vm.provision "shell", inline: "ping -c 2 k3s.io" + + if role.include?("server") && role_num == 0 + vm.provision 'file' do |scp| + scp.source = '../../../contrib/util/generate-custom-ca-certs.sh' + scp.destination = '/tmp/generate-custom-ca-certs.sh' + end + vm.provision 'custom-ca', type: 'shell', run: 'once' do |script| + script.inline = 'bash /tmp/generate-custom-ca-certs.sh' + script.env = {'PRODUCT' => 'vagrant-e2e-test', 'DATA_DIR' => '/var/lib/rancher/k3s'} + end + vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| + k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1] + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + end + elsif role.include?("server") && role_num != 0 + vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| + k3s.args = %W[server --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + end + elsif role.include?("agent") + vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| + k3s.args = %W[agent --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] + k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 + end + end + if vm.box.to_s.include?("microos") + vm.provision 'k3s-reload', type: 'reload', run: 'once' + end +end + +Vagrant.configure("2") do |config| + config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"] + # Default provider is libvirt, virtualbox is only provided as a backup + config.vm.provider "libvirt" do |v| + v.cpus = NODE_CPUS + v.memory = NODE_MEMORY + end + config.vm.provider "virtualbox" do |v| + v.cpus = NODE_CPUS + v.memory = NODE_MEMORY + end + + if NODE_ROLES.kind_of?(String) + NODE_ROLES = NODE_ROLES.split(" ", -1) + end + if NODE_BOXES.kind_of?(String) + NODE_BOXES = NODE_BOXES.split(" ", -1) + end + + # Must iterate on the index, vagrant does not understand iterating + # over the node roles themselves + NODE_ROLES.length.times do |i| + name = NODE_ROLES[i] + role_num = name.split("-", -1).pop.to_i + config.vm.define name do |node| + provision(node.vm, name, role_num, i) + end + end +end diff --git a/tests/e2e/multiclustercidr/multiclustercidr_test.go b/tests/e2e/multiclustercidr/multiclustercidr_test.go index 09b95fabc70b..bd5b3e074460 100644 --- a/tests/e2e/multiclustercidr/multiclustercidr_test.go +++ b/tests/e2e/multiclustercidr/multiclustercidr_test.go @@ -106,7 +106,7 @@ var _ = Describe("Verify MultiClusterCIDR Configuration", Ordered, func() { It("Restart agent-0", func() { agents := []string{"agent-0"} - err := e2e.RestartClusterAgent(agents) + err := e2e.RestartCluster(agents) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) }) @@ -223,7 +223,7 @@ var _ = Describe("Verify MultiClusterCIDR Configuration", Ordered, func() { It("Delete and restart agent-0", func() { agents := []string{"agent-0"} - err := e2e.RestartClusterAgent(agents) + err := e2e.RestartCluster(agents) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) }) diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 484b06ad7fe8..4e1641534cad 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -424,10 +424,32 @@ func RestartCluster(nodeNames []string) error { return nil } -// RestartCluster restarts the k3s service on each node given -func RestartClusterAgent(nodeNames []string) error { +// StartCluster starts the k3s service on each node given +func StartCluster(nodeNames []string) error { + for _, nodeName := range nodeNames { + cmd := "sudo systemctl start k3s*" + if _, err := RunCmdOnNode(cmd, nodeName); err != nil { + return err + } + } + return nil +} + +// StopCluster starts the k3s service on each node given +func StopCluster(nodeNames []string) error { + for _, nodeName := range nodeNames { + cmd := "sudo systemctl stop k3s*" + if _, err := RunCmdOnNode(cmd, nodeName); err != nil { + return err + } + } + return nil +} + +// RotateCertificate rotate the Certificate on each node given +func RotateCertificate(nodeNames []string) error { for _, nodeName := range nodeNames { - cmd := "sudo systemctl restart k3s-agent" + cmd := "sudo k3s --debug certificate rotate" if _, err := RunCmdOnNode(cmd, nodeName); err != nil { return err } From 55168fb07626cc0a0453133a757e5d556c08b545 Mon Sep 17 00:00:00 2001 From: est-suse Date: Sun, 2 Apr 2023 19:43:01 -0600 Subject: [PATCH 2/7] Delete Additional comments in the Test file Signed-off-by: est-suse --- tests/e2e/certrotation/certrotation_test.go | 50 --------------------- 1 file changed, 50 deletions(-) diff --git a/tests/e2e/certrotation/certrotation_test.go b/tests/e2e/certrotation/certrotation_test.go index c00adef2b40c..4a64fd8ec038 100644 --- a/tests/e2e/certrotation/certrotation_test.go +++ b/tests/e2e/certrotation/certrotation_test.go @@ -53,8 +53,6 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() { It("Verifies Certificate Rotation", func() { const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls" - //const expectResult = "client-ca.crt,client-ca.key,client-ca.nochain.crt,dynamic-cert.json,peer-ca.crt,peer-ca.key,server-ca.crt,server-ca.key,request-header-ca.crt,request-header-ca.key,server-ca.crt,server-ca.key,server-ca.nochain.crt,service.current.key,service.key,apiserver-loopback-client__.crt,apiserver-loopback-client__.key," - var expectResult = []string{"client-ca.crt", "client-ca.key", "client-ca.nochain.crt", "client-ca.pem", "dynamic-cert.json", "peer-ca.crt", "peer-ca.key", "peer-ca.pem", "server-ca.crt", "server-ca.key", "server-ca.pem", "intermediate-ca.crt", "intermediate-ca.key", @@ -96,54 +94,6 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() { }) - /*It("Generates New CA Certificates", func() { - cmds := []string{ - "sudo mkdir -p /opt/rancher/k3s/server", - "sudo cp -r /var/lib/rancher/k3s/server/tls /opt/rancher/k3s/server", - "sudo DATA_DIR=/opt/rancher/k3s /tmp/generate-custom-ca-certs.sh", - } - for _, cmd := range cmds { - _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) - Expect(err).NotTo(HaveOccurred()) - } - }) - - It("Rotates CA Certificates", func() { - cmd := "sudo k3s certificate rotate-ca --path=/opt/rancher/k3s/server" - _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Restarts K3s servers", func() { - Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) - }) - - It("Restarts K3s agents", func() { - Expect(e2e.RestartCluster(agentNodeNames)).To(Succeed()) - }) - - It("Checks node and pod status", func() { - Eventually(func(g Gomega) { - nodes, err := e2e.ParseNodes(kubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready")) - } - }, "420s", "5s").Should(Succeed()) - - Eventually(func(g Gomega) { - pods, err := e2e.ParsePods(kubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - for _, pod := range pods { - if strings.Contains(pod.Name, "helm-install") { - g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) - } else { - g.Expect(pod.Status).Should(Equal("Running"), pod.Name) - } - } - }, "420s", "5s").Should(Succeed()) - _, _ = e2e.ParseNodes(kubeConfigFile, true) - })*/ }) }) From f3ebe2f6ef152fa969c702cc1e56f43e87d3c954 Mon Sep 17 00:00:00 2001 From: est-suse Date: Mon, 3 Apr 2023 12:10:58 -0600 Subject: [PATCH 3/7] Correct different issues, formats, move certifcate rotation func Signed-off-by: est-suse --- tests/e2e/certrotation/certrotation_test.go | 40 +++++++++++++++------ tests/e2e/testutils.go | 11 ------ 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/tests/e2e/certrotation/certrotation_test.go b/tests/e2e/certrotation/certrotation_test.go index 4a64fd8ec038..209bbe59538e 100644 --- a/tests/e2e/certrotation/certrotation_test.go +++ b/tests/e2e/certrotation/certrotation_test.go @@ -37,6 +37,17 @@ var ( var _ = ReportAfterEach(e2e.GenReport) +// RotateCertificate rotate the Certificate on each node given +func RotateCertificate(nodeNames []string) error { + for _, nodeName := range nodeNames { + cmd := "sudo k3s --debug certificate rotate" + if _, err := e2e.RunCmdOnNode(cmd, nodeName); err != nil { + return err + } + } + return nil +} + var _ = Describe("Verify Custom CA Rotation", Ordered, func() { Context("Custom CA is rotated:", func() { It("Starts up with no issues", func() { @@ -53,20 +64,29 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() { It("Verifies Certificate Rotation", func() { const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls" - var expectResult = []string{"client-ca.crt", "client-ca.key", "client-ca.nochain.crt", "client-ca.pem", - "dynamic-cert.json", "peer-ca.crt", "peer-ca.key", "peer-ca.pem", "server-ca.crt", "server-ca.key", "server-ca.pem", - "intermediate-ca.crt", "intermediate-ca.key", - "intermediate-ca.pem", "request-header-ca.crt", "request-header-ca.key", - "request-header-ca.pem", "root-ca.crt", "root-ca.key", "root-ca.pem", "server-ca.crt", "server-ca.key", - "server-ca.nochain.crt", "server-ca.pem", "service.current.key", "service.key", "apiserver-loopback-client__.crt", - "apiserver-loopback-client__.key", "", + expectedResult := []string{ + "client-ca.crt", "client-ca.key", + "client-ca.nochain.crt", "client-ca.pem", + "dynamic-cert.json", "peer-ca.crt", + "peer-ca.key", "peer-ca.pem", + "server-ca.crt", "server-ca.key", + "server-ca.pem", "intermediate-ca.crt", + "intermediate-ca.key", "intermediate-ca.pem", + "request-header-ca.crt", "request-header-ca.key", + "request-header-ca.pem", "root-ca.crt", + "root-ca.key", "root-ca.pem", + "server-ca.crt", "server-ca.key", + "server-ca.nochain.crt", "server-ca.pem", + "service.current.key", "service.key", + "apiserver-loopback-client__.crt", "apiserver-loopback-client__.key", + "", } var finalResult string var finalErr error errStop := e2e.StopCluster(serverNodeNames) Expect(errStop).NotTo(HaveOccurred(), "Server not stop correctly") - errRotate := e2e.RotateCertificate(serverNodeNames) + errRotate := RotateCertificate(serverNodeNames) Expect(errRotate).NotTo(HaveOccurred(), "Certificate not rotate correctly") errStart := e2e.StartCluster(serverNodeNames) Expect(errStart).NotTo(HaveOccurred(), "Server not start correctly") @@ -87,9 +107,7 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() { Eventually(func(g Gomega) { finalCert := strings.Replace(finalResult, "\n", ",", -1) finalCertArray := strings.Split(finalCert, ",") - fmt.Println("FINAL ESTEBAN ARRAY", finalCertArray) - fmt.Println("FINAL ESTEBAN expectResult", expectResult) - Expect((finalCertArray)).Should((Equal(expectResult)), "Final certification does not match the expected results") + Expect((finalCertArray)).Should((Equal(expectedResult)), "Final certification does not match the expected results") }, "620s", "5s").Should(Succeed()) }) diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 4e1641534cad..ab965112ece3 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -446,17 +446,6 @@ func StopCluster(nodeNames []string) error { return nil } -// RotateCertificate rotate the Certificate on each node given -func RotateCertificate(nodeNames []string) error { - for _, nodeName := range nodeNames { - cmd := "sudo k3s --debug certificate rotate" - if _, err := RunCmdOnNode(cmd, nodeName); err != nil { - return err - } - } - return nil -} - // RunCmdOnNode executes a command from within the given node func RunCmdOnNode(cmd string, nodename string) (string, error) { runcmd := "vagrant ssh -c \"" + cmd + "\" " + nodename From b3eaef908d649bb54a05b1ceb2a73484e5bd5059 Mon Sep 17 00:00:00 2001 From: est-suse Date: Wed, 5 Apr 2023 10:22:07 -0600 Subject: [PATCH 4/7] Fix limit lenght, remove Eventually Block and create a new file for RotateCertificate command Signed-off-by: est-suse --- .../rotatecertificate/rotatecertificate.go | 14 +++ .../rotatecertificate_test.go | 118 ++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 tests/e2e/rotatecertificate/rotatecertificate.go create mode 100644 tests/e2e/rotatecertificate/rotatecertificate_test.go diff --git a/tests/e2e/rotatecertificate/rotatecertificate.go b/tests/e2e/rotatecertificate/rotatecertificate.go new file mode 100644 index 000000000000..45db1217f641 --- /dev/null +++ b/tests/e2e/rotatecertificate/rotatecertificate.go @@ -0,0 +1,14 @@ +package rotatecertificate + +import "github.com/k3s-io/k3s/tests/e2e" + +// rotateCertificate rotate the Certificate on each node given +func rotateCertificate(nodeNames []string) error { + for _, nodeName := range nodeNames { + cmd := "sudo k3s --debug certificate rotate" + if _, err := e2e.RunCmdOnNode(cmd, nodeName); err != nil { + return err + } + } + return nil +} diff --git a/tests/e2e/rotatecertificate/rotatecertificate_test.go b/tests/e2e/rotatecertificate/rotatecertificate_test.go new file mode 100644 index 000000000000..b4b8fc2bd0c7 --- /dev/null +++ b/tests/e2e/rotatecertificate/rotatecertificate_test.go @@ -0,0 +1,118 @@ +package rotatecertificate + +import ( + "flag" + "fmt" + "os" + "regexp" + "strings" + "testing" + + "github.com/k3s-io/k3s/tests/e2e" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") +var serverCount = flag.Int("serverCount", 3, "number of server nodes") +var agentCount = flag.Int("agentCount", 1, "number of agent nodes") +var ci = flag.Bool("ci", false, "running on CI") + +// Environment Variables Info: +// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master + +func Test_E2ECustomCARotation(t *testing.T) { + RegisterFailHandler(Fail) + flag.Parse() + suiteConfig, reporterConfig := GinkgoConfiguration() + RunSpecs(t, "Custom Certificate Rotation Test Suite", suiteConfig, reporterConfig) +} + +var ( + kubeConfigFile string + agentNodeNames []string + serverNodeNames []string +) + +var _ = ReportAfterEach(e2e.GenReport) + +var _ = Describe("Verify Custom CA Rotation", Ordered, func() { + Context("Custom CA is rotated:", func() { + It("Starts up with no issues", func() { + var err error + serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) + Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) + fmt.Println("CLUSTER CONFIG") + fmt.Println("OS:", *nodeOS) + fmt.Println("Server Nodes:", serverNodeNames) + fmt.Println("Agent Nodes:", agentNodeNames) + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Verifies Certificate Rotation", func() { + const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls" + expectedResult := []string{ + "client-ca.crt", "client-ca.key", + "client-ca.nochain.crt", "client-ca.pem", + "dynamic-cert.json", "peer-ca.crt", + "peer-ca.key", "peer-ca.pem", + "server-ca.crt", "server-ca.key", + "server-ca.pem", "intermediate-ca.crt", + "intermediate-ca.key", "intermediate-ca.pem", + "request-header-ca.crt", "request-header-ca.key", + "request-header-ca.pem", "root-ca.crt", + "root-ca.key", "root-ca.pem", + "server-ca.crt", "server-ca.key", + "server-ca.nochain.crt", "server-ca.pem", + "service.current.key", "service.key", + "apiserver-loopback-client__.crt", "apiserver-loopback-client__.key", + "", + } + + var finalResult string + var finalErr error + errStop := e2e.StopCluster(serverNodeNames) + Expect(errStop).NotTo(HaveOccurred(), "Server not stop correctly") + errRotate := rotateCertificate(serverNodeNames) + Expect(errRotate).NotTo(HaveOccurred(), "Certificate not rotate correctly") + errStart := e2e.StartCluster(serverNodeNames) + Expect(errStart).NotTo(HaveOccurred(), "Server not start correctly") + + for _, nodeName := range serverNodeNames { + grCert, errGrep := e2e.RunCmdOnNode(grepCert, nodeName) + Expect(errGrep).NotTo(HaveOccurred(), "Certificate not created correctly") + re := regexp.MustCompile("tls-[0-9]+") + tls := re.FindAllString(grCert, -1)[0] + final := fmt.Sprintf("sudo diff -sr /var/lib/rancher/k3s/server/tls/ /var/lib/rancher/k3s/server/%s/"+ + "| grep -i identical | cut -f4 -d ' ' | xargs basename -a \n", tls) + finalResult, finalErr = e2e.RunCmdOnNode(final, nodeName) + Expect(finalErr).NotTo(HaveOccurred(), "Final Certification does not created correctly") + } + if len(agentNodeNames) > 0 { + errRestartAgent := e2e.RestartCluster(agentNodeNames) + Expect(errRestartAgent).NotTo(HaveOccurred(), "Restart Agent not happened correctly") + } + finalCert := strings.Replace(finalResult, "\n", ",", -1) + finalCertArray := strings.Split(finalCert, ",") + Expect((finalCertArray)).Should((Equal(expectedResult)), "Final certification does not match the expected results") + + }) + + }) +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + +var _ = AfterSuite(func() { + if failed && !*ci { + fmt.Println("FAILED!") + } else { + Expect(e2e.DestroyCluster()).To(Succeed()) + Expect(os.Remove(kubeConfigFile)).To(Succeed()) + } +}) From e1e98bcc8a6bb9aeba8ee3d33e006faac1d3ea84 Mon Sep 17 00:00:00 2001 From: est-suse Date: Wed, 5 Apr 2023 11:50:57 -0600 Subject: [PATCH 5/7] Move Function Rotate Certificate to Utils file Signed-off-by: est-suse --- tests/e2e/certrotation/certrotation_test.go | 130 ------------------ .../rotatecertificate/rotatecertificate.go | 14 -- .../rotatecertificate_test.go | 2 +- .../vagrantfile | 0 tests/e2e/testutils.go | 11 ++ 5 files changed, 12 insertions(+), 145 deletions(-) delete mode 100644 tests/e2e/certrotation/certrotation_test.go delete mode 100644 tests/e2e/rotatecertificate/rotatecertificate.go rename tests/e2e/{certrotation => rotatecertificate}/vagrantfile (100%) diff --git a/tests/e2e/certrotation/certrotation_test.go b/tests/e2e/certrotation/certrotation_test.go deleted file mode 100644 index 209bbe59538e..000000000000 --- a/tests/e2e/certrotation/certrotation_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package secretsencryption - -import ( - "flag" - "fmt" - "os" - "regexp" - "strings" - "testing" - - "github.com/k3s-io/k3s/tests/e2e" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") -var serverCount = flag.Int("serverCount", 3, "number of server nodes") -var agentCount = flag.Int("agentCount", 1, "number of agent nodes") -var ci = flag.Bool("ci", false, "running on CI") - -// Environment Variables Info: -// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master - -func Test_E2ECustomCARotation(t *testing.T) { - RegisterFailHandler(Fail) - flag.Parse() - suiteConfig, reporterConfig := GinkgoConfiguration() - RunSpecs(t, "Custom Certificate Rotation Test Suite", suiteConfig, reporterConfig) -} - -var ( - kubeConfigFile string - agentNodeNames []string - serverNodeNames []string -) - -var _ = ReportAfterEach(e2e.GenReport) - -// RotateCertificate rotate the Certificate on each node given -func RotateCertificate(nodeNames []string) error { - for _, nodeName := range nodeNames { - cmd := "sudo k3s --debug certificate rotate" - if _, err := e2e.RunCmdOnNode(cmd, nodeName); err != nil { - return err - } - } - return nil -} - -var _ = Describe("Verify Custom CA Rotation", Ordered, func() { - Context("Custom CA is rotated:", func() { - It("Starts up with no issues", func() { - var err error - serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) - Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) - fmt.Println("CLUSTER CONFIG") - fmt.Println("OS:", *nodeOS) - fmt.Println("Server Nodes:", serverNodeNames) - fmt.Println("Agent Nodes:", agentNodeNames) - kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Verifies Certificate Rotation", func() { - const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls" - expectedResult := []string{ - "client-ca.crt", "client-ca.key", - "client-ca.nochain.crt", "client-ca.pem", - "dynamic-cert.json", "peer-ca.crt", - "peer-ca.key", "peer-ca.pem", - "server-ca.crt", "server-ca.key", - "server-ca.pem", "intermediate-ca.crt", - "intermediate-ca.key", "intermediate-ca.pem", - "request-header-ca.crt", "request-header-ca.key", - "request-header-ca.pem", "root-ca.crt", - "root-ca.key", "root-ca.pem", - "server-ca.crt", "server-ca.key", - "server-ca.nochain.crt", "server-ca.pem", - "service.current.key", "service.key", - "apiserver-loopback-client__.crt", "apiserver-loopback-client__.key", - "", - } - - var finalResult string - var finalErr error - errStop := e2e.StopCluster(serverNodeNames) - Expect(errStop).NotTo(HaveOccurred(), "Server not stop correctly") - errRotate := RotateCertificate(serverNodeNames) - Expect(errRotate).NotTo(HaveOccurred(), "Certificate not rotate correctly") - errStart := e2e.StartCluster(serverNodeNames) - Expect(errStart).NotTo(HaveOccurred(), "Server not start correctly") - - for _, nodeName := range serverNodeNames { - grCert, errGrep := e2e.RunCmdOnNode(grepCert, nodeName) - Expect(errGrep).NotTo(HaveOccurred(), "Certificate not created correctly") - re := regexp.MustCompile("tls-[0-9]+") - tls := re.FindAllString(grCert, -1)[0] - final := fmt.Sprintf("sudo diff -sr /var/lib/rancher/k3s/server/tls/ /var/lib/rancher/k3s/server/%s/ | grep -i identical | cut -f4 -d ' ' | xargs basename -a \n", tls) - finalResult, finalErr = e2e.RunCmdOnNode(final, nodeName) - Expect(finalErr).NotTo(HaveOccurred(), "Final Certification does not created correctly") - } - if len(agentNodeNames) > 0 { - errRestartAgent := e2e.RestartCluster(agentNodeNames) - Expect(errRestartAgent).NotTo(HaveOccurred(), "Restart Agent not happened correctly") - } - Eventually(func(g Gomega) { - finalCert := strings.Replace(finalResult, "\n", ",", -1) - finalCertArray := strings.Split(finalCert, ",") - Expect((finalCertArray)).Should((Equal(expectedResult)), "Final certification does not match the expected results") - }, "620s", "5s").Should(Succeed()) - - }) - - }) -}) - -var failed bool -var _ = AfterEach(func() { - failed = failed || CurrentSpecReport().Failed() -}) - -var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { - Expect(e2e.DestroyCluster()).To(Succeed()) - Expect(os.Remove(kubeConfigFile)).To(Succeed()) - } -}) diff --git a/tests/e2e/rotatecertificate/rotatecertificate.go b/tests/e2e/rotatecertificate/rotatecertificate.go deleted file mode 100644 index 45db1217f641..000000000000 --- a/tests/e2e/rotatecertificate/rotatecertificate.go +++ /dev/null @@ -1,14 +0,0 @@ -package rotatecertificate - -import "github.com/k3s-io/k3s/tests/e2e" - -// rotateCertificate rotate the Certificate on each node given -func rotateCertificate(nodeNames []string) error { - for _, nodeName := range nodeNames { - cmd := "sudo k3s --debug certificate rotate" - if _, err := e2e.RunCmdOnNode(cmd, nodeName); err != nil { - return err - } - } - return nil -} diff --git a/tests/e2e/rotatecertificate/rotatecertificate_test.go b/tests/e2e/rotatecertificate/rotatecertificate_test.go index b4b8fc2bd0c7..083ef782b5ac 100644 --- a/tests/e2e/rotatecertificate/rotatecertificate_test.go +++ b/tests/e2e/rotatecertificate/rotatecertificate_test.go @@ -75,7 +75,7 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() { var finalErr error errStop := e2e.StopCluster(serverNodeNames) Expect(errStop).NotTo(HaveOccurred(), "Server not stop correctly") - errRotate := rotateCertificate(serverNodeNames) + errRotate := e2e.RotateCertificate(serverNodeNames) Expect(errRotate).NotTo(HaveOccurred(), "Certificate not rotate correctly") errStart := e2e.StartCluster(serverNodeNames) Expect(errStart).NotTo(HaveOccurred(), "Server not start correctly") diff --git a/tests/e2e/certrotation/vagrantfile b/tests/e2e/rotatecertificate/vagrantfile similarity index 100% rename from tests/e2e/certrotation/vagrantfile rename to tests/e2e/rotatecertificate/vagrantfile diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index ab965112ece3..dc3b9bd1d0c6 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -446,6 +446,17 @@ func StopCluster(nodeNames []string) error { return nil } +// rotateCertificate rotate the Certificate on each node given +func RotateCertificate(nodeNames []string) error { + for _, nodeName := range nodeNames { + cmd := "sudo k3s --debug certificate rotate" + if _, err := RunCmdOnNode(cmd, nodeName); err != nil { + return err + } + } + return nil +} + // RunCmdOnNode executes a command from within the given node func RunCmdOnNode(cmd string, nodename string) (string, error) { runcmd := "vagrant ssh -c \"" + cmd + "\" " + nodename From adde679820baa1dc6bbc6d9daf3ba8ecab317c8e Mon Sep 17 00:00:00 2001 From: est-suse Date: Fri, 5 May 2023 12:47:56 -0600 Subject: [PATCH 6/7] Move the Certification Test to Validate Cluster Signed-off-by: est-suse --- .../rotatecertificate_test.go | 118 ------------------ tests/e2e/rotatecertificate/vagrantfile | 88 ------------- tests/e2e/testutils.go | 11 -- .../validatecluster/validatecluster_test.go | 56 ++++++++- 4 files changed, 54 insertions(+), 219 deletions(-) delete mode 100644 tests/e2e/rotatecertificate/rotatecertificate_test.go delete mode 100644 tests/e2e/rotatecertificate/vagrantfile diff --git a/tests/e2e/rotatecertificate/rotatecertificate_test.go b/tests/e2e/rotatecertificate/rotatecertificate_test.go deleted file mode 100644 index 083ef782b5ac..000000000000 --- a/tests/e2e/rotatecertificate/rotatecertificate_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package rotatecertificate - -import ( - "flag" - "fmt" - "os" - "regexp" - "strings" - "testing" - - "github.com/k3s-io/k3s/tests/e2e" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") -var serverCount = flag.Int("serverCount", 3, "number of server nodes") -var agentCount = flag.Int("agentCount", 1, "number of agent nodes") -var ci = flag.Bool("ci", false, "running on CI") - -// Environment Variables Info: -// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master - -func Test_E2ECustomCARotation(t *testing.T) { - RegisterFailHandler(Fail) - flag.Parse() - suiteConfig, reporterConfig := GinkgoConfiguration() - RunSpecs(t, "Custom Certificate Rotation Test Suite", suiteConfig, reporterConfig) -} - -var ( - kubeConfigFile string - agentNodeNames []string - serverNodeNames []string -) - -var _ = ReportAfterEach(e2e.GenReport) - -var _ = Describe("Verify Custom CA Rotation", Ordered, func() { - Context("Custom CA is rotated:", func() { - It("Starts up with no issues", func() { - var err error - serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) - Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) - fmt.Println("CLUSTER CONFIG") - fmt.Println("OS:", *nodeOS) - fmt.Println("Server Nodes:", serverNodeNames) - fmt.Println("Agent Nodes:", agentNodeNames) - kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Verifies Certificate Rotation", func() { - const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls" - expectedResult := []string{ - "client-ca.crt", "client-ca.key", - "client-ca.nochain.crt", "client-ca.pem", - "dynamic-cert.json", "peer-ca.crt", - "peer-ca.key", "peer-ca.pem", - "server-ca.crt", "server-ca.key", - "server-ca.pem", "intermediate-ca.crt", - "intermediate-ca.key", "intermediate-ca.pem", - "request-header-ca.crt", "request-header-ca.key", - "request-header-ca.pem", "root-ca.crt", - "root-ca.key", "root-ca.pem", - "server-ca.crt", "server-ca.key", - "server-ca.nochain.crt", "server-ca.pem", - "service.current.key", "service.key", - "apiserver-loopback-client__.crt", "apiserver-loopback-client__.key", - "", - } - - var finalResult string - var finalErr error - errStop := e2e.StopCluster(serverNodeNames) - Expect(errStop).NotTo(HaveOccurred(), "Server not stop correctly") - errRotate := e2e.RotateCertificate(serverNodeNames) - Expect(errRotate).NotTo(HaveOccurred(), "Certificate not rotate correctly") - errStart := e2e.StartCluster(serverNodeNames) - Expect(errStart).NotTo(HaveOccurred(), "Server not start correctly") - - for _, nodeName := range serverNodeNames { - grCert, errGrep := e2e.RunCmdOnNode(grepCert, nodeName) - Expect(errGrep).NotTo(HaveOccurred(), "Certificate not created correctly") - re := regexp.MustCompile("tls-[0-9]+") - tls := re.FindAllString(grCert, -1)[0] - final := fmt.Sprintf("sudo diff -sr /var/lib/rancher/k3s/server/tls/ /var/lib/rancher/k3s/server/%s/"+ - "| grep -i identical | cut -f4 -d ' ' | xargs basename -a \n", tls) - finalResult, finalErr = e2e.RunCmdOnNode(final, nodeName) - Expect(finalErr).NotTo(HaveOccurred(), "Final Certification does not created correctly") - } - if len(agentNodeNames) > 0 { - errRestartAgent := e2e.RestartCluster(agentNodeNames) - Expect(errRestartAgent).NotTo(HaveOccurred(), "Restart Agent not happened correctly") - } - finalCert := strings.Replace(finalResult, "\n", ",", -1) - finalCertArray := strings.Split(finalCert, ",") - Expect((finalCertArray)).Should((Equal(expectedResult)), "Final certification does not match the expected results") - - }) - - }) -}) - -var failed bool -var _ = AfterEach(func() { - failed = failed || CurrentSpecReport().Failed() -}) - -var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { - Expect(e2e.DestroyCluster()).To(Succeed()) - Expect(os.Remove(kubeConfigFile)).To(Succeed()) - } -}) diff --git a/tests/e2e/rotatecertificate/vagrantfile b/tests/e2e/rotatecertificate/vagrantfile deleted file mode 100644 index bb50599ca58f..000000000000 --- a/tests/e2e/rotatecertificate/vagrantfile +++ /dev/null @@ -1,88 +0,0 @@ -ENV['VAGRANT_NO_PARALLEL'] = 'no' -NODE_ROLES = (ENV['E2E_NODE_ROLES'] || - ["server-0", "server-1", "server-2", "agent-0"]) -NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204']) -GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") -RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") -NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i -NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i -# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks -NETWORK_PREFIX = "10.10.10" -install_type = "" - -def provision(vm, role, role_num, node_num) - vm.box = NODE_BOXES[node_num] - vm.hostname = role - # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 - vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0" - - vagrant_defaults = '../vagrantdefaults.rb' - load vagrant_defaults if File.exists?(vagrant_defaults) - - defaultOSConfigure(vm) - install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) - - vm.provision "shell", inline: "ping -c 2 k3s.io" - - if role.include?("server") && role_num == 0 - vm.provision 'file' do |scp| - scp.source = '../../../contrib/util/generate-custom-ca-certs.sh' - scp.destination = '/tmp/generate-custom-ca-certs.sh' - end - vm.provision 'custom-ca', type: 'shell', run: 'once' do |script| - script.inline = 'bash /tmp/generate-custom-ca-certs.sh' - script.env = {'PRODUCT' => 'vagrant-e2e-test', 'DATA_DIR' => '/var/lib/rancher/k3s'} - end - vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1] - k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] - k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - elsif role.include?("server") && role_num != 0 - vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = %W[server --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] - k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] - k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - elsif role.include?("agent") - vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = %W[agent --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1] - k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] - k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - end - if vm.box.to_s.include?("microos") - vm.provision 'k3s-reload', type: 'reload', run: 'once' - end -end - -Vagrant.configure("2") do |config| - config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"] - # Default provider is libvirt, virtualbox is only provided as a backup - config.vm.provider "libvirt" do |v| - v.cpus = NODE_CPUS - v.memory = NODE_MEMORY - end - config.vm.provider "virtualbox" do |v| - v.cpus = NODE_CPUS - v.memory = NODE_MEMORY - end - - if NODE_ROLES.kind_of?(String) - NODE_ROLES = NODE_ROLES.split(" ", -1) - end - if NODE_BOXES.kind_of?(String) - NODE_BOXES = NODE_BOXES.split(" ", -1) - end - - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) - end - end -end diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index dc3b9bd1d0c6..ab965112ece3 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -446,17 +446,6 @@ func StopCluster(nodeNames []string) error { return nil } -// rotateCertificate rotate the Certificate on each node given -func RotateCertificate(nodeNames []string) error { - for _, nodeName := range nodeNames { - cmd := "sudo k3s --debug certificate rotate" - if _, err := RunCmdOnNode(cmd, nodeName); err != nil { - return err - } - } - return nil -} - // RunCmdOnNode executes a command from within the given node func RunCmdOnNode(cmd string, nodename string) (string, error) { runcmd := "vagrant ssh -c \"" + cmd + "\" " + nodename diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index 6ec6fa9251be..9dccf92264a1 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "regexp" "strings" "testing" @@ -99,7 +100,6 @@ var _ = Describe("Verify Create", Ordered, func() { clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false) cmd := "curl -L --insecure http://" + clusterip + "/name.html" - fmt.Println(cmd) for _, nodeName := range serverNodeNames { Eventually(func(g Gomega) { res, err := e2e.RunCmdOnNode(cmd, nodeName) @@ -127,7 +127,7 @@ var _ = Describe("Verify Create", Ordered, func() { }, "240s", "5s").Should(Succeed()) cmd = "curl -L --insecure http://" + nodeExternalIP + ":" + nodeport + "/name.html" - fmt.Println(cmd) + Eventually(func(g Gomega) { res, err := e2e.RunCommand(cmd) g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) @@ -210,6 +210,7 @@ var _ = Describe("Verify Create", Ordered, func() { Eventually(func(g Gomega) { cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default" + res, err := e2e.RunCommand(cmd) g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) @@ -313,6 +314,57 @@ var _ = Describe("Verify Create", Ordered, func() { g.Expect(res).Should(ContainSubstring("local-path-test")) }, "180s", "2s").Should(Succeed()) }) + + It("Verifies Certificate Rotation", func() { + const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls" + var expectResult = []string{"client-ca.crt", + "client-ca.key", + "client-ca.nochain.crt", + "dynamic-cert.json", "peer-ca.crt", + "peer-ca.key", "server-ca.crt", + "server-ca.key", "request-header-ca.crt", + "request-header-ca.key", "server-ca.crt", + "server-ca.key", "server-ca.nochain.crt", + "service.current.key", "service.key", + "apiserver-loopback-client__.crt", + "apiserver-loopback-client__.key", "", + } + + var finalResult string + var finalErr error + errStop := e2e.StopCluster(serverNodeNames) + Expect(errStop).NotTo(HaveOccurred(), "Cluster could not be stoped successfully") + + for _, nodeName := range serverNodeNames { + cmd := "sudo k3s --debug certificate rotate" + if _, err := e2e.RunCmdOnNode(cmd, nodeName); err != nil { + Expect(err).NotTo(HaveOccurred(), "Certificate could not be rotated successfully") + } + } + + errStart := e2e.StartCluster(serverNodeNames) + Expect(errStart).NotTo(HaveOccurred(), "Cluster could not be started successfully") + + for _, nodeName := range serverNodeNames { + grCert, errGrep := e2e.RunCmdOnNode(grepCert, nodeName) + Expect(errGrep).NotTo(HaveOccurred(), "Certificate could not be created successfully") + re := regexp.MustCompile("tls-[0-9]+") + tls := re.FindAllString(grCert, -1)[0] + final := fmt.Sprintf("sudo diff -sr /var/lib/rancher/k3s/server/tls/ /var/lib/rancher/k3s/server/%s/"+ + "| grep -i identical | cut -f4 -d ' ' | xargs basename -a \n", tls) + finalResult, finalErr = e2e.RunCmdOnNode(final, nodeName) + Expect(finalErr).NotTo(HaveOccurred(), "Final Certification does not created successfully") + } + if len(agentNodeNames) > 0 { + errRestartAgent := e2e.RestartCluster(agentNodeNames) + Expect(errRestartAgent).NotTo(HaveOccurred(), "Agent could not be restart successfully") + } + finalCert := strings.Replace(finalResult, "\n", ",", -1) + finalCertArray := strings.Split(finalCert, ",") + Expect((finalCertArray)).Should((Equal(expectResult)), "Final certification does not match the expected results") + + }) + }) }) From 9b9df2f8ee121b5b660bf8ccaed65f5dcc2d1ec2 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 11 May 2023 15:09:05 -0700 Subject: [PATCH 7/7] Fix to stop/start for k3s certificate rotation Signed-off-by: Derek Nola --- tests/e2e/testutils.go | 5 +- .../validatecluster/validatecluster_test.go | 154 +++++++++++------- 2 files changed, 96 insertions(+), 63 deletions(-) diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index ab965112ece3..2778912f13f5 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -427,7 +427,10 @@ func RestartCluster(nodeNames []string) error { // StartCluster starts the k3s service on each node given func StartCluster(nodeNames []string) error { for _, nodeName := range nodeNames { - cmd := "sudo systemctl start k3s*" + cmd := "sudo systemctl start k3s" + if strings.Contains(nodeName, "agent") { + cmd += "-agent" + } if _, err := RunCmdOnNode(cmd, nodeName); err != nil { return err } diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index 9dccf92264a1..955f6c700b70 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -44,7 +44,7 @@ var ( var _ = ReportAfterEach(e2e.GenReport) var _ = Describe("Verify Create", Ordered, func() { - Context("Cluster :", func() { + Context("Cluster Starts up and deploys basic components", func() { It("Starts up with no issues", func() { var err error if *local { @@ -217,49 +217,6 @@ var _ = Describe("Verify Create", Ordered, func() { }, "420s", "2s").Should(Succeed()) }) - It("Verifies Restart", func() { - _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened) - Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") - defer e2e.DeleteWorkload("daemonset.yaml", kubeConfigFile) - nodes, _ := e2e.ParseNodes(kubeConfigFile, false) - - Eventually(func(g Gomega) { - pods, _ := e2e.ParsePods(kubeConfigFile, false) - count := e2e.CountOfStringInSlice("test-daemonset", pods) - g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") - podsRunning := 0 - for _, pod := range pods { - if strings.Contains(pod.Name, "test-daemonset") && pod.Status == "Running" && pod.Ready == "1/1" { - podsRunning++ - } - } - g.Expect(len(nodes)).Should((Equal(podsRunning)), "Daemonset running pods count does not match node count") - }, "620s", "5s").Should(Succeed()) - errRestart := e2e.RestartCluster(serverNodeNames) - Expect(errRestart).NotTo(HaveOccurred(), "Restart Nodes not happened correctly") - if len(agentNodeNames) > 0 { - errRestartAgent := e2e.RestartCluster(agentNodeNames) - Expect(errRestartAgent).NotTo(HaveOccurred(), "Restart Agent not happened correctly") - } - Eventually(func(g Gomega) { - nodes, err := e2e.ParseNodes(kubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready")) - } - pods, _ := e2e.ParsePods(kubeConfigFile, false) - count := e2e.CountOfStringInSlice("test-daemonset", pods) - g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") - podsRunningAr := 0 - for _, pod := range pods { - if strings.Contains(pod.Name, "test-daemonset") && pod.Status == "Running" && pod.Ready == "1/1" { - podsRunningAr++ - } - } - g.Expect(len(nodes)).Should((Equal(podsRunningAr)), "Daemonset pods are not running after the restart") - }, "620s", "5s").Should(Succeed()) - }) - It("Verifies Local Path Provisioner storage ", func() { res, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed: "+res) @@ -314,8 +271,95 @@ var _ = Describe("Verify Create", Ordered, func() { g.Expect(res).Should(ContainSubstring("local-path-test")) }, "180s", "2s").Should(Succeed()) }) + }) - It("Verifies Certificate Rotation", func() { + Context("Validate restart", func() { + It("Deletes daemonset", func() { + _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened) + Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") + defer e2e.DeleteWorkload("daemonset.yaml", kubeConfigFile) + nodes, _ := e2e.ParseNodes(kubeConfigFile, false) + + Eventually(func(g Gomega) { + pods, _ := e2e.ParsePods(kubeConfigFile, false) + count := e2e.CountOfStringInSlice("test-daemonset", pods) + g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") + podsRunning := 0 + for _, pod := range pods { + if strings.Contains(pod.Name, "test-daemonset") && pod.Status == "Running" && pod.Ready == "1/1" { + podsRunning++ + } + } + g.Expect(len(nodes)).Should((Equal(podsRunning)), "Daemonset running pods count does not match node count") + }, "620s", "5s").Should(Succeed()) + }) + It("Restarts normally", func() { + errRestart := e2e.RestartCluster(append(serverNodeNames, agentNodeNames...)) + Expect(errRestart).NotTo(HaveOccurred(), "Restart Nodes not happened correctly") + + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + pods, _ := e2e.ParsePods(kubeConfigFile, false) + count := e2e.CountOfStringInSlice("test-daemonset", pods) + g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") + podsRunningAr := 0 + for _, pod := range pods { + if strings.Contains(pod.Name, "test-daemonset") && pod.Status == "Running" && pod.Ready == "1/1" { + podsRunningAr++ + } + } + g.Expect(len(nodes)).Should((Equal(podsRunningAr)), "Daemonset pods are not running after the restart") + }, "620s", "5s").Should(Succeed()) + }) + }) + + Context("Valdiate Certificate Rotation", func() { + It("Stops K3s and rotates certificates", func() { + errStop := e2e.StopCluster(serverNodeNames) + Expect(errStop).NotTo(HaveOccurred(), "Cluster could not be stoped successfully") + + for _, nodeName := range serverNodeNames { + cmd := "sudo k3s certificate rotate" + if _, err := e2e.RunCmdOnNode(cmd, nodeName); err != nil { + Expect(err).NotTo(HaveOccurred(), "Certificate could not be rotated successfully") + } + } + }) + + It("Start normally", func() { + // Since we stopped all the server, we have to start 2 at once to get it back up + // If we only start one at a time, the first will hang waiting for the second to be up + _, err := e2e.RunCmdOnNode("sudo systemctl --no-block start k3s", serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + err = e2e.StartCluster(serverNodeNames[1:]) + Expect(err).NotTo(HaveOccurred(), "Cluster could not be started successfully") + + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + fmt.Println("help") + }, "620s", "5s").Should(Succeed()) + + Eventually(func(g Gomega) { + pods, err := e2e.ParsePods(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) + } else { + g.Expect(pod.Status).Should(Equal("Running"), pod.Name) + } + } + }, "620s", "5s").Should(Succeed()) + }) + It("Validates certificates", func() { const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls" var expectResult = []string{"client-ca.crt", "client-ca.key", @@ -332,19 +376,6 @@ var _ = Describe("Verify Create", Ordered, func() { var finalResult string var finalErr error - errStop := e2e.StopCluster(serverNodeNames) - Expect(errStop).NotTo(HaveOccurred(), "Cluster could not be stoped successfully") - - for _, nodeName := range serverNodeNames { - cmd := "sudo k3s --debug certificate rotate" - if _, err := e2e.RunCmdOnNode(cmd, nodeName); err != nil { - Expect(err).NotTo(HaveOccurred(), "Certificate could not be rotated successfully") - } - } - - errStart := e2e.StartCluster(serverNodeNames) - Expect(errStart).NotTo(HaveOccurred(), "Cluster could not be started successfully") - for _, nodeName := range serverNodeNames { grCert, errGrep := e2e.RunCmdOnNode(grepCert, nodeName) Expect(errGrep).NotTo(HaveOccurred(), "Certificate could not be created successfully") @@ -355,10 +386,9 @@ var _ = Describe("Verify Create", Ordered, func() { finalResult, finalErr = e2e.RunCmdOnNode(final, nodeName) Expect(finalErr).NotTo(HaveOccurred(), "Final Certification does not created successfully") } - if len(agentNodeNames) > 0 { - errRestartAgent := e2e.RestartCluster(agentNodeNames) - Expect(errRestartAgent).NotTo(HaveOccurred(), "Agent could not be restart successfully") - } + errRestartAgent := e2e.RestartCluster(agentNodeNames) + Expect(errRestartAgent).NotTo(HaveOccurred(), "Agent could not be restart successfully") + finalCert := strings.Replace(finalResult, "\n", ",", -1) finalCertArray := strings.Split(finalCert, ",") Expect((finalCertArray)).Should((Equal(expectResult)), "Final certification does not match the expected results")