[Release-1.26] E2E and Dep Backports - June (k3s-io#7693)

* Bump docker go.mod (k3s-io#7681)
* Shortcircuit commands with version or help flags (k3s-io#7683)
* Fix for longhorn integration test
* Add Rotation certification Check (k3s-io#7097)
* Add Certification Test to Validate Cluster
* Fix to stop/start for k3s certificate rotation
* E2E: Use sudo for all RunCmdOnNode
* Remove unnecessary daemonset addition/deletion

Signed-off-by: Derek Nola <[email protected]>
Signed-off-by: est-suse <[email protected]>
Signed-off-by: Brad Davidson <[email protected]>
Co-authored-by: Esteban Esquivel Alvarado <[email protected]>
Co-authored-by: est-suse <[email protected]>
Co-authored-by: Brad Davidson <[email protected]>

go.mod

@@ -10,7 +10,7 @@ replace (
 	github.com/containerd/stargz-snapshotter => github.com/k3s-io/stargz-snapshotter v0.13.0-k3s1
 	github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
 	github.com/docker/distribution => github.com/docker/distribution v2.8.1+incompatible
-	github.com/docker/docker => github.com/docker/docker v20.10.12+incompatible
+	github.com/docker/docker => github.com/docker/docker v20.10.24+incompatible
 	github.com/docker/libnetwork => github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34
 
 	github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 v3.9.0

go.sum

@@ -287,8 +287,8 @@ github.com/docker/cli v23.0.3+incompatible h1:Zcse1DuDqBdgI7OQDV8Go7b83xLgfhW1ez
 github.com/docker/cli v23.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
 github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v20.10.12+incompatible h1:CEeNmFM0QZIsJCZKMkZx0ZcahTiewkrgiwfYD+dfl1U=
-github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE=
+github.com/docker/docker v20.10.24+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
 github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
 github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=

pkg/configfilearg/defaultparser.go

@@ -9,7 +9,7 @@ import (
 
 var DefaultParser = &Parser{
 	After:         []string{"server", "agent", "etcd-snapshot:1"},
-	FlagNames:     []string{"--config", "-c"},
+	ConfigFlags:   []string{"--config", "-c"},
 	EnvName:       version.ProgramUpper + "_CONFIG_FILE",
 	DefaultConfig: "/etc/rancher/" + version.Program + "/config.yaml",
 	ValidFlags:    map[string][]cli.Flag{"server": cmds.ServerFlags, "etcd-snapshot": cmds.EtcdSnapshotFlags},
@@ -25,8 +25,7 @@ func MustParse(args []string) []string {
 
 func MustFindString(args []string, target string) string {
 	parser := &Parser{
-		After:         []string{},
-		FlagNames:     []string{},
+		OverrideFlags: []string{"--help", "-h", "--version", "-v"},
 		EnvName:       version.ProgramUpper + "_CONFIG_FILE",
 		DefaultConfig: "/etc/rancher/" + version.Program + "/config.yaml",
 	}

pkg/configfilearg/parser.go

@@ -20,10 +20,13 @@ import (
 
 type Parser struct {
 	After         []string
-	FlagNames     []string
+	ConfigFlags   []string
+	OverrideFlags []string
 	EnvName       string
 	DefaultConfig string
-	ValidFlags    map[string][]cli.Flag
+	// ValidFlags are maps of flags that are valid for that particular conmmand. This enables us to ignore flags in
+	// the config file that do no apply to the current command.
+	ValidFlags map[string][]cli.Flag
 }
 
 // Parse will parse an os.Args style slice looking for Parser.FlagNames after Parse.After.
@@ -97,6 +100,12 @@ func (p *Parser) stripInvalidFlags(command string, args []string) ([]string, err
 }
 
 func (p *Parser) FindString(args []string, target string) (string, error) {
+
+	// Check for --help or --version flags, which override any other flags
+	if val, found := p.findOverrideFlag(args); found {
+		return val, nil
+	}
+
 	configFile, isSet := p.findConfigFileFlag(args)
 	var lastVal string
 	if configFile != "" {
@@ -140,13 +149,25 @@ func (p *Parser) FindString(args []string, target string) (string, error) {
 	return lastVal, nil
 }
 
+func (p *Parser) findOverrideFlag(args []string) (string, bool) {
+	for _, arg := range args {
+		for _, flagName := range p.OverrideFlags {
+			if flagName == arg {
+				return arg, true
+			}
+		}
+	}
+
+	return "", false
+}
+
 func (p *Parser) findConfigFileFlag(args []string) (string, bool) {
 	if envVal := os.Getenv(p.EnvName); p.EnvName != "" && envVal != "" {
 		return envVal, true
 	}
 
 	for i, arg := range args {
-		for _, flagName := range p.FlagNames {
+		for _, flagName := range p.ConfigFlags {
 			if flagName == arg {
 				if len(args) > i+1 {
 					return args[i+1], true

pkg/configfilearg/parser_test.go

@@ -199,7 +199,7 @@ func Test_UnitParser_findConfigFileFlag(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			p := Parser{
-				FlagNames:     []string{"--config", "-c"},
+				ConfigFlags:   []string{"--config", "-c"},
 				EnvName:       "_TEST_FLAG_ENV",
 				DefaultConfig: tt.fields.DefaultConfig,
 			}
@@ -328,7 +328,7 @@ func Test_UnitParser_Parse(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			p := &Parser{
 				After:         tt.fields.After,
-				FlagNames:     tt.fields.FlagNames,
+				ConfigFlags:   tt.fields.FlagNames,
 				EnvName:       tt.fields.EnvName,
 				DefaultConfig: tt.fields.DefaultConfig,
 			}
@@ -447,7 +447,7 @@ func Test_UnitParser_FindString(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			p := &Parser{
 				After:         tt.fields.After,
-				FlagNames:     tt.fields.FlagNames,
+				ConfigFlags:   tt.fields.FlagNames,
 				EnvName:       tt.fields.EnvName,
 				DefaultConfig: tt.fields.DefaultConfig,
 			}

tests/e2e/multiclustercidr/multiclustercidr_test.go

@@ -106,7 +106,7 @@ var _ = Describe("Verify MultiClusterCIDR Configuration", Ordered, func() {
 
 	It("Restart agent-0", func() {
 		agents := []string{"agent-0"}
-		err := e2e.RestartClusterAgent(agents)
+		err := e2e.RestartCluster(agents)
 		Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 	})
 
@@ -223,7 +223,7 @@ var _ = Describe("Verify MultiClusterCIDR Configuration", Ordered, func() {
 
 	It("Delete and restart agent-0", func() {
 		agents := []string{"agent-0"}
-		err := e2e.RestartClusterAgent(agents)
+		err := e2e.RestartCluster(agents)
 		Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 	})
 

tests/e2e/rotateca/rotateca_test.go

@@ -78,9 +78,9 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() {
 
 		It("Generates New CA Certificates", func() {
 			cmds := []string{
-				"sudo mkdir -p /opt/rancher/k3s/server",
-				"sudo cp -r /var/lib/rancher/k3s/server/tls /opt/rancher/k3s/server",
-				"sudo DATA_DIR=/opt/rancher/k3s /tmp/generate-custom-ca-certs.sh",
+				"mkdir -p /opt/rancher/k3s/server",
+				"cp -r /var/lib/rancher/k3s/server/tls /opt/rancher/k3s/server",
+				"DATA_DIR=/opt/rancher/k3s /tmp/generate-custom-ca-certs.sh",
 			}
 			for _, cmd := range cmds {
 				_, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
@@ -89,7 +89,7 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() {
 		})
 
 		It("Rotates CA Certificates", func() {
-			cmd := "sudo k3s certificate rotate-ca --path=/opt/rancher/k3s/server"
+			cmd := "k3s certificate rotate-ca --path=/opt/rancher/k3s/server"
 			_, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
 		})

tests/e2e/secretsencryption/secretsencryption_test.go

@@ -85,7 +85,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 		})
 
 		It("Verifies encryption start stage", func() {
-			cmd := "sudo k3s secrets-encrypt status"
+			cmd := "k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
 				res, err := e2e.RunCmdOnNode(cmd, nodeName)
 				Expect(err).NotTo(HaveOccurred())
@@ -96,11 +96,11 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 		})
 
 		It("Prepares for Secrets-Encryption Rotation", func() {
-			cmd := "sudo k3s secrets-encrypt prepare"
+			cmd := "k3s secrets-encrypt prepare"
 			res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred(), res)
 			for i, nodeName := range serverNodeNames {
-				cmd := "sudo k3s secrets-encrypt status"
+				cmd := "k3s secrets-encrypt status"
 				res, err := e2e.RunCmdOnNode(cmd, nodeName)
 				Expect(err).NotTo(HaveOccurred(), res)
 				Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
@@ -140,7 +140,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 		})
 
 		It("Verifies encryption prepare stage", func() {
-			cmd := "sudo k3s secrets-encrypt status"
+			cmd := "k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
 				Eventually(func(g Gomega) {
 					res, err := e2e.RunCmdOnNode(cmd, nodeName)
@@ -153,12 +153,12 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 		})
 
 		It("Rotates the Secrets-Encryption Keys", func() {
-			cmd := "sudo k3s secrets-encrypt rotate"
+			cmd := "k3s secrets-encrypt rotate"
 			res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred(), res)
 			for i, nodeName := range serverNodeNames {
 				Eventually(func(g Gomega) {
-					cmd := "sudo k3s secrets-encrypt status"
+					cmd := "k3s secrets-encrypt status"
 					res, err := e2e.RunCmdOnNode(cmd, nodeName)
 					g.Expect(err).NotTo(HaveOccurred(), res)
 					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
@@ -176,7 +176,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 		})
 
 		It("Verifies encryption rotate stage", func() {
-			cmd := "sudo k3s secrets-encrypt status"
+			cmd := "k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
 				Eventually(func(g Gomega) {
 					res, err := e2e.RunCmdOnNode(cmd, nodeName)
@@ -189,11 +189,11 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 		})
 
 		It("Reencrypts the Secrets-Encryption Keys", func() {
-			cmd := "sudo k3s secrets-encrypt reencrypt"
+			cmd := "k3s secrets-encrypt reencrypt"
 			res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred(), res)
 
-			cmd = "sudo k3s secrets-encrypt status"
+			cmd = "k3s secrets-encrypt status"
 			Eventually(func() (string, error) {
 				return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
@@ -211,7 +211,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 		})
 
 		It("Verifies Encryption Reencrypt Stage", func() {
-			cmd := "sudo k3s secrets-encrypt status"
+			cmd := "k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
 				Eventually(func(g Gomega) {
 					res, err := e2e.RunCmdOnNode(cmd, nodeName)
@@ -226,15 +226,15 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 
 	Context("Disabling Secrets-Encryption", func() {
 		It("Disables encryption", func() {
-			cmd := "sudo k3s secrets-encrypt disable"
+			cmd := "k3s secrets-encrypt disable"
 			res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred(), res)
 
-			cmd = "sudo k3s secrets-encrypt reencrypt -f --skip"
+			cmd = "k3s secrets-encrypt reencrypt -f --skip"
 			res, err = e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred(), res)
 
-			cmd = "sudo k3s secrets-encrypt status"
+			cmd = "k3s secrets-encrypt status"
 			Eventually(func() (string, error) {
 				return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
@@ -257,7 +257,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 		})
 
 		It("Verifies encryption disabled on all nodes", func() {
-			cmd := "sudo k3s secrets-encrypt status"
+			cmd := "k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
 				Eventually(func(g Gomega) {
 					g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Disabled"))
@@ -269,15 +269,15 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 
 	Context("Enabling Secrets-Encryption", func() {
 		It("Enables encryption", func() {
-			cmd := "sudo k3s secrets-encrypt enable"
+			cmd := "k3s secrets-encrypt enable"
 			res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred(), res)
 
-			cmd = "sudo k3s secrets-encrypt reencrypt -f --skip"
+			cmd = "k3s secrets-encrypt reencrypt -f --skip"
 			res, err = e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred(), res)
 
-			cmd = "sudo k3s secrets-encrypt status"
+			cmd = "k3s secrets-encrypt status"
 			Eventually(func() (string, error) {
 				return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
@@ -288,7 +288,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 		})
 
 		It("Verifies encryption enabled on all nodes", func() {
-			cmd := "sudo k3s secrets-encrypt status"
+			cmd := "k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
 				Eventually(func(g Gomega) {
 					g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled"))

tests/e2e/snapshotrestore/snapshotrestore_test.go

@@ -102,10 +102,10 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
 
 		It("Verifies Snapshot is created", func() {
 			Eventually(func(g Gomega) {
-				cmd := "sudo k3s etcd-snapshot save"
+				cmd := "k3s etcd-snapshot save"
 				_, err := e2e.RunCmdOnNode(cmd, "server-0")
 				g.Expect(err).NotTo(HaveOccurred())
-				cmd = "sudo ls /var/lib/rancher/k3s/server/db/snapshots/"
+				cmd = "ls /var/lib/rancher/k3s/server/db/snapshots/"
 				snapshotname, err = e2e.RunCmdOnNode(cmd, "server-0")
 				g.Expect(err).NotTo(HaveOccurred())
 				fmt.Println("Snapshot Name", snapshotname)
@@ -128,20 +128,20 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
 	Context("Cluster is reset normally", func() {
 		It("Resets the cluster", func() {
 			for _, nodeName := range serverNodeNames {
-				cmd := "sudo systemctl stop k3s"
+				cmd := "systemctl stop k3s"
 				Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
 				if nodeName != serverNodeNames[0] {
 					cmd = "k3s-killall.sh"
 					Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
 				}
 			}
 
-			cmd := "sudo k3s server --cluster-reset"
+			cmd := "k3s server --cluster-reset"
 			res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
 			Expect(res).Should(ContainSubstring("Managed etcd cluster membership has been reset, restart without --cluster-reset flag now"))
 
-			cmd = "sudo systemctl start k3s"
+			cmd = "systemctl start k3s"
 			Expect(e2e.RunCmdOnNode(cmd, serverNodeNames[0])).Error().NotTo(HaveOccurred())
 		})
 
@@ -165,12 +165,12 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
 			// We must remove the db directory on the other servers before restarting k3s
 			// otherwise the nodes may join the old cluster
 			for _, nodeName := range serverNodeNames[1:] {
-				cmd := "sudo rm -rf /var/lib/rancher/k3s/server/db"
+				cmd := "rm -rf /var/lib/rancher/k3s/server/db"
 				Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
 			}
 
 			for _, nodeName := range serverNodeNames[1:] {
-				cmd := "sudo systemctl start k3s"
+				cmd := "systemctl start k3s"
 				Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
 				time.Sleep(20 * time.Second) //Stagger the restarts for etcd leaners
 			}
@@ -214,20 +214,20 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
 		It("Restores the snapshot", func() {
 			//Stop k3s on all nodes
 			for _, nodeName := range serverNodeNames {
-				cmd := "sudo systemctl stop k3s"
+				cmd := "systemctl stop k3s"
 				Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
 				if nodeName != serverNodeNames[0] {
 					cmd = "k3s-killall.sh"
 					Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
 				}
 			}
 			//Restores from snapshot on server-0
-			cmd := "sudo k3s server --cluster-init --cluster-reset --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/" + snapshotname
+			cmd := "k3s server --cluster-init --cluster-reset --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/" + snapshotname
 			res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
 			Expect(res).Should(ContainSubstring("Managed etcd cluster membership has been reset, restart without --cluster-reset flag now"))
 
-			cmd = "sudo systemctl start k3s"
+			cmd = "systemctl start k3s"
 			Expect(e2e.RunCmdOnNode(cmd, serverNodeNames[0])).Error().NotTo(HaveOccurred())
 
 		})
@@ -252,12 +252,12 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
 			// We must remove the db directory on the other servers before restarting k3s
 			// otherwise the nodes may join the old cluster
 			for _, nodeName := range serverNodeNames[1:] {
-				cmd := "sudo rm -rf /var/lib/rancher/k3s/server/db"
+				cmd := "rm -rf /var/lib/rancher/k3s/server/db"
 				Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
 			}
 
 			for _, nodeName := range serverNodeNames[1:] {
-				cmd := "sudo systemctl start k3s"
+				cmd := "systemctl start k3s"
 				Expect(e2e.RunCmdOnNode(cmd, nodeName)).Error().NotTo(HaveOccurred())
 			}
 		})