From 1a75d9c7af264bb2c7b4570b1890d05c887853a4 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Sun, 13 Feb 2022 23:26:53 +0100 Subject: [PATCH 1/8] tests: Use restart in downgrade tests --- tests/e2e/cluster_downgrade_test.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 93a54884b54b..8ef868f8e4a6 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -26,7 +26,7 @@ import ( ) func TestDowngradeUpgrade(t *testing.T) { - currentEtcdBinary := "" + currentEtcdBinary := e2e.BinDir + "/etcd" lastReleaseBinary := e2e.BinDir + "/etcd-last-release" if !fileutil.Exist(lastReleaseBinary) { t.Skipf("%q does not exist", lastReleaseBinary) @@ -39,24 +39,24 @@ func TestDowngradeUpgrade(t *testing.T) { e2e.BeforeTest(t) dataDirPath := t.TempDir() - epc := startEtcd(t, currentEtcdBinary, dataDirPath) + epc := newCluster(t, currentEtcdBinary, dataDirPath) validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) downgradeEnable(t, epc, lastVersion) expectLog(t, epc, "The server is ready to downgrade") validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: currentVersionStr}) - stopEtcd(t, epc) - epc = startEtcd(t, lastReleaseBinary, dataDirPath) + stopEtcd(t, epc.Procs[0]) + startEtcd(t, epc, lastReleaseBinary) expectLog(t, epc, "the cluster has been downgraded") validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: lastVersionStr}) - stopEtcd(t, epc) - epc = startEtcd(t, currentEtcdBinary, dataDirPath) + stopEtcd(t, epc.Procs[0]) + startEtcd(t, epc, currentEtcdBinary) validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) } -func startEtcd(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessCluster { +func newCluster(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessCluster { epc, err := e2e.NewEtcdProcessCluster(t, &e2e.EtcdProcessClusterConfig{ ExecPath: execPath, DataDirPath: dataDirPath, @@ -75,6 +75,14 @@ func startEtcd(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessClust return epc } +func startEtcd(t *testing.T, epc *e2e.EtcdProcessCluster, execPath string) { + epc.Procs[0].Config().ExecPath = execPath + err := epc.Procs[0].Restart() + if err != nil { + t.Fatalf("could not start etcd process cluster (%v)", err) + } +} + func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Version) { t.Log("etcdctl downgrade...") c := e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3()) @@ -86,9 +94,9 @@ func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Versi }) } -func stopEtcd(t *testing.T, epc *e2e.EtcdProcessCluster) { +func stopEtcd(t *testing.T, ep e2e.EtcdProcess) { t.Log("Stopping the server...") - if err := epc.Procs[0].Stop(); err != nil { + if err := ep.Stop(); err != nil { t.Fatal(err) } } From b2cf21ef62af2837b32d3f356003973aa5e1cff7 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Sun, 13 Feb 2022 23:31:08 +0100 Subject: [PATCH 2/8] tests: Simplify downgrade tests --- tests/e2e/cluster_downgrade_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 8ef868f8e4a6..986ca24e0be4 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -37,9 +37,8 @@ func TestDowngradeUpgrade(t *testing.T) { lastVersionStr := fmt.Sprintf("%d.%d", lastVersion.Major, lastVersion.Minor) e2e.BeforeTest(t) - dataDirPath := t.TempDir() - epc := newCluster(t, currentEtcdBinary, dataDirPath) + epc := newCluster(t, currentEtcdBinary) validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) downgradeEnable(t, epc, lastVersion) @@ -56,10 +55,9 @@ func TestDowngradeUpgrade(t *testing.T) { validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) } -func newCluster(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessCluster { +func newCluster(t *testing.T, execPath string) *e2e.EtcdProcessCluster { epc, err := e2e.NewEtcdProcessCluster(t, &e2e.EtcdProcessClusterConfig{ ExecPath: execPath, - DataDirPath: dataDirPath, ClusterSize: 1, InitialToken: "new", KeepDataDir: true, From c8955d164ca8019d7f474879e0c81e06bf354724 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Feb 2022 10:04:50 +0100 Subject: [PATCH 3/8] tests: Allow sepecting member to in curl commands --- tests/e2e/cluster_downgrade_test.go | 5 +++-- tests/e2e/v3_curl_test.go | 5 +++-- tests/framework/e2e/curl.go | 20 ++++++++++---------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 986ca24e0be4..11eaf76424aa 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -16,6 +16,7 @@ package e2e import ( "fmt" + "math/rand" "testing" "time" @@ -104,13 +105,13 @@ func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.V // Two separate calls to expect as it doesn't support multiple matches on the same line e2e.ExecuteWithTimeout(t, 20*time.Second, func() { if expect.Server != "" { - err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc, "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) + err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) if err != nil { t.Fatal(err) } } if expect.Cluster != "" { - err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc, "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) + err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) if err != nil { t.Fatal(err) } diff --git a/tests/e2e/v3_curl_test.go b/tests/e2e/v3_curl_test.go index 17fe6590be8e..35450bbe418c 100644 --- a/tests/e2e/v3_curl_test.go +++ b/tests/e2e/v3_curl_test.go @@ -18,6 +18,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "math/rand" "path" "strconv" "testing" @@ -243,7 +244,7 @@ func testV3CurlAuth(cx ctlCtx) { lineFunc = func(txt string) bool { return true } ) - cmdArgs = e2e.CURLPrefixArgs(cx.epc, "POST", e2e.CURLReq{Endpoint: path.Join(p, "/auth/authenticate"), Value: string(authreq)}) + cmdArgs = e2e.CURLPrefixArgs(cx.epc.Cfg, cx.epc.Procs[rand.Intn(cx.epc.Cfg.ClusterSize)], "POST", e2e.CURLReq{Endpoint: path.Join(p, "/auth/authenticate"), Value: string(authreq)}) proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) testutil.AssertNil(cx.t, err) defer proc.Close() @@ -282,7 +283,7 @@ func testV3CurlCampaign(cx ctlCtx) { if err != nil { cx.t.Fatal(err) } - cargs := e2e.CURLPrefixArgs(cx.epc, "POST", e2e.CURLReq{ + cargs := e2e.CURLPrefixArgs(cx.epc.Cfg, cx.epc.Procs[rand.Intn(cx.epc.Cfg.ClusterSize)], "POST", e2e.CURLReq{ Endpoint: path.Join(cx.apiPrefix, "/election/campaign"), Value: string(cdata), }) diff --git a/tests/framework/e2e/curl.go b/tests/framework/e2e/curl.go index 284b49aaa91d..a3b11de8591b 100644 --- a/tests/framework/e2e/curl.go +++ b/tests/framework/e2e/curl.go @@ -40,20 +40,20 @@ type CURLReq struct { // CURLPrefixArgs builds the beginning of a curl command for a given key // addressed to a random URL in the given cluster. -func CURLPrefixArgs(clus *EtcdProcessCluster, method string, req CURLReq) []string { +func CURLPrefixArgs(cfg *EtcdProcessClusterConfig, member EtcdProcess, method string, req CURLReq) []string { var ( cmdArgs = []string{"curl"} - acurl = clus.Procs[rand.Intn(clus.Cfg.ClusterSize)].Config().Acurl + acurl = member.Config().Acurl ) if req.MetricsURLScheme != "https" { if req.IsTLS { - if clus.Cfg.ClientTLS != ClientTLSAndNonTLS { + if cfg.ClientTLS != ClientTLSAndNonTLS { panic("should not use cURLPrefixArgsUseTLS when serving only TLS or non-TLS") } cmdArgs = append(cmdArgs, "--cacert", CaPath, "--cert", CertPath, "--key", PrivateKeyPath) - acurl = ToTLS(clus.Procs[rand.Intn(clus.Cfg.ClusterSize)].Config().Acurl) - } else if clus.Cfg.ClientTLS == ClientTLS { - if !clus.Cfg.NoCN { + acurl = ToTLS(member.Config().Acurl) + } else if cfg.ClientTLS == ClientTLS { + if !cfg.NoCN { cmdArgs = append(cmdArgs, "--cacert", CaPath, "--cert", CertPath, "--key", PrivateKeyPath) } else { cmdArgs = append(cmdArgs, "--cacert", CaPath, "--cert", CertPath3, "--key", PrivateKeyPath3) @@ -61,7 +61,7 @@ func CURLPrefixArgs(clus *EtcdProcessCluster, method string, req CURLReq) []stri } } if req.MetricsURLScheme != "" { - acurl = clus.Procs[rand.Intn(clus.Cfg.ClusterSize)].EndpointsMetrics()[0] + acurl = member.EndpointsMetrics()[0] } ep := acurl + req.Endpoint @@ -94,13 +94,13 @@ func CURLPrefixArgs(clus *EtcdProcessCluster, method string, req CURLReq) []stri } func CURLPost(clus *EtcdProcessCluster, req CURLReq) error { - return SpawnWithExpect(CURLPrefixArgs(clus, "POST", req), req.Expected) + return SpawnWithExpect(CURLPrefixArgs(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "POST", req), req.Expected) } func CURLPut(clus *EtcdProcessCluster, req CURLReq) error { - return SpawnWithExpect(CURLPrefixArgs(clus, "PUT", req), req.Expected) + return SpawnWithExpect(CURLPrefixArgs(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "PUT", req), req.Expected) } func CURLGet(clus *EtcdProcessCluster, req CURLReq) error { - return SpawnWithExpect(CURLPrefixArgs(clus, "GET", req), req.Expected) + return SpawnWithExpect(CURLPrefixArgs(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "GET", req), req.Expected) } From 37af4b59f291a1bd67874e91bb432f9f44c65ed0 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Feb 2022 10:30:36 +0100 Subject: [PATCH 4/8] server: Move fatal calls out of internal function to make stacktrace cleaner --- tests/e2e/cluster_downgrade_test.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 11eaf76424aa..c9a5b49d039c 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -103,28 +103,33 @@ func stopEtcd(t *testing.T, ep e2e.EtcdProcess) { func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.Versions) { t.Log("Validate version") // Two separate calls to expect as it doesn't support multiple matches on the same line + var err error e2e.ExecuteWithTimeout(t, 20*time.Second, func() { if expect.Server != "" { - err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) if err != nil { - t.Fatal(err) + return } } if expect.Cluster != "" { - err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) if err != nil { - t.Fatal(err) + return } } }) + if err != nil { + t.Fatal(err) + } } func expectLog(t *testing.T, epc *e2e.EtcdProcessCluster, expectLog string) { t.Helper() + var err error e2e.ExecuteWithTimeout(t, 30*time.Second, func() { - _, err := epc.Procs[0].Logs().Expect(expectLog) - if err != nil { - t.Fatal(err) - } + _, err = epc.Procs[0].Logs().Expect(expectLog) }) + if err != nil { + t.Fatal(err) + } } From 40d4f15e20374d3e4d9780224aefca0f5b782dfb Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Feb 2022 10:40:23 +0100 Subject: [PATCH 5/8] tests: Improve logging in downgrade tests --- tests/e2e/cluster_downgrade_test.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index c9a5b49d039c..66b27e78d001 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -39,21 +39,33 @@ func TestDowngradeUpgrade(t *testing.T) { e2e.BeforeTest(t) + t.Logf("Create cluster with version %s", currentVersionStr) epc := newCluster(t, currentEtcdBinary) validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + t.Logf("Cluster created") + t.Logf("etcdctl downgrade enable %s", lastVersion) downgradeEnable(t, epc, lastVersion) + + t.Log("Downgrade enabled, validating if cluster is ready for downgrade") expectLog(t, epc, "The server is ready to downgrade") validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: currentVersionStr}) + t.Log("Cluster is ready for downgrade") + t.Log("Starting downgrade process") stopEtcd(t, epc.Procs[0]) startEtcd(t, epc, lastReleaseBinary) expectLog(t, epc, "the cluster has been downgraded") + t.Log("All members downgraded, validating downgrade") validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: lastVersionStr}) + t.Log("Downgrade complete") + t.Log("Starting upgrade process") stopEtcd(t, epc.Procs[0]) startEtcd(t, epc, currentEtcdBinary) + t.Log("All members upgraded, validating upgrade") validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + t.Log("Upgrade complete") } func newCluster(t *testing.T, execPath string) *e2e.EtcdProcessCluster { @@ -83,7 +95,6 @@ func startEtcd(t *testing.T, epc *e2e.EtcdProcessCluster, execPath string) { } func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Version) { - t.Log("etcdctl downgrade...") c := e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3()) e2e.ExecuteWithTimeout(t, 20*time.Second, func() { err := c.DowngradeEnable(ver.String()) @@ -94,14 +105,12 @@ func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Versi } func stopEtcd(t *testing.T, ep e2e.EtcdProcess) { - t.Log("Stopping the server...") if err := ep.Stop(); err != nil { t.Fatal(err) } } func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.Versions) { - t.Log("Validate version") // Two separate calls to expect as it doesn't support multiple matches on the same line var err error e2e.ExecuteWithTimeout(t, 20*time.Second, func() { From 07f262d4993110697667de7f0ea4a4b1d968f699 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Feb 2022 12:09:19 +0100 Subject: [PATCH 6/8] tests: Retry validating version during downgrades --- tests/e2e/cluster_downgrade_test.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 66b27e78d001..a3a805a6ebee 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -114,17 +114,22 @@ func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.V // Two separate calls to expect as it doesn't support multiple matches on the same line var err error e2e.ExecuteWithTimeout(t, 20*time.Second, func() { - if expect.Server != "" { - err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) - if err != nil { - return + for { + if expect.Server != "" { + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) + if err != nil { + time.Sleep(time.Second) + continue + } } - } - if expect.Cluster != "" { - err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) - if err != nil { - return + if expect.Cluster != "" { + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) + if err != nil { + time.Sleep(time.Second) + continue + } } + break } }) if err != nil { From 25f6f21e520105d6b987da7e71f220578094ffed Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Feb 2022 13:02:06 +0100 Subject: [PATCH 7/8] server: Implement HA downgrade test --- tests/e2e/cluster_downgrade_test.go | 77 ++++++++++++++++++++--------- tests/framework/e2e/cluster.go | 24 +++++++++ 2 files changed, 78 insertions(+), 23 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index a3a805a6ebee..dac0bce6224f 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -16,7 +16,6 @@ package e2e import ( "fmt" - "math/rand" "testing" "time" @@ -26,7 +25,15 @@ import ( "go.etcd.io/etcd/tests/v3/framework/e2e" ) -func TestDowngradeUpgrade(t *testing.T) { +func TestDowngradeUpgradeClusterOf1(t *testing.T) { + testDowngradeUpgrade(t, 1) +} + +func TestDowngradeUpgradeClusterOf3(t *testing.T) { + testDowngradeUpgrade(t, 3) +} + +func testDowngradeUpgrade(t *testing.T, clusterSize int) { currentEtcdBinary := e2e.BinDir + "/etcd" lastReleaseBinary := e2e.BinDir + "/etcd-last-release" if !fileutil.Exist(lastReleaseBinary) { @@ -40,38 +47,52 @@ func TestDowngradeUpgrade(t *testing.T) { e2e.BeforeTest(t) t.Logf("Create cluster with version %s", currentVersionStr) - epc := newCluster(t, currentEtcdBinary) - validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + epc := newCluster(t, currentEtcdBinary, clusterSize) + for i := 0; i < len(epc.Procs); i++ { + validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + } t.Logf("Cluster created") t.Logf("etcdctl downgrade enable %s", lastVersion) downgradeEnable(t, epc, lastVersion) t.Log("Downgrade enabled, validating if cluster is ready for downgrade") - expectLog(t, epc, "The server is ready to downgrade") - validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: currentVersionStr}) + for i := 0; i < len(epc.Procs); i++ { + expectLog(t, epc.Procs[i], "The server is ready to downgrade") + validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{Cluster: lastVersionStr, Server: currentVersionStr}) + } t.Log("Cluster is ready for downgrade") t.Log("Starting downgrade process") - stopEtcd(t, epc.Procs[0]) - startEtcd(t, epc, lastReleaseBinary) - expectLog(t, epc, "the cluster has been downgraded") + for i := 0; i < len(epc.Procs); i++ { + t.Logf("Downgrading member %d", i) + stopEtcd(t, epc.Procs[i]) + startEtcd(t, epc.Procs[i], lastReleaseBinary) + } t.Log("All members downgraded, validating downgrade") - validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: lastVersionStr}) + expectLog(t, leader(t, epc), "the cluster has been downgraded") + for i := 0; i < len(epc.Procs); i++ { + validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{Cluster: lastVersionStr, Server: lastVersionStr}) + } t.Log("Downgrade complete") t.Log("Starting upgrade process") - stopEtcd(t, epc.Procs[0]) - startEtcd(t, epc, currentEtcdBinary) + for i := 0; i < len(epc.Procs); i++ { + t.Logf("Upgrading member %d", i) + stopEtcd(t, epc.Procs[i]) + startEtcd(t, epc.Procs[i], currentEtcdBinary) + } t.Log("All members upgraded, validating upgrade") - validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + for i := 0; i < len(epc.Procs); i++ { + validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + } t.Log("Upgrade complete") } -func newCluster(t *testing.T, execPath string) *e2e.EtcdProcessCluster { +func newCluster(t *testing.T, execPath string, clusterSize int) *e2e.EtcdProcessCluster { epc, err := e2e.NewEtcdProcessCluster(t, &e2e.EtcdProcessClusterConfig{ ExecPath: execPath, - ClusterSize: 1, + ClusterSize: clusterSize, InitialToken: "new", KeepDataDir: true, }) @@ -86,9 +107,9 @@ func newCluster(t *testing.T, execPath string) *e2e.EtcdProcessCluster { return epc } -func startEtcd(t *testing.T, epc *e2e.EtcdProcessCluster, execPath string) { - epc.Procs[0].Config().ExecPath = execPath - err := epc.Procs[0].Restart() +func startEtcd(t *testing.T, ep e2e.EtcdProcess, execPath string) { + ep.Config().ExecPath = execPath + err := ep.Restart() if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } @@ -110,20 +131,20 @@ func stopEtcd(t *testing.T, ep e2e.EtcdProcess) { } } -func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.Versions) { +func validateVersion(t *testing.T, cfg *e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess, expect version.Versions) { // Two separate calls to expect as it doesn't support multiple matches on the same line var err error e2e.ExecuteWithTimeout(t, 20*time.Second, func() { for { if expect.Server != "" { - err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(cfg, member, "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) if err != nil { time.Sleep(time.Second) continue } } if expect.Cluster != "" { - err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(cfg, member, "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) if err != nil { time.Sleep(time.Second) continue @@ -137,13 +158,23 @@ func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.V } } -func expectLog(t *testing.T, epc *e2e.EtcdProcessCluster, expectLog string) { +func expectLog(t *testing.T, ep e2e.EtcdProcess, expectLog string) { t.Helper() var err error e2e.ExecuteWithTimeout(t, 30*time.Second, func() { - _, err = epc.Procs[0].Logs().Expect(expectLog) + _, err = ep.Logs().Expect(expectLog) }) if err != nil { t.Fatal(err) } } + +func leader(t *testing.T, epc *e2e.EtcdProcessCluster) e2e.EtcdProcess { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + leader, err := epc.Leader(ctx) + cancel() + if err != nil { + t.Fatal(err) + } + return leader +} diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index e1de1951047f..825bb7ba5979 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -15,6 +15,7 @@ package e2e import ( + "context" "fmt" "net/url" "os" @@ -23,6 +24,7 @@ import ( "testing" "time" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -497,3 +499,25 @@ func (epc *EtcdProcessCluster) WithStopSignal(sig os.Signal) (ret os.Signal) { } return ret } +func (epc *EtcdProcessCluster) Leader(ctx context.Context) (EtcdProcess, error) { + for i := 0; i < len(epc.Procs); i++ { + endpoints := epc.Procs[i].EndpointsV3() + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 3 * time.Second, + }) + if err != nil { + return nil, err + } + defer cli.Close() + resp, err := cli.Status(ctx, endpoints[0]) + if err != nil { + return nil, err + } + if resp.Header.GetMemberId() == resp.Leader { + return epc.Procs[i], nil + } + } + + return nil, fmt.Errorf("Leader not found") +} From 963a78d9b3cff5e5dc1d1bc7709b49b8b18c062c Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 21 Feb 2022 15:46:36 +0100 Subject: [PATCH 8/8] tests: Apply suggestions from code-review --- tests/e2e/cluster_downgrade_test.go | 38 ++++++++++++++++++++++------- tests/framework/e2e/cluster.go | 24 ------------------ 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index dac0bce6224f..3b39774fbfcf 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -15,6 +15,7 @@ package e2e import ( + "context" "fmt" "testing" "time" @@ -22,6 +23,7 @@ import ( "github.com/coreos/go-semver/semver" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/fileutil" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" ) @@ -53,7 +55,7 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int) { } t.Logf("Cluster created") - t.Logf("etcdctl downgrade enable %s", lastVersion) + t.Logf("etcdctl downgrade enable %s", lastVersionStr) downgradeEnable(t, epc, lastVersion) t.Log("Downgrade enabled, validating if cluster is ready for downgrade") @@ -63,9 +65,9 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int) { } t.Log("Cluster is ready for downgrade") - t.Log("Starting downgrade process") + t.Logf("Starting downgrade process to %q", lastVersionStr) for i := 0; i < len(epc.Procs); i++ { - t.Logf("Downgrading member %d", i) + t.Logf("Downgrading member %d by running %s binary", i, lastReleaseBinary) stopEtcd(t, epc.Procs[i]) startEtcd(t, epc.Procs[i], lastReleaseBinary) } @@ -76,11 +78,14 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int) { } t.Log("Downgrade complete") - t.Log("Starting upgrade process") + t.Logf("Starting upgrade process to %q", currentVersionStr) for i := 0; i < len(epc.Procs); i++ { t.Logf("Upgrading member %d", i) stopEtcd(t, epc.Procs[i]) startEtcd(t, epc.Procs[i], currentEtcdBinary) + if i+1 < len(epc.Procs) { + validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{Cluster: lastVersionStr, Server: currentVersionStr}) + } } t.Log("All members upgraded, validating upgrade") for i := 0; i < len(epc.Procs); i++ { @@ -171,10 +176,25 @@ func expectLog(t *testing.T, ep e2e.EtcdProcess, expectLog string) { func leader(t *testing.T, epc *e2e.EtcdProcessCluster) e2e.EtcdProcess { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - leader, err := epc.Leader(ctx) - cancel() - if err != nil { - t.Fatal(err) + defer cancel() + for i := 0; i < len(epc.Procs); i++ { + endpoints := epc.Procs[i].EndpointsV3() + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 3 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + defer cli.Close() + resp, err := cli.Status(ctx, endpoints[0]) + if err != nil { + t.Fatal(err) + } + if resp.Header.GetMemberId() == resp.Leader { + return epc.Procs[i] + } } - return leader + t.Fatal("Leader not found") + return nil } diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index 825bb7ba5979..e1de1951047f 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -15,7 +15,6 @@ package e2e import ( - "context" "fmt" "net/url" "os" @@ -24,7 +23,6 @@ import ( "testing" "time" - clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -499,25 +497,3 @@ func (epc *EtcdProcessCluster) WithStopSignal(sig os.Signal) (ret os.Signal) { } return ret } -func (epc *EtcdProcessCluster) Leader(ctx context.Context) (EtcdProcess, error) { - for i := 0; i < len(epc.Procs); i++ { - endpoints := epc.Procs[i].EndpointsV3() - cli, err := clientv3.New(clientv3.Config{ - Endpoints: endpoints, - DialTimeout: 3 * time.Second, - }) - if err != nil { - return nil, err - } - defer cli.Close() - resp, err := cli.Status(ctx, endpoints[0]) - if err != nil { - return nil, err - } - if resp.Header.GetMemberId() == resp.Leader { - return epc.Procs[i], nil - } - } - - return nil, fmt.Errorf("Leader not found") -}