From 6c78768e64ba007f626e4de99d343ec701b5174d Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 29 Aug 2022 12:11:34 +0200 Subject: [PATCH] tests: Add linearizability tests scenario for #14370 Signed-off-by: Marek Siarkowicz --- .github/workflows/linearizability.yaml | 3 ++ Makefile | 4 +- scripts/build_lib.sh | 16 ++++++ tests/framework/e2e/cluster.go | 11 ++++- tests/linearizability/failpoints.go | 49 ++++++++++++++++++- tests/linearizability/linearizability_test.go | 8 +++ tools/mod/go.mod | 1 + tools/mod/go.sum | 2 + tools/mod/tools.go | 1 + 9 files changed, 91 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linearizability.yaml b/.github/workflows/linearizability.yaml index 1a1a2feafc9f..87a92c22dfa4 100644 --- a/.github/workflows/linearizability.yaml +++ b/.github/workflows/linearizability.yaml @@ -9,7 +9,10 @@ jobs: with: go-version: "1.19.1" - run: | + pushd tools/mod; go install go.etcd.io/gofail; popd mkdir -p /tmp/linearizability + FAILPOINTS=true make build + cat server/etcdserver/raft.fail.go EXPECT_DEBUG=true GO_TEST_FLAGS=-v RESULTS_DIR=/tmp/linearizability make test-linearizability - uses: actions/upload-artifact@v2 if: always() diff --git a/Makefile b/Makefile index 4c3bcfab52b8..6f8a27f8c170 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,8 @@ test-e2e-release: build PASSES="release e2e" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: test-linearizability -test-linearizability: build - PASSES="linearizability" ./scripts/test.sh $(GO_TEST_FLAGS) +test-linearizability: + FAILPOINTS=true PASSES="linearizability" ./scripts/test.sh $(GO_TEST_FLAGS) # Static analysis diff --git a/scripts/build_lib.sh b/scripts/build_lib.sh index ad5c732a5491..2bbbce8d3562 100755 --- a/scripts/build_lib.sh +++ b/scripts/build_lib.sh @@ -19,6 +19,22 @@ toggle_failpoints() { mode="$1" if command -v gofail >/dev/null 2>&1; then run gofail "$mode" server/etcdserver/ server/storage/backend/ + ( + cd ./server + run go get go.etcd.io/gofail/runtime + ) || exit 2 + ( + cd ./etcdutl + run go get go.etcd.io/gofail/runtime + ) || exit 2 + ( + cd ./etcdctl + run go get go.etcd.io/gofail/runtime + ) || exit 2 + ( + cd ./tests + run go get go.etcd.io/gofail/runtime + ) || exit 2 elif [[ "$mode" != "disable" ]]; then log_error "FAILPOINTS set but gofail not found" exit 1 diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index 5096df714f51..ff57f35ebee4 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -182,6 +182,7 @@ type EtcdProcessClusterConfig struct { CorruptCheckTime time.Duration CompactHashCheckEnabled bool CompactHashCheckTime time.Duration + GoFailEnabled bool } // NewEtcdProcessCluster launches a new cluster from etcd processes, returning @@ -388,12 +389,20 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in if cfg.CompactHashCheckTime != 0 { args = append(args, "--experimental-compact-hash-check-time", cfg.CompactHashCheckTime.String()) } + envVars := map[string]string{} + for key, value := range cfg.EnvVars { + envVars[key] = value + } + if cfg.GoFailEnabled { + port = (i+1)*10000 + 2381 + envVars["GOFAIL_HTTP"] = fmt.Sprintf("127.0.0.1:%d", port) + } return &EtcdServerProcessConfig{ lg: cfg.Logger, ExecPath: cfg.ExecPath, Args: args, - EnvVars: cfg.EnvVars, + EnvVars: envVars, TlsArgs: cfg.TlsArgs(), DataDirPath: dataDirPath, KeepDataDir: cfg.KeepDataDir, diff --git a/tests/linearizability/failpoints.go b/tests/linearizability/failpoints.go index eddbd53c8f3a..744eb1a8d451 100644 --- a/tests/linearizability/failpoints.go +++ b/tests/linearizability/failpoints.go @@ -15,14 +15,19 @@ package linearizability import ( + "bytes" "context" + "fmt" "math/rand" + "net/http" + "time" "go.etcd.io/etcd/tests/v3/framework/e2e" ) var ( - KillFailpoint Failpoint = killFailpoint{} + KillFailpoint Failpoint = killFailpoint{} + RaftBeforeSavePanic Failpoint = goFailpoint{"etcdserver/raftBeforeSave", "panic"} ) type Failpoint interface { @@ -47,3 +52,45 @@ func (f killFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster } return nil } + +type goFailpoint struct { + failpoint string + payload string +} + +func (f goFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster) error { + err := triggerGoFailpoint(f.failpoint, f.payload) + if err != nil { + return fmt.Errorf("failed to trigger failpoint %q, err: %v", f.failpoint, err) + } + err = clus.Procs[0].Wait() + if err != nil { + return err + } + err = clus.Procs[0].Start(ctx) + if err != nil { + return err + } + return nil +} + +func triggerGoFailpoint(failpoint, payload string) error { + // TODO: Send failpoints to different members + r, err := http.NewRequest("PUT", "http://127.0.0.1:12381/"+failpoint, bytes.NewBuffer([]byte(payload))) + if err != nil { + return err + } + resp, err := httpClient.Do(r) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("bad status code: %d", resp.StatusCode) + } + return nil +} + +var httpClient = http.Client{ + Timeout: 10 * time.Millisecond, +} diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index fc4d0629db3d..1f94b1754152 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -60,6 +60,14 @@ func TestLinearizability(t *testing.T) { ClusterSize: 3, }, }, + { + name: "Issue14370", + failpoint: RaftBeforeSavePanic, + config: e2e.EtcdProcessClusterConfig{ + ClusterSize: 1, + GoFailEnabled: true, + }, + }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { diff --git a/tools/mod/go.mod b/tools/mod/go.mod index a1453f0d15f5..2cca3bdd4c7a 100644 --- a/tools/mod/go.mod +++ b/tools/mod/go.mod @@ -15,6 +15,7 @@ require ( github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f github.com/mgechev/revive v1.2.1 github.com/mikefarah/yq/v4 v4.24.2 + go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 gotest.tools/gotestsum v1.7.0 gotest.tools/v3 v3.1.0 diff --git a/tools/mod/go.sum b/tools/mod/go.sum index e17785453461..f18f73c929ba 100644 --- a/tools/mod/go.sum +++ b/tools/mod/go.sum @@ -238,6 +238,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 h1:TcXBU/YdVROXQ7FUowVK1ih9gu2yi3YMLE+tQb9q964= +go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0/go.mod h1:bOzzUWJ5bNHifkNkoIN6Ydf/z/UPT0bYuPghFYVC8+4= go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 h1:QQiUXlqz+d96jyNG71NE+IGTgOK6Xlhdx+PzvfbLHlQ= go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116/go.mod h1:F9kog+iVAuvPJucb1dkYcDcbV0g4uyGEHllTP5NrXiw= go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs= diff --git a/tools/mod/tools.go b/tools/mod/tools.go index ccc3338eb85d..c2d1ce5819ec 100644 --- a/tools/mod/tools.go +++ b/tools/mod/tools.go @@ -34,6 +34,7 @@ import ( _ "github.com/mdempsky/unconvert" _ "github.com/mgechev/revive" _ "github.com/mikefarah/yq/v4" + _ "go.etcd.io/gofail" _ "go.etcd.io/protodoc" _ "gotest.tools/gotestsum" _ "gotest.tools/v3"