From 6b034466aa0ac2b46fe01fb5bd2233946f46d453 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Wed, 24 Apr 2024 12:14:27 +0800 Subject: [PATCH 1/4] server/mvcc: introduce compactBeforeSetFinishedCompact failpoint Signed-off-by: Wei Fu --- server/mvcc/kvstore_compaction.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/mvcc/kvstore_compaction.go b/server/mvcc/kvstore_compaction.go index c7d343d5c34..89defbd9e79 100644 --- a/server/mvcc/kvstore_compaction.go +++ b/server/mvcc/kvstore_compaction.go @@ -59,6 +59,7 @@ func (s *store) scheduleCompaction(compactMainRev, prevCompactRev int64) (KeyVal } if len(keys) < s.cfg.CompactionBatchLimit { + // gofail: var compactBeforeSetFinishedCompact struct{} rbytes := make([]byte, 8+1+8) revToBytes(revision{main: compactMainRev}, rbytes) tx.UnsafePut(buckets.Meta, finishedCompactKeyName, rbytes) From 7beff571077f94ee9e0f12d409b898756af0b0f1 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Wed, 17 Apr 2024 20:31:48 +0800 Subject: [PATCH 2/4] tests/e2e: reproduce #17780 Signed-off-by: Wei Fu (cherry picked from commit 71733911544f8fce6d06d2a8e9cca0944b3659be) Signed-off-by: Wei Fu --- tests/e2e/reproduce_17780_test.go | 115 ++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 tests/e2e/reproduce_17780_test.go diff --git a/tests/e2e/reproduce_17780_test.go b/tests/e2e/reproduce_17780_test.go new file mode 100644 index 00000000000..697ef0e0374 --- /dev/null +++ b/tests/e2e/reproduce_17780_test.go @@ -0,0 +1,115 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/pkg/v3/stringutil" + "go.etcd.io/etcd/tests/v3/framework/e2e" +) + +// TestReproduce17780 reproduces the issue: https://github.com/etcd-io/etcd/issues/17780. +func TestReproduce17780(t *testing.T) { + e2e.BeforeTest(t) + + compactionBatchLimit := 10 + + ctx := context.TODO() + clus, cerr := e2e.NewEtcdProcessCluster(t, + &e2e.EtcdProcessClusterConfig{ + ClusterSize: 3, + GoFailEnabled: true, + SnapshotCount: 1000, + CompactionBatchLimit: compactionBatchLimit, + WatchProcessNotifyInterval: 100 * time.Millisecond, + }, + ) + require.NoError(t, cerr) + + t.Cleanup(func() { require.NoError(t, clus.Stop()) }) + + leaderIdx := clus.WaitLeader(t) + targetIdx := (leaderIdx + 1) % clus.Cfg.ClusterSize + + cli := newClient(t, clus.Procs[targetIdx].EndpointsGRPC(), e2e.ClientNonTLS, false) + + // Revision: 2 -> 8 for new keys + n := compactionBatchLimit - 2 + valueSize := 16 + for i := 2; i <= n; i++ { + _, err := cli.Put(ctx, fmt.Sprintf("%d", i), stringutil.RandString(uint(valueSize))) + require.NoError(t, err) + } + + // Revision: 9 -> 11 for delete keys with compared revision + // + // We need last compaction batch is no-op and all the tombstones should + // be deleted in previous compaction batch. So that we just lost the + // finishedCompactRev after panic. + for i := 9; i <= compactionBatchLimit+1; i++ { + rev := i - 5 + key := fmt.Sprintf("%d", rev) + + _, err := cli.Delete(ctx, key) + require.NoError(t, err) + } + + require.NoError(t, clus.Procs[targetIdx].Failpoints().SetupHTTP(ctx, "compactBeforeSetFinishedCompact", `panic`)) + + _, err := cli.Compact(ctx, 11, clientv3.WithCompactPhysical()) + require.Error(t, err) + + require.Error(t, clus.Procs[targetIdx].Stop()) + // NOTE: The proc panics and exit code is 2. It's impossible to restart + // that etcd proc because last exit code is 2 and Restart() refuses to + // start new one. Using IsRunning() function is to cleanup status. + require.False(t, clus.Procs[targetIdx].IsRunning()) + require.NoError(t, clus.Procs[targetIdx].Restart()) + + // NOTE: We should not decrease the revision if there is no record + // about finished compact operation. + resp, err := cli.Get(ctx, fmt.Sprintf("%d", n)) + require.NoError(t, err) + assert.GreaterOrEqual(t, resp.Header.Revision, int64(11)) + + // Revision 4 should be deleted by compaction. + resp, err = cli.Get(ctx, fmt.Sprintf("%d", 4)) + require.NoError(t, err) + require.True(t, resp.Count == 0) + + next := 20 + for i := 12; i <= next; i++ { + _, err := cli.Put(ctx, fmt.Sprintf("%d", i), stringutil.RandString(uint(valueSize))) + require.NoError(t, err) + } + + expectedRevision := next + for procIdx, proc := range clus.Procs { + cli = newClient(t, proc.EndpointsGRPC(), e2e.ClientNonTLS, false) + resp, err := cli.Get(ctx, fmt.Sprintf("%d", next)) + require.NoError(t, err) + + assert.GreaterOrEqual(t, resp.Header.Revision, int64(expectedRevision), + fmt.Sprintf("LeaderIdx: %d, Current: %d", leaderIdx, procIdx)) + } +} From c06b17b9ffdf482b097359108948c0332d77bbf5 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Thu, 18 Apr 2024 22:50:45 +0800 Subject: [PATCH 3/4] server/storage: update currentRev if scheduledCompact > currentRev Signed-off-by: Wei Fu (cherry picked from commit 9ea234913a99670d18b66aa23915781f89713177) Signed-off-by: Wei Fu --- server/mvcc/kvstore.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/mvcc/kvstore.go b/server/mvcc/kvstore.go index 66e612b0a81..1e46e2ec33a 100644 --- a/server/mvcc/kvstore.go +++ b/server/mvcc/kvstore.go @@ -402,6 +402,17 @@ func (s *store) restore() error { } } + // If the latest revision was a tombstone revision and etcd just compacted + // it, but crashed right before persisting the FinishedCompactRevision, + // then it would lead to revision decreasing in bbolt db file. In such + // a scenario, we should adjust the current revision using the scheduled + // compact revision on bootstrap when etcd gets started again. + // + // See https://github.com/etcd-io/etcd/issues/17780#issuecomment-2061900231 + if s.currentRev < scheduledCompact { + s.currentRev = scheduledCompact + } + tx.Unlock() s.lg.Info("kvstore restored", zap.Int64("current-rev", s.currentRev)) From 0af22abc6c2183c4dfdbd8faed8bca1125743392 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Wed, 24 Apr 2024 11:08:33 +0800 Subject: [PATCH 4/4] server/mvcc: should update currentRev in revMu Signed-off-by: Wei Fu --- server/mvcc/kvstore.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/mvcc/kvstore.go b/server/mvcc/kvstore.go index 1e46e2ec33a..828d81549a9 100644 --- a/server/mvcc/kvstore.go +++ b/server/mvcc/kvstore.go @@ -380,6 +380,17 @@ func (s *store) restore() error { if s.currentRev < s.compactMainRev { s.currentRev = s.compactMainRev } + + // If the latest revision was a tombstone revision and etcd just compacted + // it, but crashed right before persisting the FinishedCompactRevision, + // then it would lead to revision decreasing in bbolt db file. In such + // a scenario, we should adjust the current revision using the scheduled + // compact revision on bootstrap when etcd gets started again. + // + // See https://github.com/etcd-io/etcd/issues/17780#issuecomment-2061900231 + if s.currentRev < scheduledCompact { + s.currentRev = scheduledCompact + } s.revMu.Unlock() } @@ -402,17 +413,6 @@ func (s *store) restore() error { } } - // If the latest revision was a tombstone revision and etcd just compacted - // it, but crashed right before persisting the FinishedCompactRevision, - // then it would lead to revision decreasing in bbolt db file. In such - // a scenario, we should adjust the current revision using the scheduled - // compact revision on bootstrap when etcd gets started again. - // - // See https://github.com/etcd-io/etcd/issues/17780#issuecomment-2061900231 - if s.currentRev < scheduledCompact { - s.currentRev = scheduledCompact - } - tx.Unlock() s.lg.Info("kvstore restored", zap.Int64("current-rev", s.currentRev))