Skip to content

Commit

Permalink
tests: Fix member id in CORRUPT alarm
Browse files Browse the repository at this point in the history
Signed-off-by: Marek Siarkowicz <[email protected]>
  • Loading branch information
serathius committed Jul 25, 2022
1 parent f0f750f commit 7645ac7
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 18 deletions.
25 changes: 12 additions & 13 deletions server/etcdserver/corrupt.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ type Hasher interface {
MemberId() types.ID
PeerHashByRev(int64) []*peerHashKVResp
LinearizableReadNotify(context.Context) error
TriggerCorruptAlarm(uint64)
TriggerCorruptAlarm(types.ID)
}

func NewCorruptionChecker(lg *zap.Logger, s *EtcdServer) *corruptionChecker {
Expand All @@ -72,7 +72,7 @@ func (h hasherAdapter) PeerHashByRev(rev int64) []*peerHashKVResp {
return h.EtcdServer.getPeerHashKVs(rev)
}

func (h hasherAdapter) TriggerCorruptAlarm(memberID uint64) {
func (h hasherAdapter) TriggerCorruptAlarm(memberID types.ID) {
h.EtcdServer.triggerCorruptAlarm(memberID)
}

Expand Down Expand Up @@ -178,7 +178,7 @@ func (cm *corruptionChecker) PeriodicCheck() error {
}

alarmed := false
mismatch := func(id uint64) {
mismatch := func(id types.ID) {
if alarmed {
return
}
Expand All @@ -196,7 +196,7 @@ func (cm *corruptionChecker) PeriodicCheck() error {
zap.Int64("compact-revision-2", h2.CompactRevision),
zap.Uint32("hash-2", h2.Hash),
)
mismatch(uint64(cm.hasher.MemberId()))
mismatch(cm.hasher.MemberId())
}

checkedCount := 0
Expand All @@ -205,17 +205,16 @@ func (cm *corruptionChecker) PeriodicCheck() error {
continue
}
checkedCount++
id := p.resp.Header.MemberId

// leader expects follower's latest revision less than or equal to leader's
if p.resp.Header.Revision > rev2 {
cm.lg.Warn(
"revision from follower must be less than or equal to leader's",
zap.Int64("leader-revision", rev2),
zap.Int64("follower-revision", p.resp.Header.Revision),
zap.String("follower-peer-id", types.ID(id).String()),
zap.String("follower-peer-id", p.id.String()),
)
mismatch(id)
mismatch(p.id)
}

// leader expects follower's latest compact revision less than or equal to leader's
Expand All @@ -224,9 +223,9 @@ func (cm *corruptionChecker) PeriodicCheck() error {
"compact revision from follower must be less than or equal to leader's",
zap.Int64("leader-compact-revision", h2.CompactRevision),
zap.Int64("follower-compact-revision", p.resp.CompactRevision),
zap.String("follower-peer-id", types.ID(id).String()),
zap.String("follower-peer-id", p.id.String()),
)
mismatch(id)
mismatch(p.id)
}

// follower's compact revision is leader's old one, then hashes must match
Expand All @@ -237,18 +236,18 @@ func (cm *corruptionChecker) PeriodicCheck() error {
zap.Uint32("leader-hash", h.Hash),
zap.Int64("follower-compact-revision", p.resp.CompactRevision),
zap.Uint32("follower-hash", p.resp.Hash),
zap.String("follower-peer-id", types.ID(id).String()),
zap.String("follower-peer-id", p.id.String()),
)
mismatch(id)
mismatch(p.id)
}
}
cm.lg.Info("finished peer corruption check", zap.Int("number-of-peers-checked", checkedCount))
return nil
}

func (s *EtcdServer) triggerCorruptAlarm(id uint64) {
func (s *EtcdServer) triggerCorruptAlarm(id types.ID) {
a := &pb.AlarmRequest{
MemberID: id,
MemberID: uint64(id),
Action: pb.AlarmRequest_ACTIVATE,
Alarm: pb.AlarmType_CORRUPT,
}
Expand Down
2 changes: 1 addition & 1 deletion server/etcdserver/corrupt_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ func (f *fakeHasher) LinearizableReadNotify(ctx context.Context) error {
return f.linearizableReadNotify
}

func (f *fakeHasher) TriggerCorruptAlarm(memberId uint64) {
func (f *fakeHasher) TriggerCorruptAlarm(memberId types.ID) {
f.actions = append(f.actions, fmt.Sprintf("TriggerCorruptAlarm(%d)", memberId))
f.alarmTriggered = true
}
12 changes: 10 additions & 2 deletions tests/e2e/corrupt_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,15 @@ func TestPeriodicCheckDetectsCorruption(t *testing.T) {
assert.NoError(t, err, "error on put")
}

members, err := cc.MemberList()
assert.NoError(t, err, "error on member list")
var memberID uint64
for _, m := range members.Members {
if m.Name == epc.Procs[0].Config().Name {
memberID = m.ID
}
}
assert.NotZero(t, memberID, "member not found")
epc.Procs[0].Stop()
err = testutil.CorruptBBolt(datadir.ToBackendFileName(epc.Procs[0].Config().DataDirPath))
assert.NoError(t, err)
Expand All @@ -130,6 +139,5 @@ func TestPeriodicCheckDetectsCorruption(t *testing.T) {
time.Sleep(checkTime * 11 / 10)
alarmResponse, err := cc.AlarmList()
assert.NoError(t, err, "error on alarm list")
// TODO: Investigate why MemberID is 0?
assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: 0}}, alarmResponse.Alarms)
assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: memberID}}, alarmResponse.Alarms)
}
3 changes: 1 addition & 2 deletions tests/integration/corrupt_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,5 @@ func TestPeriodicCheckDetectsCorruption(t *testing.T) {
time.Sleep(50 * time.Millisecond)
alarmResponse, err := cc.AlarmList(ctx)
assert.NoError(t, err, "error on alarm list")
// TODO: Investigate why MemberID is 0?
assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: 0}}, alarmResponse.Alarms)
assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: uint64(clus.Members[0].ID())}}, alarmResponse.Alarms)
}

0 comments on commit 7645ac7

Please sign in to comment.