From 29fd03791c6d6051fed1ceb755e163bf0dc95c1c Mon Sep 17 00:00:00 2001
From: Marek Siarkowicz <siarkowicz@google.com>
Date: Tue, 17 Oct 2023 15:40:41 +0200
Subject: [PATCH] Implement terminate failpoint

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
---
 tests/robustness/failpoint/failpoint.go |  3 ++-
 tests/robustness/failpoint/kill.go      | 28 +++++++++++++++++++------
 tests/robustness/scenarios.go           |  2 +-
 3 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/tests/robustness/failpoint/failpoint.go b/tests/robustness/failpoint/failpoint.go
index 568a38f8abc7..6c463504d7ac 100644
--- a/tests/robustness/failpoint/failpoint.go
+++ b/tests/robustness/failpoint/failpoint.go
@@ -37,7 +37,7 @@ const (
 
 var (
 	allFailpoints = []Failpoint{
-		KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
+		ForcefulShutdown, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
 		DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
 		BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
 		BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
@@ -48,6 +48,7 @@ var (
 		BeforeApplyOneConfChangeSleep,
 		MemberReplace,
 		DropPeerNetwork,
+		GracefulShutdown,
 	}
 )
 
diff --git a/tests/robustness/failpoint/kill.go b/tests/robustness/failpoint/kill.go
index ef1d26e1394f..a2c49a828c4d 100644
--- a/tests/robustness/failpoint/kill.go
+++ b/tests/robustness/failpoint/kill.go
@@ -27,12 +27,13 @@ import (
 )
 
 var (
-	KillFailpoint Failpoint = killFailpoint{}
+	ForcefulShutdown Failpoint = forcefulShutdownFailpoint{}
+	GracefulShutdown Failpoint = gracefulShutdownFailpoint{}
 )
 
-type killFailpoint struct{}
+type forcefulShutdownFailpoint struct{}
 
-func (f killFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) error {
+func (f forcefulShutdownFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) error {
 	member := clus.Procs[rand.Int()%len(clus.Procs)]
 
 	for member.IsRunning() {
@@ -60,10 +61,25 @@ func (f killFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger,
 	return nil
 }
 
-func (f killFailpoint) Name() string {
-	return "Kill"
+func (f forcefulShutdownFailpoint) Name() string {
+	return "forceful-shutdown"
 }
 
-func (f killFailpoint) Available(e2e.EtcdProcessClusterConfig, e2e.EtcdProcess) bool {
+func (f forcefulShutdownFailpoint) Available(e2e.EtcdProcessClusterConfig, e2e.EtcdProcess) bool {
+	return true
+}
+
+type gracefulShutdownFailpoint struct{}
+
+func (f gracefulShutdownFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) error {
+	member := clus.Procs[rand.Int()%len(clus.Procs)]
+	return member.Restart(ctx)
+}
+
+func (f gracefulShutdownFailpoint) Name() string {
+	return "graceful-shutdown"
+}
+
+func (f gracefulShutdownFailpoint) Available(e2e.EtcdProcessClusterConfig, e2e.EtcdProcess) bool {
 	return true
 }
diff --git a/tests/robustness/scenarios.go b/tests/robustness/scenarios.go
index 7ada046f9d4c..3bdef33725ee 100644
--- a/tests/robustness/scenarios.go
+++ b/tests/robustness/scenarios.go
@@ -125,7 +125,7 @@ func scenarios(t *testing.T) []testScenario {
 	})
 	scenarios = append(scenarios, testScenario{
 		name:      "Issue13766",
-		failpoint: failpoint.KillFailpoint,
+		failpoint: failpoint.ForcefulShutdown,
 		profile:   traffic.HighTrafficProfile,
 		traffic:   traffic.EtcdPut,
 		cluster: *e2e.NewConfig(