From 6566b6be0030d92d7c7b99a43ad648ae25c01a74 Mon Sep 17 00:00:00 2001
From: Jackson Owens <jackson@cockroachlabs.com>
Date: Sat, 2 Mar 2024 12:13:44 -0500
Subject: [PATCH] metamorphic: inject random IO latency

Inject random latency in some IO operations in some random configurations of
the metamorphic test. This helps exercise code that's dependent on IO latency,
most notably WAL failover.

Close #2482.
---
 metamorphic/meta.go    |  9 +++++++++
 metamorphic/options.go | 41 +++++++++++++++++++++++++++++++++++++++--
 2 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/metamorphic/meta.go b/metamorphic/meta.go
index 1d681ea4cf..fa865aca11 100644
--- a/metamorphic/meta.go
+++ b/metamorphic/meta.go
@@ -479,6 +479,15 @@ func RunOnce(t TestingT, runDir string, seed uint64, historyPath string, rOpts .
 	} else {
 		opts.Cleaner = base.ArchiveCleaner{}
 	}
+	// Wrap the filesystem with a VFS that will inject random latency if
+	// the test options require it.
+	if testOpts.ioLatencyProbability > 0.0 {
+		opts.FS = errorfs.Wrap(opts.FS, errorfs.RandomLatency(
+			errorfs.Randomly(testOpts.ioLatencyProbability, testOpts.ioLatencySeed),
+			testOpts.ioLatencyMean,
+			testOpts.ioLatencySeed,
+		))
+	}
 
 	// Wrap the filesystem with one that will inject errors into read
 	// operations with *errorRate probability.
diff --git a/metamorphic/options.go b/metamorphic/options.go
index 36ad1fd0f4..24a21c7a0f 100644
--- a/metamorphic/options.go
+++ b/metamorphic/options.go
@@ -125,6 +125,27 @@ func parseOptions(
 				}
 				opts.seedEFOS = v
 				return true
+			case "TestOptions.io_latency_mean":
+				v, err := time.ParseDuration(value)
+				if err != nil {
+					panic(err)
+				}
+				opts.ioLatencyMean = v
+				return true
+			case "TestOptions.io_latency_probability":
+				v, err := strconv.ParseFloat(value, 64)
+				if err != nil {
+					panic(err)
+				}
+				opts.ioLatencyProbability = v
+				return true
+			case "TestOptions.io_latency_seed":
+				v, err := strconv.ParseInt(value, 10, 64)
+				if err != nil {
+					panic(err)
+				}
+				opts.ioLatencySeed = v
+				return true
 			case "TestOptions.ingest_split":
 				opts.ingestSplit = true
 				opts.Opts.Experimental.IngestSplit = func() bool {
@@ -216,6 +237,11 @@ func optionsToString(opts *TestOptions) string {
 	if opts.ingestSplit {
 		fmt.Fprintf(&buf, "  ingest_split=%v\n", opts.ingestSplit)
 	}
+	if opts.ioLatencyProbability > 0.0 {
+		fmt.Fprintf(&buf, "  io_latency_mean=%s\n", opts.ioLatencyMean)
+		fmt.Fprintf(&buf, "  io_latency_probability=%f\n", opts.ioLatencyProbability)
+		fmt.Fprintf(&buf, "  io_latency_seed=%d\n", opts.ioLatencySeed)
+	}
 	if opts.useSharedReplicate {
 		fmt.Fprintf(&buf, "  use_shared_replicate=%v\n", opts.useSharedReplicate)
 	}
@@ -317,6 +343,13 @@ type TestOptions struct {
 	// are actually created as EventuallyFileOnlySnapshots is deterministically
 	// derived from the seed and the operation index.
 	seedEFOS uint64
+	// If nonzero, enables the injection of random IO latency. The mechanics of
+	// a Pebble operation can be very timing dependent, so artificial latency
+	// can ensure a wide variety of mechanics are exercised. Additionally,
+	// exercising some mechanics such as WAL failover require IO latency.
+	ioLatencyProbability float64
+	ioLatencySeed        int64
+	ioLatencyMean        time.Duration
 	// Enables ingest splits. Saved here for serialization as Options does not
 	// serialize this.
 	ingestSplit bool
@@ -575,8 +608,6 @@ func RandomOptions(
 	}
 
 	// Half the time enable WAL failover.
-	// TODO(jackson): Add I/O latency injection (#2482). Until then WAL failover
-	// will rarely trigger.
 	if rng.Intn(2) == 0 {
 		// Use 10x longer durations when writing directly to FS; we don't want
 		// WAL failover to trigger excessively frequently.
@@ -668,6 +699,12 @@ func RandomOptions(
 	// sufficient.
 	testOpts.useDisk = false
 	testOpts.strictFS = rng.Intn(2) != 0 // Only relevant for MemFS.
+	// 50% of the time, enable IO latency injection.
+	if rng.Intn(2) == 0 {
+		testOpts.ioLatencyMean = expRandDuration(rng, 3*time.Millisecond, time.Second)
+		testOpts.ioLatencyProbability = 0.01 * rng.Float64() // 0-1%
+		testOpts.ioLatencySeed = rng.Int63()
+	}
 	testOpts.Threads = rng.Intn(runtime.GOMAXPROCS(0)) + 1
 	if testOpts.strictFS {
 		opts.DisableWAL = false