metamorphic: randomly use disk for tests

Currently, by default, the metamorphic tests use a MemFS-backed DB by
default. The `-disk` flag can be used to override all tests to use a
disk-backed filesystem. The behavior is "all or nothing".

To better simulate use of Pebble with a real, disk-backed filesystem,
randomly generate test configurations that use disk.

Tweak the flags to allow specifying an override filesystem type. In the
case that "mem" or "disk" are specified, all metamorphic tests will use
that FS type, ignoring the randomly generated FS type. The default is to
use the randomly generated FS types. Disk-backed filesystems are used
10% of the time for the randomly generated configurations.

Related to cockroachdb/cockroach#69414.

internal/metamorphic/meta_test.go

@@ -40,8 +40,8 @@ import (
 var (
 	dir = flag.String("dir", "_meta",
 		"the directory storing test state")
-	disk = flag.Bool("disk", false,
-		"whether to use an in-mem DB or on-disk (in-mem is significantly faster)")
+	fs = flag.String("fs", "rand",
+		`force the tests to use either memory or disk-backed filesystems (valid: "mem", "disk", "rand")`)
 	// TODO: default error rate to a non-zero value. Currently, retrying is
 	// non-deterministic because of the Ierator.*WithLimit() methods since
 	// they may say that the Iterator is not valid, but be positioned at a
@@ -93,7 +93,7 @@ func testMetaRun(t *testing.T, runDir string, seed uint64) {
 
 	// Set up the filesystem to use for the test. Note that by default we use an
 	// in-memory FS.
-	if *disk && !testOpts.strictFS {
+	if testOpts.useDisk {
 		opts.FS = vfs.Default
 		require.NoError(t, os.RemoveAll(opts.FS.PathJoin(runDir, "data")))
 	} else {
@@ -134,7 +134,7 @@ func testMetaRun(t *testing.T, runDir string, seed uint64) {
 		}
 	}
 
-	if *keep && !*disk {
+	if *keep && !testOpts.useDisk {
 		m.maybeSaveData()
 	}
 }
@@ -211,12 +211,23 @@ func TestMeta(t *testing.T) {
 		runDir := filepath.Join(metaDir, path.Base(t.Name()))
 		require.NoError(t, os.MkdirAll(runDir, 0755))
 
+		// If the filesystem type was forced, all tests will use that value.
+		switch *fs {
+		case "rand":
+			// No-op. Use the generated value for the filesystem.
+		case "disk":
+			opts.useDisk = true
+		case "mem":
+			opts.useDisk = false
+		default:
+			t.Fatalf("unknown forced filesystem type: %q", *fs)
+		}
+
 		optionsPath := filepath.Join(runDir, "OPTIONS")
 		optionsStr := optionsToString(opts)
 		require.NoError(t, ioutil.WriteFile(optionsPath, []byte(optionsStr), 0644))
 
 		args := []string{
-			"-disk=" + fmt.Sprint(*disk),
 			"-keep=" + fmt.Sprint(*keep),
 			"-run-dir=" + runDir,
 			"-test.run=" + rootName + "$",

internal/metamorphic/options.go

@@ -8,7 +8,7 @@ import (
 	"github.com/cockroachdb/pebble"
 	"github.com/cockroachdb/pebble/bloom"
 	"github.com/cockroachdb/pebble/internal/cache"
-	"github.com/cockroachdb/pebble/vfs"
+	"github.com/cockroachdb/pebble/internal/randvar"
 	"golang.org/x/exp/rand"
 )
 
@@ -42,6 +42,9 @@ func parseOptions(opts *testOptions, data string) error {
 			case "TestOptions.replace_single_delete":
 				opts.replaceSingleDelete = true
 				return true
+			case "TestOptions.use_disk":
+				opts.useDisk = true
+				return true
 			default:
 				return false
 			}
@@ -53,7 +56,7 @@ func parseOptions(opts *testOptions, data string) error {
 
 func optionsToString(opts *testOptions) string {
 	str := opts.opts.String()
-	if opts.strictFS || opts.ingestUsingApply || opts.replaceSingleDelete {
+	if opts.strictFS || opts.ingestUsingApply || opts.replaceSingleDelete || opts.useDisk {
 		str += "\n[TestOptions]\n"
 	}
 	if opts.strictFS {
@@ -65,13 +68,15 @@ func optionsToString(opts *testOptions) string {
 	if opts.replaceSingleDelete {
 		str += "  replace_single_delete=true\n"
 	}
+	if opts.useDisk {
+		str += "  use_disk=true\n"
+	}
 	return str
 }
 
 func defaultOptions() *pebble.Options {
 	opts := &pebble.Options{
 		Comparer: &comparer,
-		FS:       vfs.NewMem(),
 		Levels: []pebble.LevelOptions{{
 			FilterPolicy: bloom.FilterPolicy(10),
 		}},
@@ -82,6 +87,7 @@ func defaultOptions() *pebble.Options {
 
 type testOptions struct {
 	opts     *pebble.Options
+	useDisk  bool
 	strictFS bool
 	// Use Batch.Apply rather than DB.Ingest.
 	ingestUsingApply bool
@@ -177,6 +183,10 @@ func standardOptions() []*testOptions {
 		20: `
 [TestOptions]
   replace_single_delete=true
+`,
+		21: `
+[TestOptions]
+  use_disk=true
 `,
 	}
 
@@ -221,7 +231,10 @@ func randomOptions(rng *rand.Rand) *testOptions {
 	opts.Levels = []pebble.LevelOptions{lopts}
 
 	testOpts.opts = opts
-	testOpts.strictFS = rng.Intn(2) != 0
+	testOpts.useDisk = randvar.NewWeighted(rng, 0.9, 0.1).Int() == 1 // Use disk 10% of the time.
+	if !testOpts.useDisk {
+		testOpts.strictFS = rng.Intn(2) != 0 // Only relevant for MemFS.
+	}
 	if testOpts.strictFS {
 		opts.DisableWAL = false
 	}