diff --git a/compaction_test.go b/compaction_test.go index 4ea8595037..69548363c7 100644 --- a/compaction_test.go +++ b/compaction_test.go @@ -2900,7 +2900,7 @@ func TestCompactionErrorCleanup(t *testing.T) { mem := vfs.NewMem() ii := errorfs.OnIndex(math.MaxInt32) // start disabled opts := (&Options{ - FS: errorfs.Wrap(mem, ii), + FS: errorfs.Wrap(mem, errorfs.ErrInjected.If(ii)), Levels: make([]LevelOptions, numLevels), EventListener: &EventListener{ TableCreated: func(info TableCreateInfo) { @@ -3286,8 +3286,8 @@ func TestFlushError(t *testing.T) { // Error the first five times we try to write a sstable. var errorOps atomic.Int32 errorOps.Store(3) - fs := errorfs.Wrap(vfs.NewMem(), errorfs.InjectorFunc(func(op errorfs.Op, path string) error { - if op == errorfs.OpCreate && filepath.Ext(path) == ".sst" && errorOps.Add(-1) >= 0 { + fs := errorfs.Wrap(vfs.NewMem(), errorfs.InjectorFunc(func(op errorfs.Op) error { + if op.Kind == errorfs.OpCreate && filepath.Ext(op.Path) == ".sst" && errorOps.Add(-1) >= 0 { return errorfs.ErrInjected } return nil @@ -3704,19 +3704,24 @@ type createManifestErrorInjector struct { enabled atomic.Bool } +// TODO(jackson): Replace the createManifestErrorInjector with the composition +// of primitives defined in errorfs. This may require additional primitives. + +func (i *createManifestErrorInjector) String() string { return "MANIFEST-Creates" } + // enable enables error injection for the vfs.FS. func (i *createManifestErrorInjector) enable() { i.enabled.Store(true) } // MaybeError implements errorfs.Injector. -func (i *createManifestErrorInjector) MaybeError(op errorfs.Op, path string) error { +func (i *createManifestErrorInjector) MaybeError(op errorfs.Op) error { if !i.enabled.Load() { return nil } // This necessitates having a MaxManifestSize of 1, to reliably induce // logAndApply errors. - if strings.Contains(path, "MANIFEST") && op == errorfs.OpCreate { + if strings.Contains(op.Path, "MANIFEST") && op.Kind == errorfs.OpCreate { return errorfs.ErrInjected } return nil @@ -3884,6 +3889,11 @@ type WriteErrorInjector struct { enabled atomic.Bool } +// TODO(jackson): Replace WriteErrorInjector with use of primitives in errorfs, +// adding new primitives as necessary. + +func (i *WriteErrorInjector) String() string { return "FileWrites(ErrInjected)" } + // enable enables error injection for the vfs.FS. func (i *WriteErrorInjector) enable() { i.enabled.Store(true) @@ -3895,12 +3905,12 @@ func (i *WriteErrorInjector) disable() { } // MaybeError implements errorfs.Injector. -func (i *WriteErrorInjector) MaybeError(op errorfs.Op, path string) error { +func (i *WriteErrorInjector) MaybeError(op errorfs.Op) error { if !i.enabled.Load() { return nil } // Fail any future write. - if op == errorfs.OpFileWrite { + if op.Kind == errorfs.OpFileWrite { return errorfs.ErrInjected } return nil diff --git a/data_test.go b/data_test.go index fa1cc1ae26..2d7fe87a2b 100644 --- a/data_test.go +++ b/data_test.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/datadriven" "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/bloom" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/humanize" "github.com/cockroachdb/pebble/internal/keyspan" @@ -29,6 +30,7 @@ import ( "github.com/cockroachdb/pebble/objstorage/remote" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" + "github.com/cockroachdb/pebble/vfs/errorfs" "github.com/stretchr/testify/require" ) @@ -1447,3 +1449,75 @@ func runLSMCmd(td *datadriven.TestData, d *DB) string { } return d.mu.versions.currentVersion().String() } + +func parseDBOptionsArgs(opts *Options, args []datadriven.CmdArg) error { + for _, cmdArg := range args { + switch cmdArg.Key { + case "inject-errors": + injs := make([]errorfs.Injector, len(cmdArg.Vals)) + for i := 0; i < len(cmdArg.Vals); i++ { + inj, err := errorfs.ParseInjectorFromDSL(cmdArg.Vals[i]) + if err != nil { + return err + } + injs[i] = inj + } + opts.FS = errorfs.Wrap(opts.FS, errorfs.Any(injs...)) + case "enable-table-stats": + enable, err := strconv.ParseBool(cmdArg.Vals[0]) + if err != nil { + return errors.Errorf("%s: could not parse %q as bool: %s", cmdArg.Key, cmdArg.Vals[0], err) + } + opts.private.disableTableStats = !enable + case "format-major-version": + v, err := strconv.Atoi(cmdArg.Vals[0]) + if err != nil { + return err + } + // Override the DB version. + opts.FormatMajorVersion = FormatMajorVersion(v) + case "block-size": + v, err := strconv.Atoi(cmdArg.Vals[0]) + if err != nil { + return err + } + for i := range opts.Levels { + opts.Levels[i].BlockSize = v + } + case "index-block-size": + v, err := strconv.Atoi(cmdArg.Vals[0]) + if err != nil { + return err + } + for i := range opts.Levels { + opts.Levels[i].IndexBlockSize = v + } + case "target-file-size": + v, err := strconv.Atoi(cmdArg.Vals[0]) + if err != nil { + return err + } + for i := range opts.Levels { + opts.Levels[i].TargetFileSize = int64(v) + } + case "bloom-bits-per-key": + v, err := strconv.Atoi(cmdArg.Vals[0]) + if err != nil { + return err + } + fp := bloom.FilterPolicy(v) + opts.Filters = map[string]FilterPolicy{fp.Name(): fp} + for i := range opts.Levels { + opts.Levels[i].FilterPolicy = fp + } + case "merger": + switch cmdArg.Vals[0] { + case "appender": + opts.Merger = base.DefaultMerger + default: + return errors.Newf("unrecognized Merger %q\n", cmdArg.Vals[0]) + } + } + } + return nil +} diff --git a/error_test.go b/error_test.go index 68f546f5fe..8c6c03cf7d 100644 --- a/error_test.go +++ b/error_test.go @@ -136,7 +136,7 @@ func TestErrors(t *testing.T) { errorCounts := make(map[string]int) for i := int32(0); ; i++ { - fs := errorfs.Wrap(vfs.NewMem(), errorfs.OnIndex(i)) + fs := errorfs.Wrap(vfs.NewMem(), errorfs.ErrInjected.If(errorfs.OnIndex(i))) err := run(fs) if err == nil { t.Logf("success %d\n", i) @@ -166,8 +166,8 @@ func TestErrors(t *testing.T) { func TestRequireReadError(t *testing.T) { run := func(formatVersion FormatMajorVersion, index int32) (err error) { // Perform setup with error injection disabled as it involves writes/background ops. - inj := errorfs.OnIndex(-1) - fs := errorfs.Wrap(vfs.NewMem(), inj) + ii := errorfs.OnIndex(-1) + fs := errorfs.Wrap(vfs.NewMem(), errorfs.ErrInjected.If(ii)) opts := &Options{ FS: fs, Logger: panicLogger{}, @@ -210,7 +210,7 @@ func TestRequireReadError(t *testing.T) { } // Now perform foreground ops with error injection enabled. - inj.SetIndex(index) + ii.SetIndex(index) iter, _ := d.NewIter(nil) if err := iter.Error(); err != nil { return err @@ -237,7 +237,7 @@ func TestRequireReadError(t *testing.T) { // Reaching here implies all read operations succeeded. This // should only happen when we reached a large enough index at // which `errorfs.FS` did not return any error. - if i := inj.Index(); i < 0 { + if i := ii.Index(); i < 0 { t.Errorf("FS error injected %d ops ago went unreported", -i) } if numFound != 2 { @@ -367,8 +367,8 @@ func TestDBWALRotationCrash(t *testing.T) { memfs := vfs.NewStrictMem() var index atomic.Int32 - inj := errorfs.InjectorFunc(func(op errorfs.Op, _ string) error { - if op.OpKind() == errorfs.OpKindWrite && index.Add(-1) == -1 { + inj := errorfs.InjectorFunc(func(op errorfs.Op) error { + if op.Kind.ReadOrWrite() == errorfs.OpIsWrite && index.Add(-1) == -1 { memfs.SetIgnoreSyncs(true) } return nil diff --git a/ingest_test.go b/ingest_test.go index 6cfc9659b1..c8c76f608a 100644 --- a/ingest_test.go +++ b/ingest_test.go @@ -386,7 +386,7 @@ func TestIngestLinkFallback(t *testing.T) { src, err := mem.Create("source") require.NoError(t, err) - opts := &Options{FS: errorfs.Wrap(mem, errorfs.OnIndex(1))} + opts := &Options{FS: errorfs.Wrap(mem, errorfs.ErrInjected.If(errorfs.OnIndex(1)))} opts.EnsureDefaults().WithFSDefaults() objSettings := objstorageprovider.DefaultSettings(opts.FS, "") // Prevent the provider from listing the dir (where we may get an injected error). @@ -2182,9 +2182,9 @@ func TestIngestError(t *testing.T) { require.NoError(t, w.Set([]byte("d"), nil)) require.NoError(t, w.Close()) - inj := errorfs.OnIndex(-1) + ii := errorfs.OnIndex(-1) d, err := Open("", &Options{ - FS: errorfs.Wrap(mem, inj), + FS: errorfs.Wrap(mem, errorfs.ErrInjected.If(ii)), Logger: panicLogger{}, L0CompactionThreshold: 8, }) @@ -2211,7 +2211,7 @@ func TestIngestError(t *testing.T) { } }() - inj.SetIndex(i) + ii.SetIndex(i) err1 := d.Ingest([]string{"ext0"}) err2 := d.Ingest([]string{"ext1"}) err := firstError(err1, err2) @@ -2225,7 +2225,7 @@ func TestIngestError(t *testing.T) { // If the injector's index is non-negative, the i-th filesystem // operation was never executed. - if inj.Index() >= 0 { + if ii.Index() >= 0 { break } } @@ -3225,11 +3225,11 @@ func TestIngestValidation(t *testing.T) { cLoc: corruptionLocationNone, wantErr: errorfs.ErrInjected, wantErrType: errReportLocationBackgroundError, - errorfsInjector: errorfs.InjectorFunc(func(op errorfs.Op, path string) error { + errorfsInjector: errorfs.InjectorFunc(func(op errorfs.Op) error { // Inject an error on the first read-at operation on an sstable // (excluding the read on the sstable before ingestion has // linked it in). - if path != "ext" && op != errorfs.OpFileReadAt || filepath.Ext(path) != ".sst" { + if op.Path != "ext" && op.Kind != errorfs.OpFileReadAt || filepath.Ext(op.Path) != ".sst" { return nil } if errfsCounter.Add(1) == 1 { diff --git a/iterator_histories_test.go b/iterator_histories_test.go index bd178b2def..5f958ec2c9 100644 --- a/iterator_histories_test.go +++ b/iterator_histories_test.go @@ -13,8 +13,6 @@ import ( "github.com/cockroachdb/datadriven" "github.com/cockroachdb/errors" - "github.com/cockroachdb/pebble/bloom" - "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/testkeys" "github.com/cockroachdb/pebble/sstable" @@ -41,8 +39,9 @@ func TestIterHistories(t *testing.T) { iters[name] = it return it } + var opts *Options parseOpts := func(td *datadriven.TestData) (*Options, error) { - opts := &Options{ + opts = &Options{ FS: vfs.NewMem(), Comparer: testkeys.Comparer, FormatMajorVersion: FormatRangeKeys, @@ -50,61 +49,12 @@ func TestIterHistories(t *testing.T) { sstable.NewTestKeysBlockPropertyCollector, }, } + opts.DisableAutomaticCompactions = true opts.EnsureDefaults() opts.WithFSDefaults() - - for _, cmdArg := range td.CmdArgs { - switch cmdArg.Key { - case "format-major-version": - v, err := strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return nil, err - } - // Override the DB version. - opts.FormatMajorVersion = FormatMajorVersion(v) - case "block-size": - v, err := strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return nil, err - } - for i := range opts.Levels { - opts.Levels[i].BlockSize = v - } - case "index-block-size": - v, err := strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return nil, err - } - for i := range opts.Levels { - opts.Levels[i].IndexBlockSize = v - } - case "target-file-size": - v, err := strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return nil, err - } - for i := range opts.Levels { - opts.Levels[i].TargetFileSize = int64(v) - } - case "bloom-bits-per-key": - v, err := strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return nil, err - } - fp := bloom.FilterPolicy(v) - opts.Filters = map[string]FilterPolicy{fp.Name(): fp} - for i := range opts.Levels { - opts.Levels[i].FilterPolicy = fp - } - case "merger": - switch cmdArg.Vals[0] { - case "appender": - opts.Merger = base.DefaultMerger - default: - return nil, errors.Newf("unrecognized Merger %q\n", cmdArg.Vals[0]) - } - } + if err := parseDBOptionsArgs(opts, td.CmdArgs); err != nil { + return nil, err } return opts, nil } @@ -128,10 +78,11 @@ func TestIterHistories(t *testing.T) { datadriven.RunTest(t, path, func(t *testing.T, td *datadriven.TestData) string { switch td.Cmd { case "define": + var err error if err := cleanup(); err != nil { return err.Error() } - opts, err := parseOpts(td) + opts, err = parseOpts(td) if err != nil { return err.Error() } @@ -140,12 +91,23 @@ func TestIterHistories(t *testing.T) { return err.Error() } return runLSMCmd(td, d) - + case "reopen": + var err error + if err := cleanup(); err != nil { + return err.Error() + } + if err := parseDBOptionsArgs(opts, td.CmdArgs); err != nil { + return err.Error() + } + d, err = Open("", opts) + require.NoError(t, err) + return "" case "reset": + var err error if err := cleanup(); err != nil { return err.Error() } - opts, err := parseOpts(td) + opts, err = parseOpts(td) if err != nil { return err.Error() } diff --git a/metamorphic/meta.go b/metamorphic/meta.go index ccd219b129..419989bf9d 100644 --- a/metamorphic/meta.go +++ b/metamorphic/meta.go @@ -453,7 +453,7 @@ func RunOnce(t TestingT, runDir string, seed uint64, historyPath string, rOpts . // Wrap the filesystem with one that will inject errors into read // operations with *errorRate probability. - opts.FS = errorfs.Wrap(opts.FS, errorfs.WithProbability(errorfs.OpKindRead, runOpts.errorRate)) + opts.FS = errorfs.Wrap(opts.FS, errorfs.WithProbability(errorfs.OpIsRead, runOpts.errorRate)) if opts.WALDir != "" { opts.WALDir = opts.FS.PathJoin(runDir, opts.WALDir) diff --git a/open_test.go b/open_test.go index 110029906d..fae3237aac 100644 --- a/open_test.go +++ b/open_test.go @@ -927,17 +927,17 @@ func TestCrashOpenCrashAfterWALCreation(t *testing.T) { { var walCreated, dirSynced atomic.Bool d, err := Open("", &Options{ - FS: errorfs.Wrap(fs, errorfs.InjectorFunc(func(op errorfs.Op, path string) error { + FS: errorfs.Wrap(fs, errorfs.InjectorFunc(func(op errorfs.Op) error { if dirSynced.Load() { fs.SetIgnoreSyncs(true) } - if op == errorfs.OpCreate && filepath.Ext(path) == ".log" { + if op.Kind == errorfs.OpCreate && filepath.Ext(op.Path) == ".log" { walCreated.Store(true) } // Record when there's a sync of the data directory after the // WAL was created. The data directory will have an empty // path because that's what we passed into Open. - if op == errorfs.OpFileSync && path == "" && walCreated.Load() { + if op.Kind == errorfs.OpFileSync && op.Path == "" && walCreated.Load() { dirSynced.Store(true) } return nil diff --git a/record/log_writer_test.go b/record/log_writer_test.go index 9a70a624b3..9fce1a7162 100644 --- a/record/log_writer_test.go +++ b/record/log_writer_test.go @@ -515,8 +515,8 @@ func valueAtQuantileWindowed(histogram *prometheusgo.Histogram, q float64) float // blocked. func TestQueueWALBlocks(t *testing.T) { blockWriteCh := make(chan struct{}, 1) - f := errorfs.WrapFile(vfstest.DiscardFile, errorfs.InjectorFunc(func(op errorfs.Op, path string) error { - if op == errorfs.OpFileWrite { + f := errorfs.WrapFile(vfstest.DiscardFile, errorfs.InjectorFunc(func(op errorfs.Op) error { + if op.Kind == errorfs.OpFileWrite { <-blockWriteCh } return nil @@ -556,8 +556,8 @@ func BenchmarkQueueWALBlocks(b *testing.B) { for j := 0; j < b.N; j++ { b.StopTimer() blockWriteCh := make(chan struct{}, 1) - f := errorfs.WrapFile(vfstest.DiscardFile, errorfs.InjectorFunc(func(op errorfs.Op, path string) error { - if op == errorfs.OpFileWrite { + f := errorfs.WrapFile(vfstest.DiscardFile, errorfs.InjectorFunc(func(op errorfs.Op) error { + if op.Kind == errorfs.OpFileWrite { <-blockWriteCh } return nil diff --git a/sstable/reader_test.go b/sstable/reader_test.go index 4549094789..320ce3b4e9 100644 --- a/sstable/reader_test.go +++ b/sstable/reader_test.go @@ -647,7 +647,7 @@ func TestInjectedErrors(t *testing.T) { f, err := vfs.Default.Open(filepath.FromSlash(prebuiltSST)) require.NoError(t, err) - r, err := newReader(errorfs.WrapFile(f, errorfs.OnIndex(int32(i))), ReaderOptions{}) + r, err := newReader(errorfs.WrapFile(f, errorfs.ErrInjected.If(errorfs.OnIndex(int32(i)))), ReaderOptions{}) if err != nil { return firstError(err, f.Close()) } diff --git a/testdata/iter_histories/errors b/testdata/iter_histories/errors new file mode 100644 index 0000000000..bfca56376e --- /dev/null +++ b/testdata/iter_histories/errors @@ -0,0 +1,59 @@ +reset +---- + +batch commit +set a a +set b b +set c c +set d d +---- +committed 4 keys + +# Scan forward + +combined-iter +seek-ge a +next +next +next +next +---- +a: (a, .) +b: (b, .) +c: (c, .) +d: (d, .) +. + +reopen +---- + +combined-iter +first +next +next +next +next +---- +a: (a, .) +b: (b, .) +c: (c, .) +d: (d, .) +. + +reopen enable-table-stats=false inject-errors=((ErrInjected (And Reads (PathMatch "*.sst") (OnIndex 4)))) +---- + +combined-iter +first +first +next +next +next +next +---- +err=pebble: backing file 000004 error: injected error +a: (a, .) +b: (b, .) +c: (c, .) +d: (d, .) +. diff --git a/vfs/atomicfs/marker_test.go b/vfs/atomicfs/marker_test.go index 45cd16cff6..f43c90dd6a 100644 --- a/vfs/atomicfs/marker_test.go +++ b/vfs/atomicfs/marker_test.go @@ -210,9 +210,9 @@ func TestMarker_FaultTolerance(t *testing.T) { t.Run(strconv.Itoa(i), func(t *testing.T) { var count atomic.Int32 count.Store(int32(i)) - inj := errorfs.InjectorFunc(func(op errorfs.Op, path string) error { + inj := errorfs.InjectorFunc(func(op errorfs.Op) error { // Don't inject on Sync errors. They're fatal. - if op == errorfs.OpFileSync { + if op.Kind == errorfs.OpFileSync { return nil } if v := count.Add(-1); v == 0 { diff --git a/vfs/errorfs/dsl.go b/vfs/errorfs/dsl.go new file mode 100644 index 0000000000..fbe8dc6ae9 --- /dev/null +++ b/vfs/errorfs/dsl.go @@ -0,0 +1,339 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package errorfs + +import ( + "fmt" + "go/scanner" + "go/token" + "path/filepath" + "strconv" + "strings" + + "github.com/cockroachdb/errors" +) + +// Predicate encodes conditional logic that determines whether to inject an +// error. +type Predicate interface { + evaluate(Op) bool + String() string +} + +// PathMatch returns a predicate that returns true if an operation's file path +// matches the provided pattern according to filepath.Match. +func PathMatch(pattern string) Predicate { + return &pathMatch{pattern: pattern} +} + +type pathMatch struct { + pattern string +} + +func (pm *pathMatch) String() string { + return fmt.Sprintf("(PathMatch %q)", pm.pattern) +} + +func (pm *pathMatch) evaluate(op Op) bool { + matched, err := filepath.Match(pm.pattern, op.Path) + if err != nil { + // Only possible error is ErrBadPattern, indicating an issue with + // the test itself. + panic(err) + } + return matched +} + +var ( + // Reads is a predicate that returns true iff an operation is a read + // operation. + Reads Predicate = opKindPred{kind: OpIsRead} + // Writes is a predicate that returns true iff an operation is a write + // operation. + Writes Predicate = opKindPred{kind: OpIsWrite} +) + +type opKindPred struct { + kind OpReadWrite +} + +func (p opKindPred) String() string { return p.kind.String() } +func (p opKindPred) evaluate(op Op) bool { return p.kind == op.Kind.ReadOrWrite() } + +// And returns a predicate that returns true if all its operands return true. +func And(preds ...Predicate) Predicate { return and(preds) } + +type and []Predicate + +func (a and) String() string { + var sb strings.Builder + sb.WriteString("(And") + for i := 0; i < len(a); i++ { + sb.WriteRune(' ') + sb.WriteString(a[i].String()) + } + sb.WriteRune(')') + return sb.String() +} + +func (a and) evaluate(o Op) bool { + ok := true + for _, p := range a { + ok = ok && p.evaluate(o) + } + return ok +} + +// Or returns a predicate that returns true if any of its operands return true. +func Or(preds ...Predicate) Predicate { return or(preds) } + +type or []Predicate + +func (e or) String() string { + var sb strings.Builder + sb.WriteString("(Or") + for i := 0; i < len(e); i++ { + sb.WriteRune(' ') + sb.WriteString(e[i].String()) + } + sb.WriteRune(')') + return sb.String() +} + +func (e or) evaluate(o Op) bool { + ok := false + for _, p := range e { + ok = ok || p.evaluate(o) + } + return ok +} + +// OnIndex returns a predicate that returns true on its (n+1)-th invocation. +func OnIndex(index int32) *InjectIndex { + ii := &InjectIndex{} + ii.index.Store(index) + return ii +} + +// ParseInjectorFromDSL parses a string encoding a lisp-like DSL describing when +// errors should be injected. +// +// Errors: +// - ErrInjected is the only error currently supported by the DSL. +// +// Injectors: +// - : An error by itself is an injector that injects an error every +// time. +// - ( ) is an injector that injects an error only when +// the operation satisfies the predicate. +// +// Predicates: +// - Reads is a constant predicate that evalutes to true iff the operation is a +// read operation (eg, Open, Read, ReadAt, Stat) +// - Writes is a constant predicate that evaluates to true iff the operation is +// a write operation (eg, Create, Rename, Write, WriteAt, etc). +// - (PathMatch ) is a predicate that evalutes to true iff the +// operation's file path matches the provided shell pattern. +// - (OnIndex ) is a predicate that evaluates to true only on the n-th +// invocation. +// - (And [PREDICATE]...) is a predicate that evaluates to true +// iff all the provided predicates evaluate to true. And short circuits on +// the first predicate to evaluate to false. +// - (Or [PREDICATE]...) is a predicate that evaluates to true iff +// at least one of the provided predicates evaluates to true. Or short +// circuits on the first predicate to evaluate to true. +// +// Example: (ErrInjected (And (PathMatch "*.sst") (OnIndex 5))) is a rule set +// that will inject an error on the 5-th I/O operation involving an sstable. +func ParseInjectorFromDSL(d string) (inj Injector, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + panic(r) + } + } + }() + + fset := token.NewFileSet() + file := fset.AddFile("", -1, len(d)) + var s scanner.Scanner + s.Init(file, []byte(strings.TrimSpace(d)), nil /* no error handler */, 0) + pos, tok, lit := s.Scan() + inj, err = parseDSLInjectorFromPos(&s, pos, tok, lit) + if err != nil { + return nil, err + } + pos, tok, lit = s.Scan() + if tok == token.SEMICOLON { + pos, tok, lit = s.Scan() + } + if tok != token.EOF { + return nil, errors.Errorf("errorfs: unexpected token %s (%q) at char %v; expected EOF", tok, lit, pos) + } + return inj, err +} + +// LabelledError is an error that also implements Injector, unconditionally +// injecting itself. It implements String() by returning its label. It +// implements Error() by returning its underlying error. +type LabelledError struct { + error + label string + predicate Predicate +} + +// String implements fmt.Stringer. +func (le LabelledError) String() string { + if le.predicate == nil { + return le.label + } + return fmt.Sprintf("(%s %s)", le.label, le.predicate.String()) +} + +// MaybeError implements Injector. +func (le LabelledError) MaybeError(op Op) error { + if le.predicate == nil || le.predicate.evaluate(op) { + return le + } + return nil +} + +// If returns an Injector that returns the receiver error if the provided +// predicate evalutes to true. +func (le LabelledError) If(p Predicate) Injector { + le.predicate = p + return le +} + +// AddError defines a new error that may be used within the DSL parsed by +// ParseInjectorFromDSL and will inject the provided error. +func AddError(le LabelledError) { + dslKnownErrors[le.label] = le +} + +var ( + dslPredicateExprs map[string]func(*scanner.Scanner) Predicate + dslPredicateConstants map[string]func(*scanner.Scanner) Predicate + dslKnownErrors map[string]LabelledError +) + +func init() { + dslKnownErrors = map[string]LabelledError{} + dslPredicateConstants = map[string]func(*scanner.Scanner) Predicate{ + "Reads": func(s *scanner.Scanner) Predicate { return Reads }, + "Writes": func(s *scanner.Scanner) Predicate { return Writes }, + } + // Parsers for predicate exprs of the form `(ident ...)`. + dslPredicateExprs = map[string]func(*scanner.Scanner) Predicate{ + "PathMatch": func(s *scanner.Scanner) Predicate { + pattern := mustUnquote(consumeTok(s, token.STRING)) + consumeTok(s, token.RPAREN) + return PathMatch(pattern) + }, + "OnIndex": func(s *scanner.Scanner) Predicate { + i, err := strconv.ParseInt(consumeTok(s, token.INT), 10, 32) + if err != nil { + panic(err) + } + consumeTok(s, token.RPAREN) + return OnIndex(int32(i)) + }, + "And": func(s *scanner.Scanner) Predicate { + return And(parseVariadicPredicate(s)...) + }, + "Or": func(s *scanner.Scanner) Predicate { + return Or(parseVariadicPredicate(s)...) + }, + } + AddError(ErrInjected) +} + +func parseVariadicPredicate(s *scanner.Scanner) (ret []Predicate) { + pos, tok, lit := s.Scan() + for tok == token.LPAREN || tok == token.IDENT { + pred, err := parseDSLPredicateFromPos(s, pos, tok, lit) + if err != nil { + panic(err) + } + ret = append(ret, pred) + pos, tok, lit = s.Scan() + } + if tok != token.RPAREN { + panic(errors.Errorf("errorfs: unexpected token %s (%q) at char %v; expected RPAREN", tok, lit, pos)) + } + return ret +} + +func parseDSLInjectorFromPos( + s *scanner.Scanner, pos token.Pos, tok token.Token, lit string, +) (Injector, error) { + switch tok { + case token.IDENT: + // It's an injector of the form `ErrInjected`. + le, ok := dslKnownErrors[lit] + if !ok { + return nil, errors.Errorf("errorfs: unknown error %q", lit) + } + return le, nil + case token.LPAREN: + // Otherwise it's an expression, eg: (ErrInjected (And ...)) + lit = consumeTok(s, token.IDENT) + le, ok := dslKnownErrors[lit] + if !ok { + return nil, errors.Errorf("errorfs: unknown error %q", lit) + } + pos, tok, lit := s.Scan() + pred, err := parseDSLPredicateFromPos(s, pos, tok, lit) + if err != nil { + panic(err) + } + consumeTok(s, token.RPAREN) + return le.If(pred), nil + default: + return nil, errors.Errorf("errorfs: unexpected token %s (%q) at char %v; expected IDENT or LPAREN", tok, lit, pos) + } +} + +func parseDSLPredicateFromPos( + s *scanner.Scanner, pos token.Pos, tok token.Token, lit string, +) (Predicate, error) { + switch tok { + case token.IDENT: + // It's a predicate of the form `Reads`. + p, ok := dslPredicateConstants[lit] + if !ok { + return nil, errors.Errorf("errorfs: unknown predicate constant %q", lit) + } + return p(s), nil + case token.LPAREN: + // Otherwise it's an expression, eg: (OnIndex 1) + lit = consumeTok(s, token.IDENT) + p, ok := dslPredicateExprs[lit] + if !ok { + return nil, errors.Errorf("errorfs: unknown predicate func %q", lit) + } + return p(s), nil + default: + return nil, errors.Errorf("errorfs: unexpected token %s (%q) at char %v; expected IDENT or LPAREN", tok, lit, pos) + } +} + +func consumeTok(s *scanner.Scanner, expected token.Token) (lit string) { + pos, tok, lit := s.Scan() + if tok != expected { + panic(errors.Errorf("errorfs: unexpected token %s (%q) at char %v; expected %s", tok, lit, pos, expected)) + } + return lit +} + +func mustUnquote(lit string) string { + s, err := strconv.Unquote(lit) + if err != nil { + panic(errors.Newf("errorfs: unquoting %q: %v", lit, err)) + } + return s +} diff --git a/vfs/errorfs/errorfs.go b/vfs/errorfs/errorfs.go index 9ebc45a913..327052e016 100644 --- a/vfs/errorfs/errorfs.go +++ b/vfs/errorfs/errorfs.go @@ -9,6 +9,7 @@ import ( "io" "math/rand" "os" + "strings" "sync" "sync/atomic" "time" @@ -19,14 +20,25 @@ import ( ) // ErrInjected is an error artificially injected for testing fs error paths. -var ErrInjected = errors.New("injected error") +var ErrInjected = LabelledError{ + error: errors.New("injected error"), + label: "ErrInjected", +} + +// Op describes a filesystem operation. +type Op struct { + // Kind describes the particular kind of operation being performed. + Kind OpKind + // Path is the path of the file of the file being operated on. + Path string +} -// Op is an enum describing the type of operation. -type Op int +// OpKind is an enum describing the type of operation. +type OpKind int const ( // OpCreate describes a create file operation. - OpCreate Op = iota + OpCreate OpKind = iota // OpLink describes a hardlink operation. OpLink // OpOpen describes a file open operation. @@ -71,36 +83,39 @@ const ( OpFileFlush ) -// OpKind returns the operation's kind. -func (o Op) OpKind() OpKind { +// ReadOrWrite returns the operation's kind. +func (o OpKind) ReadOrWrite() OpReadWrite { switch o { case OpOpen, OpOpenDir, OpList, OpStat, OpGetDiskUsage, OpFileRead, OpFileReadAt, OpFileStat: - return OpKindRead + return OpIsRead case OpCreate, OpLink, OpRemove, OpRemoveAll, OpRename, OpReuseForRewrite, OpMkdirAll, OpLock, OpFileClose, OpFileWrite, OpFileWriteAt, OpFileSync, OpFileFlush, OpFilePreallocate: - return OpKindWrite + return OpIsWrite default: panic(fmt.Sprintf("unrecognized op %v\n", o)) } } -// OpKind is an enum describing whether an operation is a read or write +// OpReadWrite is an enum describing whether an operation is a read or write // operation. -type OpKind int +type OpReadWrite int const ( - // OpKindRead describes read operations. - OpKindRead OpKind = iota - // OpKindWrite describes write operations. - OpKindWrite + // OpIsRead describes read operations. + OpIsRead OpReadWrite = iota + // OpIsWrite describes write operations. + OpIsWrite ) -// OnIndex constructs an injector that returns an error on -// the (n+1)-th invocation of its MaybeError function. It -// may be passed to Wrap to inject an error into an FS. -func OnIndex(index int32) *InjectIndex { - ii := &InjectIndex{} - ii.index.Store(index) - return ii +// String implements fmt.Stringer. +func (kind OpReadWrite) String() string { + switch kind { + case OpIsRead: + return "Reads" + case OpIsWrite: + return "Writes" + default: + panic(fmt.Sprintf("unrecognized OpKind %d", kind)) + } } // InjectIndex implements Injector, injecting an error at a specific index. @@ -108,31 +123,41 @@ type InjectIndex struct { index atomic.Int32 } +// String implements fmt.Stringer. +func (ii *InjectIndex) String() string { + return fmt.Sprintf("(OnIndex %d)", ii.index.Load()) +} + // Index returns the index at which the error will be injected. func (ii *InjectIndex) Index() int32 { return ii.index.Load() } // SetIndex sets the index at which the error will be injected. func (ii *InjectIndex) SetIndex(v int32) { ii.index.Store(v) } +func (ii *InjectIndex) evaluate(op Op) bool { return ii.index.Add(-1) == -1 } + // MaybeError implements the Injector interface. -func (ii *InjectIndex) MaybeError(_ Op, _ string) error { - if ii.index.Add(-1) == -1 { - return errors.WithStack(ErrInjected) +// +// TODO(jackson): Remove this implementation and update callers to compose it +// with other injectors. +func (ii *InjectIndex) MaybeError(op Op) error { + if !ii.evaluate(op) { + return nil } - return nil + return ErrInjected } // WithProbability returns a function that returns an error with the provided // probability when passed op. It may be passed to Wrap to inject an error // into an ErrFS with the provided probability. p should be within the range // [0.0,1.0]. -func WithProbability(op OpKind, p float64) Injector { +func WithProbability(op OpReadWrite, p float64) Injector { mu := new(sync.Mutex) rnd := rand.New(rand.NewSource(time.Now().UnixNano())) - return InjectorFunc(func(currOp Op, _ string) error { + return InjectorFunc(func(currOp Op) error { mu.Lock() defer mu.Unlock() - if currOp.OpKind() == op && rnd.Float64() < p { + if currOp.Kind.ReadOrWrite() == op && rnd.Float64() < p { return errors.WithStack(ErrInjected) } return nil @@ -141,18 +166,51 @@ func WithProbability(op OpKind, p float64) Injector { // InjectorFunc implements the Injector interface for a function with // MaybeError's signature. -type InjectorFunc func(Op, string) error +type InjectorFunc func(Op) error + +// String implements fmt.Stringer. +func (f InjectorFunc) String() string { return "" } // MaybeError implements the Injector interface. -func (f InjectorFunc) MaybeError(op Op, path string) error { return f(op, path) } +func (f InjectorFunc) MaybeError(op Op) error { return f(op) } // Injector injects errors into FS operations. type Injector interface { + fmt.Stringer // MaybeError is invoked by an errorfs before an operation is executed. It // is passed an enum indicating the type of operation and a path of the // subject file or directory. If the operation takes two paths (eg, // Rename, Link), the original source path is provided. - MaybeError(op Op, path string) error + MaybeError(op Op) error +} + +// Any returns an injector that injects an error if any of the provided +// injectors inject an error. The error returned by the first injector to return +// an error is used. +func Any(injectors ...Injector) Injector { + return anyInjector(injectors) +} + +type anyInjector []Injector + +func (a anyInjector) String() string { + var sb strings.Builder + sb.WriteString("(Any") + for _, inj := range a { + sb.WriteString(" ") + sb.WriteString(inj.String()) + } + sb.WriteString(")") + return sb.String() +} + +func (a anyInjector) MaybeError(op Op) error { + for _, inj := range a { + if err := inj.MaybeError(op); err != nil { + return err + } + } + return nil } // FS implements vfs.FS, injecting errors into @@ -190,7 +248,7 @@ func (fs *FS) Unwrap() vfs.FS { // Create implements FS.Create. func (fs *FS) Create(name string) (vfs.File, error) { - if err := fs.inj.MaybeError(OpCreate, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpCreate, Path: name}); err != nil { return nil, err } f, err := fs.fs.Create(name) @@ -202,7 +260,7 @@ func (fs *FS) Create(name string) (vfs.File, error) { // Link implements FS.Link. func (fs *FS) Link(oldname, newname string) error { - if err := fs.inj.MaybeError(OpLink, oldname); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpLink, Path: oldname}); err != nil { return err } return fs.fs.Link(oldname, newname) @@ -210,7 +268,7 @@ func (fs *FS) Link(oldname, newname string) error { // Open implements FS.Open. func (fs *FS) Open(name string, opts ...vfs.OpenOption) (vfs.File, error) { - if err := fs.inj.MaybeError(OpOpen, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpOpen, Path: name}); err != nil { return nil, err } f, err := fs.fs.Open(name) @@ -226,7 +284,7 @@ func (fs *FS) Open(name string, opts ...vfs.OpenOption) (vfs.File, error) { // OpenReadWrite implements FS.OpenReadWrite. func (fs *FS) OpenReadWrite(name string, opts ...vfs.OpenOption) (vfs.File, error) { - if err := fs.inj.MaybeError(OpOpen, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpOpen, Path: name}); err != nil { return nil, err } f, err := fs.fs.OpenReadWrite(name) @@ -242,7 +300,7 @@ func (fs *FS) OpenReadWrite(name string, opts ...vfs.OpenOption) (vfs.File, erro // OpenDir implements FS.OpenDir. func (fs *FS) OpenDir(name string) (vfs.File, error) { - if err := fs.inj.MaybeError(OpOpenDir, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpOpenDir, Path: name}); err != nil { return nil, err } f, err := fs.fs.OpenDir(name) @@ -254,7 +312,7 @@ func (fs *FS) OpenDir(name string) (vfs.File, error) { // GetDiskUsage implements FS.GetDiskUsage. func (fs *FS) GetDiskUsage(path string) (vfs.DiskUsage, error) { - if err := fs.inj.MaybeError(OpGetDiskUsage, path); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpGetDiskUsage, Path: path}); err != nil { return vfs.DiskUsage{}, err } return fs.fs.GetDiskUsage(path) @@ -281,7 +339,7 @@ func (fs *FS) Remove(name string) error { return nil } - if err := fs.inj.MaybeError(OpRemove, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpRemove, Path: name}); err != nil { return err } return fs.fs.Remove(name) @@ -289,7 +347,7 @@ func (fs *FS) Remove(name string) error { // RemoveAll implements FS.RemoveAll. func (fs *FS) RemoveAll(fullname string) error { - if err := fs.inj.MaybeError(OpRemoveAll, fullname); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpRemoveAll, Path: fullname}); err != nil { return err } return fs.fs.RemoveAll(fullname) @@ -297,7 +355,7 @@ func (fs *FS) RemoveAll(fullname string) error { // Rename implements FS.Rename. func (fs *FS) Rename(oldname, newname string) error { - if err := fs.inj.MaybeError(OpRename, oldname); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpRename, Path: oldname}); err != nil { return err } return fs.fs.Rename(oldname, newname) @@ -305,7 +363,7 @@ func (fs *FS) Rename(oldname, newname string) error { // ReuseForWrite implements FS.ReuseForWrite. func (fs *FS) ReuseForWrite(oldname, newname string) (vfs.File, error) { - if err := fs.inj.MaybeError(OpReuseForRewrite, oldname); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpReuseForRewrite, Path: oldname}); err != nil { return nil, err } return fs.fs.ReuseForWrite(oldname, newname) @@ -313,7 +371,7 @@ func (fs *FS) ReuseForWrite(oldname, newname string) (vfs.File, error) { // MkdirAll implements FS.MkdirAll. func (fs *FS) MkdirAll(dir string, perm os.FileMode) error { - if err := fs.inj.MaybeError(OpMkdirAll, dir); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpMkdirAll, Path: dir}); err != nil { return err } return fs.fs.MkdirAll(dir, perm) @@ -321,7 +379,7 @@ func (fs *FS) MkdirAll(dir string, perm os.FileMode) error { // Lock implements FS.Lock. func (fs *FS) Lock(name string) (io.Closer, error) { - if err := fs.inj.MaybeError(OpLock, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpLock, Path: name}); err != nil { return nil, err } return fs.fs.Lock(name) @@ -329,7 +387,7 @@ func (fs *FS) Lock(name string) (io.Closer, error) { // List implements FS.List. func (fs *FS) List(dir string) ([]string, error) { - if err := fs.inj.MaybeError(OpList, dir); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpList, Path: dir}); err != nil { return nil, err } return fs.fs.List(dir) @@ -337,7 +395,7 @@ func (fs *FS) List(dir string) ([]string, error) { // Stat implements FS.Stat. func (fs *FS) Stat(name string) (os.FileInfo, error) { - if err := fs.inj.MaybeError(OpStat, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpStat, Path: name}); err != nil { return nil, err } return fs.fs.Stat(name) @@ -358,35 +416,35 @@ func (f *errorFile) Close() error { } func (f *errorFile) Read(p []byte) (int, error) { - if err := f.inj.MaybeError(OpFileRead, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileRead, Path: f.path}); err != nil { return 0, err } return f.file.Read(p) } func (f *errorFile) ReadAt(p []byte, off int64) (int, error) { - if err := f.inj.MaybeError(OpFileReadAt, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileReadAt, Path: f.path}); err != nil { return 0, err } return f.file.ReadAt(p, off) } func (f *errorFile) Write(p []byte) (int, error) { - if err := f.inj.MaybeError(OpFileWrite, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileWrite, Path: f.path}); err != nil { return 0, err } return f.file.Write(p) } func (f *errorFile) WriteAt(p []byte, ofs int64) (int, error) { - if err := f.inj.MaybeError(OpFileWriteAt, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileWriteAt, Path: f.path}); err != nil { return 0, err } return f.file.WriteAt(p, ofs) } func (f *errorFile) Stat() (os.FileInfo, error) { - if err := f.inj.MaybeError(OpFileStat, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileStat, Path: f.path}); err != nil { return nil, err } return f.file.Stat() @@ -398,14 +456,14 @@ func (f *errorFile) Prefetch(offset, length int64) error { } func (f *errorFile) Preallocate(offset, length int64) error { - if err := f.inj.MaybeError(OpFilePreallocate, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFilePreallocate, Path: f.path}); err != nil { return err } return f.file.Preallocate(offset, length) } func (f *errorFile) Sync() error { - if err := f.inj.MaybeError(OpFileSync, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileSync, Path: f.path}); err != nil { return err } return f.file.Sync() diff --git a/vfs/errorfs/errorfs_test.go b/vfs/errorfs/errorfs_test.go new file mode 100644 index 0000000000..644c4ceefd --- /dev/null +++ b/vfs/errorfs/errorfs_test.go @@ -0,0 +1,34 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package errorfs + +import ( + "fmt" + "strings" + "testing" + + "github.com/cockroachdb/datadriven" +) + +func TestErrorFS(t *testing.T) { + var sb strings.Builder + datadriven.RunTest(t, "testdata/errorfs", func(t *testing.T, td *datadriven.TestData) string { + sb.Reset() + switch td.Cmd { + case "parse-dsl": + for _, l := range strings.Split(strings.TrimSpace(td.Input), "\n") { + inj, err := ParseInjectorFromDSL(l) + if err != nil { + fmt.Fprintf(&sb, "parsing err: %s\n", err) + } else { + fmt.Fprintf(&sb, "%s\n", inj.String()) + } + } + return sb.String() + default: + return fmt.Sprintf("unrecognized command %q", td.Cmd) + } + }) +} diff --git a/vfs/errorfs/testdata/errorfs b/vfs/errorfs/testdata/errorfs new file mode 100644 index 0000000000..fa3cdea6fa --- /dev/null +++ b/vfs/errorfs/testdata/errorfs @@ -0,0 +1,47 @@ +parse-dsl +ErrInjected +(ErrInjected Reads) +(ErrInjected (PathMatch "foo/*.sst")) +(ErrInjected (OnIndex 1)) +(ErrInjected (Or Reads Writes)) +(ErrInjected (And (PathMatch "foo/bar/*.sst") (OnIndex 1))) +(ErrInjected (Or (OnIndex 2) (PathMatch "*.sst"))) +(ErrInjected (And Reads (PathMatch "*.sst"))) +(ErrInjected (Or Writes (PathMatch "*.sst"))) +---- +ErrInjected +(ErrInjected Reads) +(ErrInjected (PathMatch "foo/*.sst")) +(ErrInjected (OnIndex 1)) +(ErrInjected (Or Reads Writes)) +(ErrInjected (And (PathMatch "foo/bar/*.sst") (OnIndex 1))) +(ErrInjected (Or (OnIndex 2) (PathMatch "*.sst"))) +(ErrInjected (And Reads (PathMatch "*.sst"))) +(ErrInjected (Or Writes (PathMatch "*.sst"))) + +parse-dsl +errInjected +ErrInjected() +(ErrInjected (PathMatch foo/*.sst)) +(alwoes (PathMatch "foo/*.sst")) +(ErrInjected (PathMatch "foo/*.sst" "")) +(ErrInjected PathMatch "foo/*.sst") +(ErrInjected (OnIndex ErrInjected)) +(Or ErrInjected ErrInjected ErrInjected +(And ErrInjected ErrInjected ErrInjected) +(Or 1 4 5) +(ErrInjected (OnIndex foo)) +(ErrInjected (OnIndex 9223372036854775807)) +---- +parsing err: errorfs: unknown error "errInjected" +parsing err: errorfs: unexpected token ( ("") at char 12; expected EOF +parsing err: errorfs: unexpected token IDENT ("foo") at char 25; expected STRING +parsing err: errorfs: unknown error "alwoes" +parsing err: errorfs: unexpected token STRING ("\"\"") at char 37; expected ) +parsing err: errorfs: unknown predicate constant "PathMatch" +parsing err: errorfs: unexpected token IDENT ("ErrInjected") at char 23; expected INT +parsing err: errorfs: unknown error "Or" +parsing err: errorfs: unknown error "And" +parsing err: errorfs: unknown error "Or" +parsing err: errorfs: unexpected token IDENT ("foo") at char 23; expected INT +parsing err: strconv.ParseInt: parsing "9223372036854775807": value out of range