diff --git a/compaction_test.go b/compaction_test.go index 4ea85950379..bf36ce0108a 100644 --- a/compaction_test.go +++ b/compaction_test.go @@ -2898,7 +2898,7 @@ func TestCompactionErrorCleanup(t *testing.T) { ) mem := vfs.NewMem() - ii := errorfs.OnIndex(math.MaxInt32) // start disabled + ii := errorfs.OnIndex(math.MaxInt32, errorfs.ErrInjected) // start disabled opts := (&Options{ FS: errorfs.Wrap(mem, ii), Levels: make([]LevelOptions, numLevels), @@ -3286,8 +3286,8 @@ func TestFlushError(t *testing.T) { // Error the first five times we try to write a sstable. var errorOps atomic.Int32 errorOps.Store(3) - fs := errorfs.Wrap(vfs.NewMem(), errorfs.InjectorFunc(func(op errorfs.Op, path string) error { - if op == errorfs.OpCreate && filepath.Ext(path) == ".sst" && errorOps.Add(-1) >= 0 { + fs := errorfs.Wrap(vfs.NewMem(), errorfs.InjectorFunc(func(op errorfs.Op) error { + if op.Kind == errorfs.OpCreate && filepath.Ext(op.Path) == ".sst" && errorOps.Add(-1) >= 0 { return errorfs.ErrInjected } return nil @@ -3704,19 +3704,24 @@ type createManifestErrorInjector struct { enabled atomic.Bool } +// TODO(jackson): Replace the createManifestErrorInjector with the composition +// of primitives defined in errorfs. This may require additional primitives. + +func (i *createManifestErrorInjector) String() string { return "MANIFEST-Creates" } + // enable enables error injection for the vfs.FS. func (i *createManifestErrorInjector) enable() { i.enabled.Store(true) } // MaybeError implements errorfs.Injector. -func (i *createManifestErrorInjector) MaybeError(op errorfs.Op, path string) error { +func (i *createManifestErrorInjector) MaybeError(op errorfs.Op) error { if !i.enabled.Load() { return nil } // This necessitates having a MaxManifestSize of 1, to reliably induce // logAndApply errors. - if strings.Contains(path, "MANIFEST") && op == errorfs.OpCreate { + if strings.Contains(op.Path, "MANIFEST") && op.Kind == errorfs.OpCreate { return errorfs.ErrInjected } return nil @@ -3884,6 +3889,11 @@ type WriteErrorInjector struct { enabled atomic.Bool } +// TODO(jackson): Replace WriteErrorInjector with use of primitives in errorfs, +// adding new primitives as necessary. + +func (i *WriteErrorInjector) String() string { return "FileWrites(ErrInjected)" } + // enable enables error injection for the vfs.FS. func (i *WriteErrorInjector) enable() { i.enabled.Store(true) @@ -3895,12 +3905,12 @@ func (i *WriteErrorInjector) disable() { } // MaybeError implements errorfs.Injector. -func (i *WriteErrorInjector) MaybeError(op errorfs.Op, path string) error { +func (i *WriteErrorInjector) MaybeError(op errorfs.Op) error { if !i.enabled.Load() { return nil } // Fail any future write. - if op == errorfs.OpFileWrite { + if op.Kind == errorfs.OpFileWrite { return errorfs.ErrInjected } return nil diff --git a/data_test.go b/data_test.go index fa1cc1ae261..2d7fe87a2b3 100644 --- a/data_test.go +++ b/data_test.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/datadriven" "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/bloom" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/humanize" "github.com/cockroachdb/pebble/internal/keyspan" @@ -29,6 +30,7 @@ import ( "github.com/cockroachdb/pebble/objstorage/remote" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" + "github.com/cockroachdb/pebble/vfs/errorfs" "github.com/stretchr/testify/require" ) @@ -1447,3 +1449,75 @@ func runLSMCmd(td *datadriven.TestData, d *DB) string { } return d.mu.versions.currentVersion().String() } + +func parseDBOptionsArgs(opts *Options, args []datadriven.CmdArg) error { + for _, cmdArg := range args { + switch cmdArg.Key { + case "inject-errors": + injs := make([]errorfs.Injector, len(cmdArg.Vals)) + for i := 0; i < len(cmdArg.Vals); i++ { + inj, err := errorfs.ParseInjectorFromDSL(cmdArg.Vals[i]) + if err != nil { + return err + } + injs[i] = inj + } + opts.FS = errorfs.Wrap(opts.FS, errorfs.Any(injs...)) + case "enable-table-stats": + enable, err := strconv.ParseBool(cmdArg.Vals[0]) + if err != nil { + return errors.Errorf("%s: could not parse %q as bool: %s", cmdArg.Key, cmdArg.Vals[0], err) + } + opts.private.disableTableStats = !enable + case "format-major-version": + v, err := strconv.Atoi(cmdArg.Vals[0]) + if err != nil { + return err + } + // Override the DB version. + opts.FormatMajorVersion = FormatMajorVersion(v) + case "block-size": + v, err := strconv.Atoi(cmdArg.Vals[0]) + if err != nil { + return err + } + for i := range opts.Levels { + opts.Levels[i].BlockSize = v + } + case "index-block-size": + v, err := strconv.Atoi(cmdArg.Vals[0]) + if err != nil { + return err + } + for i := range opts.Levels { + opts.Levels[i].IndexBlockSize = v + } + case "target-file-size": + v, err := strconv.Atoi(cmdArg.Vals[0]) + if err != nil { + return err + } + for i := range opts.Levels { + opts.Levels[i].TargetFileSize = int64(v) + } + case "bloom-bits-per-key": + v, err := strconv.Atoi(cmdArg.Vals[0]) + if err != nil { + return err + } + fp := bloom.FilterPolicy(v) + opts.Filters = map[string]FilterPolicy{fp.Name(): fp} + for i := range opts.Levels { + opts.Levels[i].FilterPolicy = fp + } + case "merger": + switch cmdArg.Vals[0] { + case "appender": + opts.Merger = base.DefaultMerger + default: + return errors.Newf("unrecognized Merger %q\n", cmdArg.Vals[0]) + } + } + } + return nil +} diff --git a/error_test.go b/error_test.go index 68f546f5feb..ecc23e74fa2 100644 --- a/error_test.go +++ b/error_test.go @@ -136,7 +136,7 @@ func TestErrors(t *testing.T) { errorCounts := make(map[string]int) for i := int32(0); ; i++ { - fs := errorfs.Wrap(vfs.NewMem(), errorfs.OnIndex(i)) + fs := errorfs.Wrap(vfs.NewMem(), errorfs.OnIndex(i, errorfs.ErrInjected)) err := run(fs) if err == nil { t.Logf("success %d\n", i) @@ -166,7 +166,7 @@ func TestErrors(t *testing.T) { func TestRequireReadError(t *testing.T) { run := func(formatVersion FormatMajorVersion, index int32) (err error) { // Perform setup with error injection disabled as it involves writes/background ops. - inj := errorfs.OnIndex(-1) + inj := errorfs.OnIndex(-1, errorfs.ErrInjected) fs := errorfs.Wrap(vfs.NewMem(), inj) opts := &Options{ FS: fs, @@ -367,8 +367,8 @@ func TestDBWALRotationCrash(t *testing.T) { memfs := vfs.NewStrictMem() var index atomic.Int32 - inj := errorfs.InjectorFunc(func(op errorfs.Op, _ string) error { - if op.OpKind() == errorfs.OpKindWrite && index.Add(-1) == -1 { + inj := errorfs.InjectorFunc(func(op errorfs.Op) error { + if op.Kind.ReadOrWrite() == errorfs.OpIsWrite && index.Add(-1) == -1 { memfs.SetIgnoreSyncs(true) } return nil diff --git a/ingest_test.go b/ingest_test.go index 6cfc9659b14..712de0a8216 100644 --- a/ingest_test.go +++ b/ingest_test.go @@ -386,7 +386,7 @@ func TestIngestLinkFallback(t *testing.T) { src, err := mem.Create("source") require.NoError(t, err) - opts := &Options{FS: errorfs.Wrap(mem, errorfs.OnIndex(1))} + opts := &Options{FS: errorfs.Wrap(mem, errorfs.OnIndex(1, errorfs.ErrInjected))} opts.EnsureDefaults().WithFSDefaults() objSettings := objstorageprovider.DefaultSettings(opts.FS, "") // Prevent the provider from listing the dir (where we may get an injected error). @@ -2182,7 +2182,7 @@ func TestIngestError(t *testing.T) { require.NoError(t, w.Set([]byte("d"), nil)) require.NoError(t, w.Close()) - inj := errorfs.OnIndex(-1) + inj := errorfs.OnIndex(-1, errorfs.ErrInjected) d, err := Open("", &Options{ FS: errorfs.Wrap(mem, inj), Logger: panicLogger{}, @@ -3225,11 +3225,11 @@ func TestIngestValidation(t *testing.T) { cLoc: corruptionLocationNone, wantErr: errorfs.ErrInjected, wantErrType: errReportLocationBackgroundError, - errorfsInjector: errorfs.InjectorFunc(func(op errorfs.Op, path string) error { + errorfsInjector: errorfs.InjectorFunc(func(op errorfs.Op) error { // Inject an error on the first read-at operation on an sstable // (excluding the read on the sstable before ingestion has // linked it in). - if path != "ext" && op != errorfs.OpFileReadAt || filepath.Ext(path) != ".sst" { + if op.Path != "ext" && op.Kind != errorfs.OpFileReadAt || filepath.Ext(op.Path) != ".sst" { return nil } if errfsCounter.Add(1) == 1 { diff --git a/iterator_histories_test.go b/iterator_histories_test.go index bd178b2def9..5f958ec2c9b 100644 --- a/iterator_histories_test.go +++ b/iterator_histories_test.go @@ -13,8 +13,6 @@ import ( "github.com/cockroachdb/datadriven" "github.com/cockroachdb/errors" - "github.com/cockroachdb/pebble/bloom" - "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/testkeys" "github.com/cockroachdb/pebble/sstable" @@ -41,8 +39,9 @@ func TestIterHistories(t *testing.T) { iters[name] = it return it } + var opts *Options parseOpts := func(td *datadriven.TestData) (*Options, error) { - opts := &Options{ + opts = &Options{ FS: vfs.NewMem(), Comparer: testkeys.Comparer, FormatMajorVersion: FormatRangeKeys, @@ -50,61 +49,12 @@ func TestIterHistories(t *testing.T) { sstable.NewTestKeysBlockPropertyCollector, }, } + opts.DisableAutomaticCompactions = true opts.EnsureDefaults() opts.WithFSDefaults() - - for _, cmdArg := range td.CmdArgs { - switch cmdArg.Key { - case "format-major-version": - v, err := strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return nil, err - } - // Override the DB version. - opts.FormatMajorVersion = FormatMajorVersion(v) - case "block-size": - v, err := strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return nil, err - } - for i := range opts.Levels { - opts.Levels[i].BlockSize = v - } - case "index-block-size": - v, err := strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return nil, err - } - for i := range opts.Levels { - opts.Levels[i].IndexBlockSize = v - } - case "target-file-size": - v, err := strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return nil, err - } - for i := range opts.Levels { - opts.Levels[i].TargetFileSize = int64(v) - } - case "bloom-bits-per-key": - v, err := strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return nil, err - } - fp := bloom.FilterPolicy(v) - opts.Filters = map[string]FilterPolicy{fp.Name(): fp} - for i := range opts.Levels { - opts.Levels[i].FilterPolicy = fp - } - case "merger": - switch cmdArg.Vals[0] { - case "appender": - opts.Merger = base.DefaultMerger - default: - return nil, errors.Newf("unrecognized Merger %q\n", cmdArg.Vals[0]) - } - } + if err := parseDBOptionsArgs(opts, td.CmdArgs); err != nil { + return nil, err } return opts, nil } @@ -128,10 +78,11 @@ func TestIterHistories(t *testing.T) { datadriven.RunTest(t, path, func(t *testing.T, td *datadriven.TestData) string { switch td.Cmd { case "define": + var err error if err := cleanup(); err != nil { return err.Error() } - opts, err := parseOpts(td) + opts, err = parseOpts(td) if err != nil { return err.Error() } @@ -140,12 +91,23 @@ func TestIterHistories(t *testing.T) { return err.Error() } return runLSMCmd(td, d) - + case "reopen": + var err error + if err := cleanup(); err != nil { + return err.Error() + } + if err := parseDBOptionsArgs(opts, td.CmdArgs); err != nil { + return err.Error() + } + d, err = Open("", opts) + require.NoError(t, err) + return "" case "reset": + var err error if err := cleanup(); err != nil { return err.Error() } - opts, err := parseOpts(td) + opts, err = parseOpts(td) if err != nil { return err.Error() } diff --git a/metamorphic/meta.go b/metamorphic/meta.go index ccd219b129a..419989bf9d5 100644 --- a/metamorphic/meta.go +++ b/metamorphic/meta.go @@ -453,7 +453,7 @@ func RunOnce(t TestingT, runDir string, seed uint64, historyPath string, rOpts . // Wrap the filesystem with one that will inject errors into read // operations with *errorRate probability. - opts.FS = errorfs.Wrap(opts.FS, errorfs.WithProbability(errorfs.OpKindRead, runOpts.errorRate)) + opts.FS = errorfs.Wrap(opts.FS, errorfs.WithProbability(errorfs.OpIsRead, runOpts.errorRate)) if opts.WALDir != "" { opts.WALDir = opts.FS.PathJoin(runDir, opts.WALDir) diff --git a/open_test.go b/open_test.go index 110029906dc..fae3237aaca 100644 --- a/open_test.go +++ b/open_test.go @@ -927,17 +927,17 @@ func TestCrashOpenCrashAfterWALCreation(t *testing.T) { { var walCreated, dirSynced atomic.Bool d, err := Open("", &Options{ - FS: errorfs.Wrap(fs, errorfs.InjectorFunc(func(op errorfs.Op, path string) error { + FS: errorfs.Wrap(fs, errorfs.InjectorFunc(func(op errorfs.Op) error { if dirSynced.Load() { fs.SetIgnoreSyncs(true) } - if op == errorfs.OpCreate && filepath.Ext(path) == ".log" { + if op.Kind == errorfs.OpCreate && filepath.Ext(op.Path) == ".log" { walCreated.Store(true) } // Record when there's a sync of the data directory after the // WAL was created. The data directory will have an empty // path because that's what we passed into Open. - if op == errorfs.OpFileSync && path == "" && walCreated.Load() { + if op.Kind == errorfs.OpFileSync && op.Path == "" && walCreated.Load() { dirSynced.Store(true) } return nil diff --git a/record/log_writer_test.go b/record/log_writer_test.go index 9a70a624b3d..9fce1a71625 100644 --- a/record/log_writer_test.go +++ b/record/log_writer_test.go @@ -515,8 +515,8 @@ func valueAtQuantileWindowed(histogram *prometheusgo.Histogram, q float64) float // blocked. func TestQueueWALBlocks(t *testing.T) { blockWriteCh := make(chan struct{}, 1) - f := errorfs.WrapFile(vfstest.DiscardFile, errorfs.InjectorFunc(func(op errorfs.Op, path string) error { - if op == errorfs.OpFileWrite { + f := errorfs.WrapFile(vfstest.DiscardFile, errorfs.InjectorFunc(func(op errorfs.Op) error { + if op.Kind == errorfs.OpFileWrite { <-blockWriteCh } return nil @@ -556,8 +556,8 @@ func BenchmarkQueueWALBlocks(b *testing.B) { for j := 0; j < b.N; j++ { b.StopTimer() blockWriteCh := make(chan struct{}, 1) - f := errorfs.WrapFile(vfstest.DiscardFile, errorfs.InjectorFunc(func(op errorfs.Op, path string) error { - if op == errorfs.OpFileWrite { + f := errorfs.WrapFile(vfstest.DiscardFile, errorfs.InjectorFunc(func(op errorfs.Op) error { + if op.Kind == errorfs.OpFileWrite { <-blockWriteCh } return nil diff --git a/sstable/reader_test.go b/sstable/reader_test.go index 4549094789e..1396a049888 100644 --- a/sstable/reader_test.go +++ b/sstable/reader_test.go @@ -647,7 +647,7 @@ func TestInjectedErrors(t *testing.T) { f, err := vfs.Default.Open(filepath.FromSlash(prebuiltSST)) require.NoError(t, err) - r, err := newReader(errorfs.WrapFile(f, errorfs.OnIndex(int32(i))), ReaderOptions{}) + r, err := newReader(errorfs.WrapFile(f, errorfs.OnIndex(int32(i), errorfs.ErrInjected)), ReaderOptions{}) if err != nil { return firstError(err, f.Close()) } diff --git a/testdata/iter_histories/errors b/testdata/iter_histories/errors new file mode 100644 index 00000000000..46a91381063 --- /dev/null +++ b/testdata/iter_histories/errors @@ -0,0 +1,59 @@ +reset +---- + +batch commit +set a a +set b b +set c c +set d d +---- +committed 4 keys + +# Scan forward + +combined-iter +seek-ge a +next +next +next +next +---- +a: (a, .) +b: (b, .) +c: (c, .) +d: (d, .) +. + +reopen +---- + +combined-iter +first +next +next +next +next +---- +a: (a, .) +b: (b, .) +c: (c, .) +d: (d, .) +. + +reopen enable-table-stats=false inject-errors=((Reads (PathMatch "*.sst" (OnIndex 4 ErrInjected)))) +---- + +combined-iter +first +first +next +next +next +next +---- +err=pebble: backing file 000004 error: injected error +a: (a, .) +b: (b, .) +c: (c, .) +d: (d, .) +. diff --git a/vfs/atomicfs/marker_test.go b/vfs/atomicfs/marker_test.go index 45cd16cff6b..f43c90dd6a9 100644 --- a/vfs/atomicfs/marker_test.go +++ b/vfs/atomicfs/marker_test.go @@ -210,9 +210,9 @@ func TestMarker_FaultTolerance(t *testing.T) { t.Run(strconv.Itoa(i), func(t *testing.T) { var count atomic.Int32 count.Store(int32(i)) - inj := errorfs.InjectorFunc(func(op errorfs.Op, path string) error { + inj := errorfs.InjectorFunc(func(op errorfs.Op) error { // Don't inject on Sync errors. They're fatal. - if op == errorfs.OpFileSync { + if op.Kind == errorfs.OpFileSync { return nil } if v := count.Add(-1); v == 0 { diff --git a/vfs/errorfs/dsl.go b/vfs/errorfs/dsl.go new file mode 100644 index 00000000000..f54b9dbf703 --- /dev/null +++ b/vfs/errorfs/dsl.go @@ -0,0 +1,167 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package errorfs + +import ( + "fmt" + "go/scanner" + "go/token" + "strconv" + "strings" + + "github.com/cockroachdb/errors" +) + +// ParseInjectorFromDSL parses a string encoding a lisp-like DSL describing when +// errors should be injected. There are a handful of supported functions and +// primitives: +// - "ErrInjected" is a constant that injects an ErrInjected every time +// - "Any(injector, [injector]...)" injects the first error injected by the +// provided injectors, with short circuiting. +// - "(PathMatch pattern injector)" wraps an injector, calling into the +// provided injector only if an operation's file path matches the +// provided shell pattern. +// - "(OnIndex idx injector)" wraps an injector, calling into the provided +// injector on the idx-th time it's invoked. +// - "(Reads injector)" wraps an injector, calling into the provided injector +// only on read operations (eg, Open, Read, ReadAt, Stat, etc). +// - "(Writes injector)" wraps an injector, calling into the provided injector +// only on write operations (eg, Create, Rename, Write, WriteAt, etc) +// +// Example: (PathMatch "*.sst" (OnIndex 5 ErrInjected)) is a rule set that will +// inject an error on the 5-th I/O operation involving an sstable. +func ParseInjectorFromDSL(d string) (inj Injector, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + panic(r) + } + } + }() + + fset := token.NewFileSet() + file := fset.AddFile("", -1, len(d)) + var s scanner.Scanner + s.Init(file, []byte(strings.TrimSpace(d)), nil /* no error handler */, 0) + inj = parseInjectorDSLFunc(&s) + consumeTok(&s, token.SEMICOLON) + consumeTok(&s, token.EOF) + return inj, err +} + +// LabelledError is an error that also implements Injector, unconditionally +// injecting itself. It implements String() by returning its label. It +// implements Error() by returning its underlying error. +type LabelledError struct { + error + label string +} + +// String implements fmt.Stringer. +func (le LabelledError) String() string { return le.label } + +// MaybeError implements Injector. +func (le LabelledError) MaybeError(Op) error { return le } + +// AddErrorConstant defines a new error constant that may be used within the DSL +// parsed by ParseInjectorFromDSL and will inject the provided error. +func AddErrorConstant(le LabelledError) { + if _, ok := dslParsers[le.String()]; ok { + panic(fmt.Sprintf("errorfs: the identifier %s is already defined", le.String())) + } + dslParsers[le.String()] = func(*scanner.Scanner) Injector { return le } +} + +var dslParsers map[string]func(*scanner.Scanner) Injector + +func init() { + dslParsers = map[string]func(*scanner.Scanner) Injector{ + "Any": func(s *scanner.Scanner) Injector { + injs := []Injector{parseInjectorDSLFunc(s)} + pos, tok, lit := s.Scan() + for tok == token.LPAREN || tok == token.IDENT { + inj, err := parseInjectorDSLFuncFromPos(s, pos, tok, lit) + if err != nil { + panic(err) + } + injs = append(injs, inj) + pos, tok, lit = s.Scan() + } + if tok != token.RPAREN { + panic(errors.Errorf("errorfs: unexpected token %s (%q) at char %v", tok, lit, pos)) + } + return Any(injs...) + }, + "PathMatch": func(s *scanner.Scanner) Injector { + pattern := mustUnquote(consumeTok(s, token.STRING)) + next := parseInjectorDSLFunc(s) + consumeTok(s, token.RPAREN) + return PathMatch(pattern, next) + }, + "OnIndex": func(s *scanner.Scanner) Injector { + i, err := strconv.ParseInt(consumeTok(s, token.INT), 10, 32) + if err != nil { + panic(err) + } + next := parseInjectorDSLFunc(s) + consumeTok(s, token.RPAREN) + return OnIndex(int32(i), next) + }, + "Reads": func(s *scanner.Scanner) Injector { + next := parseInjectorDSLFunc(s) + consumeTok(s, token.RPAREN) + return Reads(next) + }, + "Writes": func(s *scanner.Scanner) Injector { + next := parseInjectorDSLFunc(s) + consumeTok(s, token.RPAREN) + return Writes(next) + }, + } + AddErrorConstant(ErrInjected) +} + +func parseInjectorDSLFunc(s *scanner.Scanner) Injector { + pos, tok, lit := s.Scan() + inj, err := parseInjectorDSLFuncFromPos(s, pos, tok, lit) + if err != nil { + panic(err) + } + return inj +} + +func parseInjectorDSLFuncFromPos( + s *scanner.Scanner, pos token.Pos, tok token.Token, lit string, +) (Injector, error) { + if tok == token.LPAREN { + pos, tok, lit = s.Scan() + } + if tok != token.IDENT { + return nil, errors.Errorf("errorfs: unexpected token %s (%q) at char %v", tok, lit, pos) + } + p, ok := dslParsers[lit] + if !ok { + return nil, errors.Errorf("errorfs: unknown ident %q", lit) + } + return p(s), nil +} + +func consumeTok(s *scanner.Scanner, expected token.Token) (lit string) { + pos, tok, lit := s.Scan() + if tok != expected { + panic(errors.Errorf("errorfs: unexpected token %s (%q) at char %v", tok, lit, pos)) + } + return lit +} + +func mustUnquote(lit string) string { + s, err := strconv.Unquote(lit) + if err != nil { + panic(errors.Newf("errorfs: unquoting %q: %v", lit, err)) + } + return s +} diff --git a/vfs/errorfs/errorfs.go b/vfs/errorfs/errorfs.go index 9ebc45a9133..79bd8d72ed3 100644 --- a/vfs/errorfs/errorfs.go +++ b/vfs/errorfs/errorfs.go @@ -9,6 +9,8 @@ import ( "io" "math/rand" "os" + "path/filepath" + "strings" "sync" "sync/atomic" "time" @@ -19,14 +21,25 @@ import ( ) // ErrInjected is an error artificially injected for testing fs error paths. -var ErrInjected = errors.New("injected error") +var ErrInjected = LabelledError{ + error: errors.New("injected error"), + label: "ErrInjected", +} + +// Op describes a filesystem operation. +type Op struct { + // Kind describes the particular kind of operation being performed. + Kind OpKind + // Path is the path of the file of the file being operated on. + Path string +} -// Op is an enum describing the type of operation. -type Op int +// OpKind is an enum describing the type of operation. +type OpKind int const ( // OpCreate describes a create file operation. - OpCreate Op = iota + OpCreate OpKind = iota // OpLink describes a hardlink operation. OpLink // OpOpen describes a file open operation. @@ -71,34 +84,70 @@ const ( OpFileFlush ) -// OpKind returns the operation's kind. -func (o Op) OpKind() OpKind { +// ReadOrWrite returns the operation's kind. +func (o OpKind) ReadOrWrite() OpReadWrite { switch o { case OpOpen, OpOpenDir, OpList, OpStat, OpGetDiskUsage, OpFileRead, OpFileReadAt, OpFileStat: - return OpKindRead + return OpIsRead case OpCreate, OpLink, OpRemove, OpRemoveAll, OpRename, OpReuseForRewrite, OpMkdirAll, OpLock, OpFileClose, OpFileWrite, OpFileWriteAt, OpFileSync, OpFileFlush, OpFilePreallocate: - return OpKindWrite + return OpIsWrite default: panic(fmt.Sprintf("unrecognized op %v\n", o)) } } -// OpKind is an enum describing whether an operation is a read or write +// OpReadWrite is an enum describing whether an operation is a read or write // operation. -type OpKind int +type OpReadWrite int const ( - // OpKindRead describes read operations. - OpKindRead OpKind = iota - // OpKindWrite describes write operations. - OpKindWrite + // OpIsRead describes read operations. + OpIsRead OpReadWrite = iota + // OpIsWrite describes write operations. + OpIsWrite ) -// OnIndex constructs an injector that returns an error on +// String implements fmt.Stringer. +func (kind OpReadWrite) String() string { + switch kind { + case OpIsRead: + return "Reads" + case OpIsWrite: + return "Writes" + default: + panic(fmt.Sprintf("unrecognized OpKind %d", kind)) + } +} + +// Reads wraps an injector, calling the inner Injector only if an operation is a +// read operation. +func Reads(next Injector) Injector { return opKindInj{kind: OpIsRead, next: next} } + +// Writes wraps an injector, calling the inner Injector only if an operation is +// a write operation. +func Writes(next Injector) Injector { return opKindInj{kind: OpIsWrite, next: next} } + +type opKindInj struct { + kind OpReadWrite + next Injector +} + +func (inj opKindInj) String() string { + return fmt.Sprintf("(%s %s)", inj.kind, inj.next.String()) +} + +func (inj opKindInj) MaybeError(op Op) error { + if inj.kind == op.Kind.ReadOrWrite() { + return inj.next.MaybeError(op) + } + return nil +} + +// OnIndex wraps an injector, calling into the injector on // the (n+1)-th invocation of its MaybeError function. It // may be passed to Wrap to inject an error into an FS. -func OnIndex(index int32) *InjectIndex { - ii := &InjectIndex{} +func OnIndex(index int32, next Injector) *InjectIndex { + ii := &InjectIndex{next: next} ii.index.Store(index) return ii } @@ -106,6 +155,12 @@ func OnIndex(index int32) *InjectIndex { // InjectIndex implements Injector, injecting an error at a specific index. type InjectIndex struct { index atomic.Int32 + next Injector +} + +// String implements fmt.Stringer. +func (ii *InjectIndex) String() string { + return fmt.Sprintf("(OnIndex %d %s)", ii.index.Load(), ii.next.String()) } // Index returns the index at which the error will be injected. @@ -115,24 +170,24 @@ func (ii *InjectIndex) Index() int32 { return ii.index.Load() } func (ii *InjectIndex) SetIndex(v int32) { ii.index.Store(v) } // MaybeError implements the Injector interface. -func (ii *InjectIndex) MaybeError(_ Op, _ string) error { - if ii.index.Add(-1) == -1 { - return errors.WithStack(ErrInjected) +func (ii *InjectIndex) MaybeError(op Op) error { + if ii.index.Add(-1) != -1 { + return nil } - return nil + return ii.next.MaybeError(op) } // WithProbability returns a function that returns an error with the provided // probability when passed op. It may be passed to Wrap to inject an error // into an ErrFS with the provided probability. p should be within the range // [0.0,1.0]. -func WithProbability(op OpKind, p float64) Injector { +func WithProbability(op OpReadWrite, p float64) Injector { mu := new(sync.Mutex) rnd := rand.New(rand.NewSource(time.Now().UnixNano())) - return InjectorFunc(func(currOp Op, _ string) error { + return InjectorFunc(func(currOp Op) error { mu.Lock() defer mu.Unlock() - if currOp.OpKind() == op && rnd.Float64() < p { + if currOp.Kind.ReadOrWrite() == op && rnd.Float64() < p { return errors.WithStack(ErrInjected) } return nil @@ -141,18 +196,77 @@ func WithProbability(op OpKind, p float64) Injector { // InjectorFunc implements the Injector interface for a function with // MaybeError's signature. -type InjectorFunc func(Op, string) error +type InjectorFunc func(Op) error + +// String implements fmt.Stringer. +func (f InjectorFunc) String() string { return "" } // MaybeError implements the Injector interface. -func (f InjectorFunc) MaybeError(op Op, path string) error { return f(op, path) } +func (f InjectorFunc) MaybeError(op Op) error { return f(op) } // Injector injects errors into FS operations. type Injector interface { + fmt.Stringer // MaybeError is invoked by an errorfs before an operation is executed. It // is passed an enum indicating the type of operation and a path of the // subject file or directory. If the operation takes two paths (eg, // Rename, Link), the original source path is provided. - MaybeError(op Op, path string) error + MaybeError(op Op) error +} + +// Any returns an injector that injects an error if any the provided injectors +// inject an error. +func Any(injectors ...Injector) Injector { + return anyInjector(injectors) +} + +type anyInjector []Injector + +func (a anyInjector) String() string { + var sb strings.Builder + sb.WriteString("(Any") + for _, inj := range a { + sb.WriteString(" ") + sb.WriteString(inj.String()) + } + sb.WriteString(")") + return sb.String() +} + +func (a anyInjector) MaybeError(op Op) error { + for _, inj := range a { + if err := inj.MaybeError(op); err != nil { + return err + } + } + return nil +} + +// PathMatch returns an injector that injects an error on file paths that +// match the provided pattern (according to filepath.Match) and for which the +// provided next injector injects an error. +func PathMatch(pattern string, next Injector) Injector { + return &pathMatch{pattern: pattern, next: next} +} + +type pathMatch struct { + pattern string + next Injector +} + +func (pm *pathMatch) String() string { + return fmt.Sprintf("(PathMatch %q %s)", pm.pattern, pm.next.String()) +} + +func (pm *pathMatch) MaybeError(op Op) error { + if matched, err := filepath.Match(pm.pattern, op.Path); err != nil { + // Only possible error is ErrBadPattern, indicating an issue with + // the test itself. + panic(err) + } else if matched { + return pm.next.MaybeError(op) + } + return nil } // FS implements vfs.FS, injecting errors into @@ -190,7 +304,7 @@ func (fs *FS) Unwrap() vfs.FS { // Create implements FS.Create. func (fs *FS) Create(name string) (vfs.File, error) { - if err := fs.inj.MaybeError(OpCreate, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpCreate, Path: name}); err != nil { return nil, err } f, err := fs.fs.Create(name) @@ -202,7 +316,7 @@ func (fs *FS) Create(name string) (vfs.File, error) { // Link implements FS.Link. func (fs *FS) Link(oldname, newname string) error { - if err := fs.inj.MaybeError(OpLink, oldname); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpLink, Path: oldname}); err != nil { return err } return fs.fs.Link(oldname, newname) @@ -210,7 +324,7 @@ func (fs *FS) Link(oldname, newname string) error { // Open implements FS.Open. func (fs *FS) Open(name string, opts ...vfs.OpenOption) (vfs.File, error) { - if err := fs.inj.MaybeError(OpOpen, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpOpen, Path: name}); err != nil { return nil, err } f, err := fs.fs.Open(name) @@ -226,7 +340,7 @@ func (fs *FS) Open(name string, opts ...vfs.OpenOption) (vfs.File, error) { // OpenReadWrite implements FS.OpenReadWrite. func (fs *FS) OpenReadWrite(name string, opts ...vfs.OpenOption) (vfs.File, error) { - if err := fs.inj.MaybeError(OpOpen, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpOpen, Path: name}); err != nil { return nil, err } f, err := fs.fs.OpenReadWrite(name) @@ -242,7 +356,7 @@ func (fs *FS) OpenReadWrite(name string, opts ...vfs.OpenOption) (vfs.File, erro // OpenDir implements FS.OpenDir. func (fs *FS) OpenDir(name string) (vfs.File, error) { - if err := fs.inj.MaybeError(OpOpenDir, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpOpenDir, Path: name}); err != nil { return nil, err } f, err := fs.fs.OpenDir(name) @@ -254,7 +368,7 @@ func (fs *FS) OpenDir(name string) (vfs.File, error) { // GetDiskUsage implements FS.GetDiskUsage. func (fs *FS) GetDiskUsage(path string) (vfs.DiskUsage, error) { - if err := fs.inj.MaybeError(OpGetDiskUsage, path); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpGetDiskUsage, Path: path}); err != nil { return vfs.DiskUsage{}, err } return fs.fs.GetDiskUsage(path) @@ -281,7 +395,7 @@ func (fs *FS) Remove(name string) error { return nil } - if err := fs.inj.MaybeError(OpRemove, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpRemove, Path: name}); err != nil { return err } return fs.fs.Remove(name) @@ -289,7 +403,7 @@ func (fs *FS) Remove(name string) error { // RemoveAll implements FS.RemoveAll. func (fs *FS) RemoveAll(fullname string) error { - if err := fs.inj.MaybeError(OpRemoveAll, fullname); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpRemoveAll, Path: fullname}); err != nil { return err } return fs.fs.RemoveAll(fullname) @@ -297,7 +411,7 @@ func (fs *FS) RemoveAll(fullname string) error { // Rename implements FS.Rename. func (fs *FS) Rename(oldname, newname string) error { - if err := fs.inj.MaybeError(OpRename, oldname); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpRename, Path: oldname}); err != nil { return err } return fs.fs.Rename(oldname, newname) @@ -305,7 +419,7 @@ func (fs *FS) Rename(oldname, newname string) error { // ReuseForWrite implements FS.ReuseForWrite. func (fs *FS) ReuseForWrite(oldname, newname string) (vfs.File, error) { - if err := fs.inj.MaybeError(OpReuseForRewrite, oldname); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpReuseForRewrite, Path: oldname}); err != nil { return nil, err } return fs.fs.ReuseForWrite(oldname, newname) @@ -313,7 +427,7 @@ func (fs *FS) ReuseForWrite(oldname, newname string) (vfs.File, error) { // MkdirAll implements FS.MkdirAll. func (fs *FS) MkdirAll(dir string, perm os.FileMode) error { - if err := fs.inj.MaybeError(OpMkdirAll, dir); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpMkdirAll, Path: dir}); err != nil { return err } return fs.fs.MkdirAll(dir, perm) @@ -321,7 +435,7 @@ func (fs *FS) MkdirAll(dir string, perm os.FileMode) error { // Lock implements FS.Lock. func (fs *FS) Lock(name string) (io.Closer, error) { - if err := fs.inj.MaybeError(OpLock, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpLock, Path: name}); err != nil { return nil, err } return fs.fs.Lock(name) @@ -329,7 +443,7 @@ func (fs *FS) Lock(name string) (io.Closer, error) { // List implements FS.List. func (fs *FS) List(dir string) ([]string, error) { - if err := fs.inj.MaybeError(OpList, dir); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpList, Path: dir}); err != nil { return nil, err } return fs.fs.List(dir) @@ -337,7 +451,7 @@ func (fs *FS) List(dir string) ([]string, error) { // Stat implements FS.Stat. func (fs *FS) Stat(name string) (os.FileInfo, error) { - if err := fs.inj.MaybeError(OpStat, name); err != nil { + if err := fs.inj.MaybeError(Op{Kind: OpStat, Path: name}); err != nil { return nil, err } return fs.fs.Stat(name) @@ -358,35 +472,35 @@ func (f *errorFile) Close() error { } func (f *errorFile) Read(p []byte) (int, error) { - if err := f.inj.MaybeError(OpFileRead, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileRead, Path: f.path}); err != nil { return 0, err } return f.file.Read(p) } func (f *errorFile) ReadAt(p []byte, off int64) (int, error) { - if err := f.inj.MaybeError(OpFileReadAt, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileReadAt, Path: f.path}); err != nil { return 0, err } return f.file.ReadAt(p, off) } func (f *errorFile) Write(p []byte) (int, error) { - if err := f.inj.MaybeError(OpFileWrite, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileWrite, Path: f.path}); err != nil { return 0, err } return f.file.Write(p) } func (f *errorFile) WriteAt(p []byte, ofs int64) (int, error) { - if err := f.inj.MaybeError(OpFileWriteAt, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileWriteAt, Path: f.path}); err != nil { return 0, err } return f.file.WriteAt(p, ofs) } func (f *errorFile) Stat() (os.FileInfo, error) { - if err := f.inj.MaybeError(OpFileStat, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileStat, Path: f.path}); err != nil { return nil, err } return f.file.Stat() @@ -398,14 +512,14 @@ func (f *errorFile) Prefetch(offset, length int64) error { } func (f *errorFile) Preallocate(offset, length int64) error { - if err := f.inj.MaybeError(OpFilePreallocate, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFilePreallocate, Path: f.path}); err != nil { return err } return f.file.Preallocate(offset, length) } func (f *errorFile) Sync() error { - if err := f.inj.MaybeError(OpFileSync, f.path); err != nil { + if err := f.inj.MaybeError(Op{Kind: OpFileSync, Path: f.path}); err != nil { return err } return f.file.Sync() diff --git a/vfs/errorfs/errorfs_test.go b/vfs/errorfs/errorfs_test.go new file mode 100644 index 00000000000..644c4ceefdf --- /dev/null +++ b/vfs/errorfs/errorfs_test.go @@ -0,0 +1,34 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package errorfs + +import ( + "fmt" + "strings" + "testing" + + "github.com/cockroachdb/datadriven" +) + +func TestErrorFS(t *testing.T) { + var sb strings.Builder + datadriven.RunTest(t, "testdata/errorfs", func(t *testing.T, td *datadriven.TestData) string { + sb.Reset() + switch td.Cmd { + case "parse-dsl": + for _, l := range strings.Split(strings.TrimSpace(td.Input), "\n") { + inj, err := ParseInjectorFromDSL(l) + if err != nil { + fmt.Fprintf(&sb, "parsing err: %s\n", err) + } else { + fmt.Fprintf(&sb, "%s\n", inj.String()) + } + } + return sb.String() + default: + return fmt.Sprintf("unrecognized command %q", td.Cmd) + } + }) +} diff --git a/vfs/errorfs/testdata/errorfs b/vfs/errorfs/testdata/errorfs new file mode 100644 index 00000000000..a2253c4aa86 --- /dev/null +++ b/vfs/errorfs/testdata/errorfs @@ -0,0 +1,43 @@ +parse-dsl +ErrInjected +(PathMatch "foo/*.sst" ErrInjected) +(OnIndex 1 ErrInjected) +(PathMatch "foo/bar/*.sst" (OnIndex 1 ErrInjected)) +(Any ErrInjected ErrInjected ErrInjected) +(Any (OnIndex 2 ErrInjected) (PathMatch "*.sst" ErrInjected)) +(Reads (PathMatch "*.sst" ErrInjected)) +(Writes (PathMatch "*.sst" ErrInjected)) +---- +ErrInjected +(PathMatch "foo/*.sst" ErrInjected) +(OnIndex 1 ErrInjected) +(PathMatch "foo/bar/*.sst" (OnIndex 1 ErrInjected)) +(Any ErrInjected ErrInjected ErrInjected) +(Any (OnIndex 2 ErrInjected) (PathMatch "*.sst" ErrInjected)) +(Reads (PathMatch "*.sst" ErrInjected)) +(Writes (PathMatch "*.sst" ErrInjected)) + +parse-dsl +errInjected +ErrInjected() +(PathMatch foo/*.sst ErrInjected) +(PathMatch "foo/*.sst" alwoes) +(PathMatch "foo/*.sst" "" ErrInjected) +(PathMatch ErrInjected "foo/*.sst") +(OnIndex ErrInjected "foo/*.sst") +(Any 1 4 5) +(Any ErrInjected ErrInjected ErrInjected +(OnIndex foo ErrInjected) +(OnIndex 9223372036854775807 ErrInjected) +---- +parsing err: errorfs: unknown ident "errInjected" +parsing err: errorfs: unexpected token ( ("") at char 12 +parsing err: errorfs: unexpected token IDENT ("foo") at char 12 +parsing err: errorfs: unknown ident "alwoes" +parsing err: errorfs: unexpected token STRING ("\"\"") at char 24 +parsing err: errorfs: unexpected token IDENT ("ErrInjected") at char 12 +parsing err: errorfs: unexpected token IDENT ("ErrInjected") at char 10 +parsing err: errorfs: unexpected token INT ("1") at char 6 +parsing err: errorfs: unexpected token ; ("\n") at char 41 +parsing err: errorfs: unexpected token IDENT ("foo") at char 10 +parsing err: strconv.ParseInt: parsing "9223372036854775807": value out of range