diff --git a/batch_test.go b/batch_test.go index e3fd0773a8..88b004572d 100644 --- a/batch_test.go +++ b/batch_test.go @@ -22,6 +22,7 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/batchskl" + "github.com/cockroachdb/pebble/internal/itertest" "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/testkeys" "github.com/cockroachdb/pebble/vfs" @@ -911,7 +912,7 @@ func TestBatchIter(t *testing.T) { } iter := b.newInternalIter(&options) defer iter.Close() - return runInternalIterCmd(t, d, iter) + return itertest.RunInternalIterCmd(t, d, iter) default: return fmt.Sprintf("unknown command: %s", d.Cmd) @@ -1028,7 +1029,7 @@ func TestFlushableBatchIter(t *testing.T) { case "iter": iter := b.newIter(nil) defer iter.Close() - return runInternalIterCmd(t, d, iter) + return itertest.RunInternalIterCmd(t, d, iter) default: return fmt.Sprintf("unknown command: %s", d.Cmd) @@ -1084,7 +1085,7 @@ func TestFlushableBatch(t *testing.T) { iter := b.newIter(&opts) defer iter.Close() - return runInternalIterCmd(t, d, iter) + return itertest.RunInternalIterCmd(t, d, iter) case "dump": if len(d.CmdArgs) != 1 || len(d.CmdArgs[0].Vals) != 1 || d.CmdArgs[0].Key != "seq" { diff --git a/data_test.go b/data_test.go index c1b209d5c5..ef0b4a21d6 100644 --- a/data_test.go +++ b/data_test.go @@ -34,21 +34,6 @@ import ( "github.com/stretchr/testify/require" ) -type iterCmdOpts struct { - verboseKey bool - stats *base.InternalIteratorStats -} - -type iterCmdOpt func(*iterCmdOpts) - -func iterCmdVerboseKey(opts *iterCmdOpts) { opts.verboseKey = true } - -func iterCmdStats(stats *base.InternalIteratorStats) iterCmdOpt { - return func(opts *iterCmdOpts) { - opts.stats = stats - } -} - func runGetCmd(t testing.TB, td *datadriven.TestData, d *DB) string { snap := Snapshot{ db: d, @@ -394,122 +379,6 @@ func writeRangeKeys(b io.Writer, iter *Iterator) { } } -func runInternalIterCmd( - t *testing.T, d *datadriven.TestData, iter internalIterator, opts ...iterCmdOpt, -) string { - var o iterCmdOpts - for _, opt := range opts { - opt(&o) - } - - getKV := func(key *InternalKey, val LazyValue) (*InternalKey, []byte) { - v, _, err := val.Value(nil) - require.NoError(t, err) - return key, v - } - var b bytes.Buffer - var prefix []byte - for _, line := range strings.Split(d.Input, "\n") { - parts := strings.Fields(line) - if len(parts) == 0 { - continue - } - var key *InternalKey - var value []byte - switch parts[0] { - case "seek-ge": - if len(parts) < 2 || len(parts) > 3 { - return "seek-ge []\n" - } - prefix = nil - var flags base.SeekGEFlags - if len(parts) == 3 { - if trySeekUsingNext, err := strconv.ParseBool(parts[2]); err != nil { - return err.Error() - } else if trySeekUsingNext { - flags = flags.EnableTrySeekUsingNext() - } - } - key, value = getKV(iter.SeekGE([]byte(strings.TrimSpace(parts[1])), flags)) - case "seek-prefix-ge": - if len(parts) != 2 && len(parts) != 3 { - return "seek-prefix-ge []\n" - } - prefix = []byte(strings.TrimSpace(parts[1])) - var flags base.SeekGEFlags - if len(parts) == 3 { - if trySeekUsingNext, err := strconv.ParseBool(parts[2]); err != nil { - return err.Error() - } else if trySeekUsingNext { - flags = flags.EnableTrySeekUsingNext() - } - } - key, value = getKV(iter.SeekPrefixGE(prefix, prefix /* key */, flags)) - case "seek-lt": - if len(parts) != 2 { - return "seek-lt \n" - } - prefix = nil - key, value = getKV(iter.SeekLT([]byte(strings.TrimSpace(parts[1])), base.SeekLTFlagsNone)) - case "first": - prefix = nil - key, value = getKV(iter.First()) - case "last": - prefix = nil - key, value = getKV(iter.Last()) - case "next": - key, value = getKV(iter.Next()) - case "prev": - key, value = getKV(iter.Prev()) - case "set-bounds": - if len(parts) <= 1 || len(parts) > 3 { - return "set-bounds lower= upper=\n" - } - var lower []byte - var upper []byte - for _, part := range parts[1:] { - arg := strings.Split(strings.TrimSpace(part), "=") - switch arg[0] { - case "lower": - lower = []byte(arg[1]) - case "upper": - upper = []byte(arg[1]) - default: - return fmt.Sprintf("set-bounds: unknown arg: %s", arg) - } - } - iter.SetBounds(lower, upper) - continue - case "stats": - if o.stats != nil { - // The timing is non-deterministic, so set to 0. - o.stats.BlockReadDuration = 0 - fmt.Fprintf(&b, "%+v\n", *o.stats) - } - continue - case "reset-stats": - if o.stats != nil { - *o.stats = base.InternalIteratorStats{} - } - continue - default: - return fmt.Sprintf("unknown op: %s", parts[0]) - } - if key != nil { - if o.verboseKey { - fmt.Fprintf(&b, "%s:%s\n", key, value) - } else { - fmt.Fprintf(&b, "%s:%s\n", key.UserKey, value) - } - } else if err := iter.Error(); err != nil { - fmt.Fprintf(&b, "err=%v\n", err) - } else { - fmt.Fprintf(&b, ".\n") - } - } - return b.String() -} - func runBatchDefineCmd(d *datadriven.TestData, b *Batch) error { for _, line := range strings.Split(d.Input, "\n") { parts := strings.Fields(line) diff --git a/external_iterator_test.go b/external_iterator_test.go index a8134275dc..77afd4dcc7 100644 --- a/external_iterator_test.go +++ b/external_iterator_test.go @@ -15,6 +15,7 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/cache" + "github.com/cockroachdb/pebble/internal/itertest" "github.com/cockroachdb/pebble/internal/testkeys" "github.com/cockroachdb/pebble/objstorage/objstorageprovider" "github.com/cockroachdb/pebble/sstable" @@ -123,7 +124,7 @@ func TestSimpleLevelIter(t *testing.T) { it := &simpleLevelIter{cmp: o.Comparer.Compare, iters: internalIters} it.init(IterOptions{}) - response := runInternalIterCmd(t, td, it) + response := itertest.RunInternalIterCmd(t, td, it) require.NoError(t, it.Close()) return response default: diff --git a/internal/itertest/datadriven.go b/internal/itertest/datadriven.go new file mode 100644 index 0000000000..6c2feef3f2 --- /dev/null +++ b/internal/itertest/datadriven.go @@ -0,0 +1,196 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +// Package itertest provides facilities for testing internal iterators. +package itertest + +import ( + "bytes" + "fmt" + "io" + "strconv" + "strings" + "testing" + + "github.com/cockroachdb/datadriven" + "github.com/cockroachdb/pebble/internal/base" + "github.com/stretchr/testify/require" +) + +type iterCmdOpts struct { + fmtKV func(io.Writer, *base.InternalKey, []byte, base.InternalIterator) + stats *base.InternalIteratorStats +} + +// An IterOpt configures the behavior of RunInternalIterCmd. +type IterOpt func(*iterCmdOpts) + +// Verbose configures RunInternalIterCmd to output verbose results. +func Verbose(opts *iterCmdOpts) { opts.fmtKV = verboseFmt } + +// Condensed configures RunInternalIterCmd to output condensed results without +// values. +func Condensed(opts *iterCmdOpts) { opts.fmtKV = condensedFmt } + +// WithStats configures RunInternalIterCmd to collect iterator stats in the +// struct pointed to by stats. +func WithStats(stats *base.InternalIteratorStats) IterOpt { + return func(opts *iterCmdOpts) { + opts.stats = stats + } +} + +func defaultFmt(w io.Writer, key *base.InternalKey, v []byte, iter base.InternalIterator) { + if key != nil { + fmt.Fprintf(w, "%s:%s\n", key.UserKey, v) + } else if err := iter.Error(); err != nil { + fmt.Fprintf(w, "err=%v\n", err) + } else { + fmt.Fprintf(w, ".\n") + } +} + +func condensedFmt(w io.Writer, key *base.InternalKey, v []byte, iter base.InternalIterator) { + if key != nil { + fmt.Fprintf(w, "<%s:%d>", key.UserKey, key.SeqNum()) + } else if err := iter.Error(); err != nil { + fmt.Fprintf(w, "err=%v", err) + } else { + fmt.Fprint(w, ".") + } +} + +func verboseFmt(w io.Writer, key *base.InternalKey, v []byte, iter base.InternalIterator) { + if key != nil { + fmt.Fprintf(w, "%s:%s\n", key, v) + return + } + defaultFmt(w, key, v, iter) +} + +// RunInternalIterCmd evaluates a datadriven command controlling an internal +// iterator, returning a string with the results of the iterator operations. +func RunInternalIterCmd( + t *testing.T, d *datadriven.TestData, iter base.InternalIterator, opts ...IterOpt, +) string { + var buf bytes.Buffer + RunInternalIterCmdWriter(t, &buf, d, iter, opts...) + return buf.String() +} + +// RunInternalIterCmdWriter evaluates a datadriven command controlling an +// internal iterator, writing the results of the iterator operations to the +// provided Writer. +func RunInternalIterCmdWriter( + t *testing.T, w io.Writer, d *datadriven.TestData, iter base.InternalIterator, opts ...IterOpt, +) { + o := iterCmdOpts{fmtKV: defaultFmt} + for _, opt := range opts { + opt(&o) + } + + getKV := func(key *base.InternalKey, val base.LazyValue) (*base.InternalKey, []byte) { + v, _, err := val.Value(nil) + require.NoError(t, err) + return key, v + } + var prefix []byte + for _, line := range strings.Split(d.Input, "\n") { + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + var key *base.InternalKey + var value []byte + switch parts[0] { + case "seek-ge": + if len(parts) < 2 || len(parts) > 3 { + fmt.Fprint(w, "seek-ge []\n") + return + } + prefix = nil + var flags base.SeekGEFlags + if len(parts) == 3 { + if trySeekUsingNext, err := strconv.ParseBool(parts[2]); err != nil { + fmt.Fprintf(w, "%s", err.Error()) + return + } else if trySeekUsingNext { + flags = flags.EnableTrySeekUsingNext() + } + } + key, value = getKV(iter.SeekGE([]byte(strings.TrimSpace(parts[1])), flags)) + case "seek-prefix-ge": + if len(parts) != 2 && len(parts) != 3 { + fmt.Fprint(w, "seek-prefix-ge []\n") + return + } + prefix = []byte(strings.TrimSpace(parts[1])) + var flags base.SeekGEFlags + if len(parts) == 3 { + if trySeekUsingNext, err := strconv.ParseBool(parts[2]); err != nil { + fmt.Fprintf(w, "%s", err.Error()) + return + } else if trySeekUsingNext { + flags = flags.EnableTrySeekUsingNext() + } + } + key, value = getKV(iter.SeekPrefixGE(prefix, prefix /* key */, flags)) + case "seek-lt": + if len(parts) != 2 { + fmt.Fprint(w, "seek-lt \n") + return + } + prefix = nil + key, value = getKV(iter.SeekLT([]byte(strings.TrimSpace(parts[1])), base.SeekLTFlagsNone)) + case "first": + prefix = nil + key, value = getKV(iter.First()) + case "last": + prefix = nil + key, value = getKV(iter.Last()) + case "next": + key, value = getKV(iter.Next()) + case "prev": + key, value = getKV(iter.Prev()) + case "set-bounds": + if len(parts) <= 1 || len(parts) > 3 { + fmt.Fprint(w, "set-bounds lower= upper=\n") + return + } + var lower []byte + var upper []byte + for _, part := range parts[1:] { + arg := strings.Split(strings.TrimSpace(part), "=") + switch arg[0] { + case "lower": + lower = []byte(arg[1]) + case "upper": + upper = []byte(arg[1]) + default: + fmt.Fprintf(w, "set-bounds: unknown arg: %s", arg) + return + } + } + iter.SetBounds(lower, upper) + continue + case "stats": + if o.stats != nil { + // The timing is non-deterministic, so set to 0. + o.stats.BlockReadDuration = 0 + fmt.Fprintf(w, "%+v\n", *o.stats) + } + continue + case "reset-stats": + if o.stats != nil { + *o.stats = base.InternalIteratorStats{} + } + continue + default: + fmt.Fprintf(w, "unknown op: %s", parts[0]) + return + } + o.fmtKV(w, key, value, iter) + + } +} diff --git a/level_iter_test.go b/level_iter_test.go index 9f7c0e70a7..63d448e68b 100644 --- a/level_iter_test.go +++ b/level_iter_test.go @@ -15,6 +15,7 @@ import ( "github.com/cockroachdb/datadriven" "github.com/cockroachdb/pebble/bloom" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/itertest" "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/rangedel" @@ -92,7 +93,7 @@ func TestLevelIter(t *testing.T) { // Fake up the range deletion initialization. iter.initRangeDel(new(keyspan.FragmentIterator)) iter.disableInvariants = true - return runInternalIterCmd(t, d, iter, iterCmdVerboseKey) + return itertest.RunInternalIterCmd(t, d, iter, itertest.Verbose) case "load": // The "load" command allows testing the iterator options passed to load @@ -331,7 +332,7 @@ func TestLevelIterBoundaries(t *testing.T) { iter = nil }() } - return runInternalIterCmd(t, d, iter, iterCmdVerboseKey) + return itertest.RunInternalIterCmd(t, d, iter, itertest.Verbose) case "file-pos": // Returns the FileNum at which the iterator is positioned. @@ -427,7 +428,7 @@ func TestLevelIterSeek(t *testing.T) { manifest.Level(level), internalIterOpts{stats: &stats}) defer iter.Close() iter.initRangeDel(&iter.rangeDelIter) - return runInternalIterCmd(t, d, iter, iterCmdVerboseKey, iterCmdStats(&stats)) + return itertest.RunInternalIterCmd(t, d, iter, itertest.Verbose, itertest.WithStats(&stats)) case "iters-created": return fmt.Sprintf("%d", lt.itersCreated) diff --git a/mem_table_test.go b/mem_table_test.go index 724f25a240..62ea9c60f2 100644 --- a/mem_table_test.go +++ b/mem_table_test.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/arenaskl" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/itertest" "github.com/cockroachdb/pebble/internal/rangekey" "github.com/stretchr/testify/require" "golang.org/x/exp/rand" @@ -287,7 +288,7 @@ func TestMemTableIter(t *testing.T) { } iter := mem.newIter(&options) defer iter.Close() - return runInternalIterCmd(t, d, iter) + return itertest.RunInternalIterCmd(t, d, iter) default: return fmt.Sprintf("unknown command: %s", d.Cmd) diff --git a/merging_iter_test.go b/merging_iter_test.go index ae37ec3d2e..2e3d994af3 100644 --- a/merging_iter_test.go +++ b/merging_iter_test.go @@ -14,6 +14,7 @@ import ( "github.com/cockroachdb/datadriven" "github.com/cockroachdb/pebble/bloom" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/itertest" "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/rangedel" @@ -68,7 +69,7 @@ func TestMergingIterSeek(t *testing.T) { iter := newMergingIter(nil /* logger */, &stats, DefaultComparer.Compare, func(a []byte) int { return len(a) }, iters...) defer iter.Close() - return runInternalIterCmd(t, d, iter) + return itertest.RunInternalIterCmd(t, d, iter) default: return fmt.Sprintf("unknown command: %s", d.Cmd) @@ -127,7 +128,7 @@ func TestMergingIterNextPrev(t *testing.T) { iter := newMergingIter(nil /* logger */, &stats, DefaultComparer.Compare, func(a []byte) int { return len(a) }, iters...) defer iter.Close() - return runInternalIterCmd(t, d, iter) + return itertest.RunInternalIterCmd(t, d, iter) default: return fmt.Sprintf("unknown command: %s", d.Cmd) @@ -276,7 +277,8 @@ func TestMergingIterCornerCases(t *testing.T) { // (https://github.com/cockroachdb/pebble/pull/3037 caused a SIGSEGV due // to a nil pointer dereference). miter.SetContext(context.Background()) - return runInternalIterCmd(t, d, miter, iterCmdVerboseKey, iterCmdStats(&stats)) + return itertest.RunInternalIterCmd(t, d, miter, + itertest.Verbose, itertest.WithStats(&stats)) default: return fmt.Sprintf("unknown command: %s", d.Cmd) } diff --git a/scan_internal_test.go b/scan_internal_test.go index 0c8afcffcc..03becb4cd8 100644 --- a/scan_internal_test.go +++ b/scan_internal_test.go @@ -17,6 +17,7 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/bloom" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/itertest" "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/testkeys" "github.com/cockroachdb/pebble/objstorage/remote" @@ -525,8 +526,7 @@ func TestPointCollapsingIter(t *testing.T) { } pcIter.iter.Init(base.DefaultComparer, f, ksIter, keyspan.InterleavingIterOpts{}) defer pcIter.Close() - - return runInternalIterCmd(t, d, pcIter, iterCmdVerboseKey) + return itertest.RunInternalIterCmd(t, d, pcIter, itertest.Verbose) default: return fmt.Sprintf("unknown command: %s", d.Cmd) diff --git a/sstable/block_test.go b/sstable/block_test.go index 7d1aed6e9f..14e6f7ff8a 100644 --- a/sstable/block_test.go +++ b/sstable/block_test.go @@ -15,6 +15,7 @@ import ( "github.com/cockroachdb/datadriven" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/itertest" "github.com/stretchr/testify/require" "golang.org/x/exp/rand" ) @@ -253,43 +254,7 @@ func TestBlockIter2(t *testing.T) { if err != nil { return err.Error() } - - var b bytes.Buffer - for _, line := range strings.Split(d.Input, "\n") { - parts := strings.Fields(line) - if len(parts) == 0 { - continue - } - switch parts[0] { - case "seek-ge": - if len(parts) != 2 { - return "seek-ge \n" - } - iter.SeekGE([]byte(strings.TrimSpace(parts[1])), base.SeekGEFlagsNone) - case "seek-lt": - if len(parts) != 2 { - return "seek-lt \n" - } - iter.SeekLT([]byte(strings.TrimSpace(parts[1])), base.SeekLTFlagsNone) - case "first": - iter.First() - case "last": - iter.Last() - case "next": - iter.Next() - case "prev": - iter.Prev() - } - if iter.valid() { - fmt.Fprintf(&b, "<%s:%d>", iter.Key().UserKey, iter.Key().SeqNum()) - } else if err := iter.Error(); err != nil { - fmt.Fprintf(&b, "", err) - } else { - fmt.Fprintf(&b, ".") - } - } - b.WriteString("\n") - return b.String() + return itertest.RunInternalIterCmd(t, d, iter, itertest.Condensed) default: return fmt.Sprintf("unknown command: %s", d.Cmd)