From 41239f892a2abb181aec7f4d16fc550b7df57e20 Mon Sep 17 00:00:00 2001 From: sumeerbhola Date: Tue, 7 Sep 2021 16:01:45 -0400 Subject: [PATCH] db: add test demonstrating current SINGLEDEL behavior --- compaction_test.go | 275 ++++++++++++++------------- testdata/singledel_manual_compaction | 77 ++++++++ 2 files changed, 216 insertions(+), 136 deletions(-) create mode 100644 testdata/singledel_manual_compaction diff --git a/compaction_test.go b/compaction_test.go index 027d1fb37f..697b92b265 100644 --- a/compaction_test.go +++ b/compaction_test.go @@ -1135,167 +1135,170 @@ func TestManualCompaction(t *testing.T) { var ongoingCompaction *compaction - datadriven.RunTest(t, "testdata/manual_compaction", func(td *datadriven.TestData) string { - switch td.Cmd { - case "reset": - reset() - return "" - - case "batch": - b := d.NewIndexedBatch() - if err := runBatchDefineCmd(td, b); err != nil { - return err.Error() - } - require.NoError(t, b.Commit(nil)) - return "" + paths := []string{"testdata/manual_compaction", "testdata/singledel_manual_compaction"} + for _, path := range paths { + datadriven.RunTest(t, path, func(td *datadriven.TestData) string { + switch td.Cmd { + case "reset": + reset() + return "" - case "build": - if err := runBuildCmd(td, d, mem); err != nil { - return err.Error() - } - return "" + case "batch": + b := d.NewIndexedBatch() + if err := runBatchDefineCmd(td, b); err != nil { + return err.Error() + } + require.NoError(t, b.Commit(nil)) + return "" - case "compact": - if err := runCompactCmd(td, d); err != nil { - return err.Error() - } - return runLSMCmd(td, d) + case "build": + if err := runBuildCmd(td, d, mem); err != nil { + return err.Error() + } + return "" - case "define": - if d != nil { - if err := d.Close(); err != nil { + case "compact": + if err := runCompactCmd(td, d); err != nil { return err.Error() } - } + return runLSMCmd(td, d) - mem = vfs.NewMem() - opts := &Options{ - FS: mem, - DebugCheck: DebugCheckLevels, - } - opts.private.disableAutomaticCompactions = true + case "define": + if d != nil { + if err := d.Close(); err != nil { + return err.Error() + } + } - var err error - if d, err = runDBDefineCmd(td, opts); err != nil { - return err.Error() - } + mem = vfs.NewMem() + opts := &Options{ + FS: mem, + DebugCheck: DebugCheckLevels, + } + opts.private.disableAutomaticCompactions = true - d.mu.Lock() - s := d.mu.versions.currentVersion().String() - d.mu.Unlock() - return s + var err error + if d, err = runDBDefineCmd(td, opts); err != nil { + return err.Error() + } - case "ingest": - if err := runIngestCmd(td, d, mem); err != nil { - return err.Error() - } - return runLSMCmd(td, d) - - case "iter": - // TODO(peter): runDBDefineCmd doesn't properly update the visible - // sequence number. So we have to use a snapshot with a very large - // sequence number, otherwise the DB appears empty. - snap := Snapshot{ - db: d, - seqNum: InternalKeySeqNumMax, - } - iter := snap.NewIter(nil) - return runIterCmd(td, iter, true) + d.mu.Lock() + s := d.mu.versions.currentVersion().String() + d.mu.Unlock() + return s - case "async-compact": - var s string - ch := make(chan error, 1) - go func() { - if err := runCompactCmd(td, d); err != nil { - ch <- err + case "ingest": + if err := runIngestCmd(td, d, mem); err != nil { + return err.Error() + } + return runLSMCmd(td, d) + + case "iter": + // TODO(peter): runDBDefineCmd doesn't properly update the visible + // sequence number. So we have to use a snapshot with a very large + // sequence number, otherwise the DB appears empty. + snap := Snapshot{ + db: d, + seqNum: InternalKeySeqNumMax, + } + iter := snap.NewIter(nil) + return runIterCmd(td, iter, true) + + case "async-compact": + var s string + ch := make(chan error, 1) + go func() { + if err := runCompactCmd(td, d); err != nil { + ch <- err + close(ch) + return + } + d.mu.Lock() + s = d.mu.versions.currentVersion().DebugString(base.DefaultFormatter) + d.mu.Unlock() close(ch) - return + }() + + manualDone := func() bool { + select { + case <-ch: + return true + default: + return false + } } - d.mu.Lock() - s = d.mu.versions.currentVersion().DebugString(base.DefaultFormatter) - d.mu.Unlock() - close(ch) - }() - manualDone := func() bool { - select { - case <-ch: - return true - default: - return false + err := try(100*time.Microsecond, 20*time.Second, func() error { + if manualDone() { + return nil + } + + d.mu.Lock() + defer d.mu.Unlock() + if len(d.mu.compact.manual) == 0 { + return errors.New("no manual compaction queued") + } + manual := d.mu.compact.manual[0] + if manual.retries == 0 { + return errors.New("manual compaction has not been retried") + } + return nil + }) + if err != nil { + return err.Error() } - } - err := try(100*time.Microsecond, 20*time.Second, func() error { if manualDone() { - return nil + return "manual compaction did not block for ongoing\n" + s } d.mu.Lock() - defer d.mu.Unlock() - if len(d.mu.compact.manual) == 0 { - return errors.New("no manual compaction queued") - } - manual := d.mu.compact.manual[0] - if manual.retries == 0 { - return errors.New("manual compaction has not been retried") + delete(d.mu.compact.inProgress, ongoingCompaction) + d.mu.compact.compactingCount-- + ongoingCompaction = nil + d.maybeScheduleCompaction() + d.mu.Unlock() + if err := <-ch; err != nil { + return err.Error() } - return nil - }) - if err != nil { - return err.Error() - } - - if manualDone() { - return "manual compaction did not block for ongoing\n" + s - } + return "manual compaction blocked until ongoing finished\n" + s - d.mu.Lock() - delete(d.mu.compact.inProgress, ongoingCompaction) - d.mu.compact.compactingCount-- - ongoingCompaction = nil - d.maybeScheduleCompaction() - d.mu.Unlock() - if err := <-ch; err != nil { - return err.Error() - } - return "manual compaction blocked until ongoing finished\n" + s - - case "add-ongoing-compaction": - var startLevel int - var outputLevel int - td.ScanArgs(t, "startLevel", &startLevel) - td.ScanArgs(t, "outputLevel", &outputLevel) - ongoingCompaction = &compaction{ - inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}}, - } - ongoingCompaction.startLevel = &ongoingCompaction.inputs[0] - ongoingCompaction.outputLevel = &ongoingCompaction.inputs[1] - d.mu.Lock() - d.mu.compact.inProgress[ongoingCompaction] = struct{}{} - d.mu.compact.compactingCount++ - d.mu.Unlock() - return "" + case "add-ongoing-compaction": + var startLevel int + var outputLevel int + td.ScanArgs(t, "startLevel", &startLevel) + td.ScanArgs(t, "outputLevel", &outputLevel) + ongoingCompaction = &compaction{ + inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}}, + } + ongoingCompaction.startLevel = &ongoingCompaction.inputs[0] + ongoingCompaction.outputLevel = &ongoingCompaction.inputs[1] + d.mu.Lock() + d.mu.compact.inProgress[ongoingCompaction] = struct{}{} + d.mu.compact.compactingCount++ + d.mu.Unlock() + return "" - case "remove-ongoing-compaction": - d.mu.Lock() - delete(d.mu.compact.inProgress, ongoingCompaction) - d.mu.compact.compactingCount-- - ongoingCompaction = nil - d.mu.Unlock() - return "" + case "remove-ongoing-compaction": + d.mu.Lock() + delete(d.mu.compact.inProgress, ongoingCompaction) + d.mu.compact.compactingCount-- + ongoingCompaction = nil + d.mu.Unlock() + return "" - case "set-concurrent-compactions": - td.ScanArgs(t, "num", &d.opts.MaxConcurrentCompactions) - return "" + case "set-concurrent-compactions": + td.ScanArgs(t, "num", &d.opts.MaxConcurrentCompactions) + return "" - case "wait-pending-table-stats": - return runTableStatsCmd(td, d) + case "wait-pending-table-stats": + return runTableStatsCmd(td, d) - default: - return fmt.Sprintf("unknown command: %s", td.Cmd) - } - }) + default: + return fmt.Sprintf("unknown command: %s", td.Cmd) + } + }) + } } func TestCompactionFindGrandparentLimit(t *testing.T) { diff --git a/testdata/singledel_manual_compaction b/testdata/singledel_manual_compaction new file mode 100644 index 0000000000..ff83e37648 --- /dev/null +++ b/testdata/singledel_manual_compaction @@ -0,0 +1,77 @@ +# This is not actually a manual compaction test, and simply uses manual +# compaction to demonstrate single delete semantics. Specifically, it +# demonstrates that the behavior can be non-deterministic if not used +# correctly. + +# Define a sequence of SET=>SET=>DEL=>SET=>SINGLEDEL. +define target-file-sizes=(1, 1, 1, 1, 1) +L1 + a.SINGLEDEL.10: +L2 + a.SET.9:v3 +L3 + a.DEL.8: +L4 + a.SET.7:v2 +L5 + a.SET.6:v1 +---- +1: + 000004:[a-a] +2: + 000005:[a-a] +3: + 000006:[a-a] +4: + 000007:[a-a] +5: + 000008:[a-a] + +# No data. +iter +first +---- +. + +# Compact away the DEL. +compact a-b L2 +---- +1: + 000004:[a#10,SINGLEDEL-a#10,SINGLEDEL] +3: + 000009:[a#9,SET-a#9,SET] +4: + 000007:[a#7,SET-a#7,SET] +5: + 000008:[a#6,SET-a#6,SET] + +# No data. +iter +first +---- +. + +# Do two compactions to compact away the SINGLEDEL and 1 SET. +compact a-b L1 +---- +2: + 000010:[a#10,SINGLEDEL-a#10,SINGLEDEL] +3: + 000009:[a#9,SET-a#9,SET] +4: + 000007:[a#7,SET-a#7,SET] +5: + 000008:[a#6,SET-a#6,SET] + +compact a-b L2 +---- +4: + 000007:[a#7,SET-a#7,SET] +5: + 000008:[a#6,SET-a#6,SET] + +# Deleted data reappears. +iter +first +---- +a:v2