db: add test demonstrating current SINGLEDEL behavior

compaction_test.go

@@ -1135,167 +1135,170 @@ func TestManualCompaction(t *testing.T) {
 
 	var ongoingCompaction *compaction
 
-	datadriven.RunTest(t, "testdata/manual_compaction", func(td *datadriven.TestData) string {
-		switch td.Cmd {
-		case "reset":
-			reset()
-			return ""
-
-		case "batch":
-			b := d.NewIndexedBatch()
-			if err := runBatchDefineCmd(td, b); err != nil {
-				return err.Error()
-			}
-			require.NoError(t, b.Commit(nil))
-			return ""
+	paths := []string{"testdata/manual_compaction", "testdata/singledel_manual_compaction"}
+	for _, path := range paths {
+		datadriven.RunTest(t, path, func(td *datadriven.TestData) string {
+			switch td.Cmd {
+			case "reset":
+				reset()
+				return ""
 
-		case "build":
-			if err := runBuildCmd(td, d, mem); err != nil {
-				return err.Error()
-			}
-			return ""
+			case "batch":
+				b := d.NewIndexedBatch()
+				if err := runBatchDefineCmd(td, b); err != nil {
+					return err.Error()
+				}
+				require.NoError(t, b.Commit(nil))
+				return ""
 
-		case "compact":
-			if err := runCompactCmd(td, d); err != nil {
-				return err.Error()
-			}
-			return runLSMCmd(td, d)
+			case "build":
+				if err := runBuildCmd(td, d, mem); err != nil {
+					return err.Error()
+				}
+				return ""
 
-		case "define":
-			if d != nil {
-				if err := d.Close(); err != nil {
+			case "compact":
+				if err := runCompactCmd(td, d); err != nil {
 					return err.Error()
 				}
-			}
+				return runLSMCmd(td, d)
 
-			mem = vfs.NewMem()
-			opts := &Options{
-				FS:         mem,
-				DebugCheck: DebugCheckLevels,
-			}
-			opts.private.disableAutomaticCompactions = true
+			case "define":
+				if d != nil {
+					if err := d.Close(); err != nil {
+						return err.Error()
+					}
+				}
 
-			var err error
-			if d, err = runDBDefineCmd(td, opts); err != nil {
-				return err.Error()
-			}
+				mem = vfs.NewMem()
+				opts := &Options{
+					FS:         mem,
+					DebugCheck: DebugCheckLevels,
+				}
+				opts.private.disableAutomaticCompactions = true
 
-			d.mu.Lock()
-			s := d.mu.versions.currentVersion().String()
-			d.mu.Unlock()
-			return s
+				var err error
+				if d, err = runDBDefineCmd(td, opts); err != nil {
+					return err.Error()
+				}
 
-		case "ingest":
-			if err := runIngestCmd(td, d, mem); err != nil {
-				return err.Error()
-			}
-			return runLSMCmd(td, d)
-
-		case "iter":
-			// TODO(peter): runDBDefineCmd doesn't properly update the visible
-			// sequence number. So we have to use a snapshot with a very large
-			// sequence number, otherwise the DB appears empty.
-			snap := Snapshot{
-				db:     d,
-				seqNum: InternalKeySeqNumMax,
-			}
-			iter := snap.NewIter(nil)
-			return runIterCmd(td, iter, true)
+				d.mu.Lock()
+				s := d.mu.versions.currentVersion().String()
+				d.mu.Unlock()
+				return s
 
-		case "async-compact":
-			var s string
-			ch := make(chan error, 1)
-			go func() {
-				if err := runCompactCmd(td, d); err != nil {
-					ch <- err
+			case "ingest":
+				if err := runIngestCmd(td, d, mem); err != nil {
+					return err.Error()
+				}
+				return runLSMCmd(td, d)
+
+			case "iter":
+				// TODO(peter): runDBDefineCmd doesn't properly update the visible
+				// sequence number. So we have to use a snapshot with a very large
+				// sequence number, otherwise the DB appears empty.
+				snap := Snapshot{
+					db:     d,
+					seqNum: InternalKeySeqNumMax,
+				}
+				iter := snap.NewIter(nil)
+				return runIterCmd(td, iter, true)
+
+			case "async-compact":
+				var s string
+				ch := make(chan error, 1)
+				go func() {
+					if err := runCompactCmd(td, d); err != nil {
+						ch <- err
+						close(ch)
+						return
+					}
+					d.mu.Lock()
+					s = d.mu.versions.currentVersion().DebugString(base.DefaultFormatter)
+					d.mu.Unlock()
 					close(ch)
-					return
+				}()
+
+				manualDone := func() bool {
+					select {
+					case <-ch:
+						return true
+					default:
+						return false
+					}
 				}
-				d.mu.Lock()
-				s = d.mu.versions.currentVersion().DebugString(base.DefaultFormatter)
-				d.mu.Unlock()
-				close(ch)
-			}()
 
-			manualDone := func() bool {
-				select {
-				case <-ch:
-					return true
-				default:
-					return false
+				err := try(100*time.Microsecond, 20*time.Second, func() error {
+					if manualDone() {
+						return nil
+					}
+
+					d.mu.Lock()
+					defer d.mu.Unlock()
+					if len(d.mu.compact.manual) == 0 {
+						return errors.New("no manual compaction queued")
+					}
+					manual := d.mu.compact.manual[0]
+					if manual.retries == 0 {
+						return errors.New("manual compaction has not been retried")
+					}
+					return nil
+				})
+				if err != nil {
+					return err.Error()
 				}
-			}
 
-			err := try(100*time.Microsecond, 20*time.Second, func() error {
 				if manualDone() {
-					return nil
+					return "manual compaction did not block for ongoing\n" + s
 				}
 
 				d.mu.Lock()
-				defer d.mu.Unlock()
-				if len(d.mu.compact.manual) == 0 {
-					return errors.New("no manual compaction queued")
-				}
-				manual := d.mu.compact.manual[0]
-				if manual.retries == 0 {
-					return errors.New("manual compaction has not been retried")
+				delete(d.mu.compact.inProgress, ongoingCompaction)
+				d.mu.compact.compactingCount--
+				ongoingCompaction = nil
+				d.maybeScheduleCompaction()
+				d.mu.Unlock()
+				if err := <-ch; err != nil {
+					return err.Error()
 				}
-				return nil
-			})
-			if err != nil {
-				return err.Error()
-			}
-
-			if manualDone() {
-				return "manual compaction did not block for ongoing\n" + s
-			}
+				return "manual compaction blocked until ongoing finished\n" + s
 
-			d.mu.Lock()
-			delete(d.mu.compact.inProgress, ongoingCompaction)
-			d.mu.compact.compactingCount--
-			ongoingCompaction = nil
-			d.maybeScheduleCompaction()
-			d.mu.Unlock()
-			if err := <-ch; err != nil {
-				return err.Error()
-			}
-			return "manual compaction blocked until ongoing finished\n" + s
-
-		case "add-ongoing-compaction":
-			var startLevel int
-			var outputLevel int
-			td.ScanArgs(t, "startLevel", &startLevel)
-			td.ScanArgs(t, "outputLevel", &outputLevel)
-			ongoingCompaction = &compaction{
-				inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}},
-			}
-			ongoingCompaction.startLevel = &ongoingCompaction.inputs[0]
-			ongoingCompaction.outputLevel = &ongoingCompaction.inputs[1]
-			d.mu.Lock()
-			d.mu.compact.inProgress[ongoingCompaction] = struct{}{}
-			d.mu.compact.compactingCount++
-			d.mu.Unlock()
-			return ""
+			case "add-ongoing-compaction":
+				var startLevel int
+				var outputLevel int
+				td.ScanArgs(t, "startLevel", &startLevel)
+				td.ScanArgs(t, "outputLevel", &outputLevel)
+				ongoingCompaction = &compaction{
+					inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}},
+				}
+				ongoingCompaction.startLevel = &ongoingCompaction.inputs[0]
+				ongoingCompaction.outputLevel = &ongoingCompaction.inputs[1]
+				d.mu.Lock()
+				d.mu.compact.inProgress[ongoingCompaction] = struct{}{}
+				d.mu.compact.compactingCount++
+				d.mu.Unlock()
+				return ""
 
-		case "remove-ongoing-compaction":
-			d.mu.Lock()
-			delete(d.mu.compact.inProgress, ongoingCompaction)
-			d.mu.compact.compactingCount--
-			ongoingCompaction = nil
-			d.mu.Unlock()
-			return ""
+			case "remove-ongoing-compaction":
+				d.mu.Lock()
+				delete(d.mu.compact.inProgress, ongoingCompaction)
+				d.mu.compact.compactingCount--
+				ongoingCompaction = nil
+				d.mu.Unlock()
+				return ""
 
-		case "set-concurrent-compactions":
-			td.ScanArgs(t, "num", &d.opts.MaxConcurrentCompactions)
-			return ""
+			case "set-concurrent-compactions":
+				td.ScanArgs(t, "num", &d.opts.MaxConcurrentCompactions)
+				return ""
 
-		case "wait-pending-table-stats":
-			return runTableStatsCmd(td, d)
+			case "wait-pending-table-stats":
+				return runTableStatsCmd(td, d)
 
-		default:
-			return fmt.Sprintf("unknown command: %s", td.Cmd)
-		}
-	})
+			default:
+				return fmt.Sprintf("unknown command: %s", td.Cmd)
+			}
+		})
+	}
 }
 
 func TestCompactionFindGrandparentLimit(t *testing.T) {

testdata/singledel_manual_compaction

@@ -0,0 +1,77 @@
+# This is not actually a manual compaction test, and simply uses manual
+# compaction to demonstrate single delete semantics. Specifically, it
+# demonstrates that the behavior can be non-deterministic if not used
+# correctly.
+
+# Define a sequence of SET=>SET=>DEL=>SET=>SINGLEDEL.
+define target-file-sizes=(1, 1, 1, 1, 1)
+L1
+  a.SINGLEDEL.10:
+L2
+  a.SET.9:v3
+L3
+  a.DEL.8:
+L4
+  a.SET.7:v2
+L5
+  a.SET.6:v1
+----
+1:
+  000004:[a-a]
+2:
+  000005:[a-a]
+3:
+  000006:[a-a]
+4:
+  000007:[a-a]
+5:
+  000008:[a-a]
+
+# No data.
+iter
+first
+----
+.
+
+# Compact away the DEL.
+compact a-b L2
+----
+1:
+  000004:[a#10,SINGLEDEL-a#10,SINGLEDEL]
+3:
+  000009:[a#9,SET-a#9,SET]
+4:
+  000007:[a#7,SET-a#7,SET]
+5:
+  000008:[a#6,SET-a#6,SET]
+
+# No data.
+iter
+first
+----
+.
+
+# Do two compactions to compact away the SINGLEDEL and 1 SET.
+compact a-b L1
+----
+2:
+  000010:[a#10,SINGLEDEL-a#10,SINGLEDEL]
+3:
+  000009:[a#9,SET-a#9,SET]
+4:
+  000007:[a#7,SET-a#7,SET]
+5:
+  000008:[a#6,SET-a#6,SET]
+
+compact a-b L2
+----
+4:
+  000007:[a#7,SET-a#7,SET]
+5:
+  000008:[a#6,SET-a#6,SET]
+
+# Deleted data reappears.
+iter
+first
+----
+a:v2