metamorphic: special handling of assertion errors

Check for assertion errors when recording history and display their
stacktrace and panic. This helps with debugging.

We also fix some cases where we return an assertion error when we are
checking an API input; assertion errors should indicate a bug inside
Pebble.

ingest.go

@@ -493,22 +493,22 @@ func ingestSortAndVerify(cmp Compare, lr ingestLoadResult, exciseSpan KeyRange)
 	// fit within the exciseSpan.
 	for _, f := range lr.shared {
 		if !exciseSpan.Contains(cmp, f.Smallest) || !exciseSpan.Contains(cmp, f.Largest) {
-			return errors.AssertionFailedf("pebble: shared file outside of excise span, span [%s-%s), file = %s", exciseSpan.Start, exciseSpan.End, f.String())
+			return errors.Newf("pebble: shared file outside of excise span, span [%s-%s), file = %s", exciseSpan.Start, exciseSpan.End, f.String())
 		}
 	}
 	if len(lr.external) > 0 {
 		if len(lr.local) > 0 || len(lr.shared) > 0 {
 			// Currently we only support external ingests on their own. If external
 			// files are present alongside local/shared files, return an error.
-			return errors.AssertionFailedf("pebble: external files cannot be ingested atomically alongside other types of files")
+			return errors.Newf("pebble: external files cannot be ingested atomically alongside other types of files")
 		}
 		// Sort according to the smallest key.
 		slices.SortFunc(lr.external, func(a, b ingestExternalMeta) int {
 			return cmp(a.Smallest.UserKey, b.Smallest.UserKey)
 		})
 		for i := 1; i < len(lr.external); i++ {
 			if sstableKeyCompare(cmp, lr.external[i-1].Largest, lr.external[i].Smallest) >= 0 {
-				return errors.AssertionFailedf("pebble: external sstables have overlapping ranges")
+				return errors.Newf("pebble: external sstables have overlapping ranges")
 			}
 		}
 		return nil
@@ -524,7 +524,7 @@ func ingestSortAndVerify(cmp Compare, lr ingestLoadResult, exciseSpan KeyRange)
 
 	for i := 1; i < len(lr.local); i++ {
 		if sstableKeyCompare(cmp, lr.local[i-1].Largest, lr.local[i].Smallest) >= 0 {
-			return errors.AssertionFailedf("pebble: local ingestion sstables have overlapping ranges")
+			return errors.Newf("pebble: local ingestion sstables have overlapping ranges")
 		}
 	}
 	if len(lr.shared) == 0 {

metamorphic/history.go

@@ -148,6 +148,13 @@ type historyRecorder struct {
 
 // Recordf records the results of a single operation.
 func (h historyRecorder) Recordf(format string, args ...interface{}) {
+	// Check for assertion errors.
+	for _, a := range args {
+		if err, ok := a.(error); ok && errors.IsAssertionFailure(err) {
+			fmt.Fprintf(os.Stderr, "%+v", err)
+			panic(err)
+		}
+	}
 	h.history.Recordf(h.op, format, args...)
 	// If the history recorder was configured with an additional record func,
 	// invoke it. This can be used to collect the per-operation output when

sstable/prefix_replacing_iterator.go

@@ -25,8 +25,13 @@ type prefixReplacingIterator struct {
 	err       error
 }
 
-var errInputPrefixMismatch = errors.New("key argument does not have prefix required for replacement")
-var errOutputPrefixMismatch = errors.New("key returned does not have prefix required for replacement")
+func errInputPrefixMismatch() error {
+	return errors.AssertionFailedf("key argument does not have prefix required for replacement")
+}
+
+func errOutputPrefixMismatch() error {
+	return errors.AssertionFailedf("key returned does not have prefix required for replacement")
+}
 
 var _ Iterator = (*prefixReplacingIterator)(nil)
 
@@ -55,7 +60,7 @@ func (p *prefixReplacingIterator) SetContext(ctx context.Context) {
 
 func (p *prefixReplacingIterator) rewriteArg(key []byte) []byte {
 	if !bytes.HasPrefix(key, p.dst) {
-		p.err = errInputPrefixMismatch
+		p.err = errInputPrefixMismatch()
 		return key
 	}
 	p.arg = append(p.arg[:len(p.src)], key[len(p.dst):]...)
@@ -64,7 +69,7 @@ func (p *prefixReplacingIterator) rewriteArg(key []byte) []byte {
 
 func (p *prefixReplacingIterator) rewriteArg2(key []byte) []byte {
 	if !bytes.HasPrefix(key, p.dst) {
-		p.err = errInputPrefixMismatch
+		p.err = errInputPrefixMismatch()
 		return key
 	}
 	p.arg2 = append(p.arg2[:len(p.src)], key[len(p.dst):]...)
@@ -78,7 +83,7 @@ func (p *prefixReplacingIterator) rewriteResult(
 		return k, v
 	}
 	if !bytes.HasPrefix(k.UserKey, p.src) {
-		p.err = errOutputPrefixMismatch
+		p.err = errOutputPrefixMismatch()
 		if invariants.Enabled {
 			panic(p.err)
 		}
@@ -209,7 +214,7 @@ func newPrefixReplacingFragmentIterator(
 
 func (p *prefixReplacingFragmentIterator) rewriteArg(key []byte) ([]byte, error) {
 	if !bytes.HasPrefix(key, p.dst) {
-		return nil, errInputPrefixMismatch
+		return nil, errInputPrefixMismatch()
 	}
 	p.arg = append(p.arg[:len(p.src)], key[len(p.dst):]...)
 	return p.arg, nil
@@ -219,7 +224,7 @@ func (p *prefixReplacingFragmentIterator) rewriteSpan(
 	sp *keyspan.Span, err error,
 ) (*keyspan.Span, error) {
 	if !bytes.HasPrefix(sp.Start, p.src) || !bytes.HasPrefix(sp.End, p.src) {
-		return nil, errInputPrefixMismatch
+		return nil, errInputPrefixMismatch()
 	}
 	sp.Start = append(p.out1[:len(p.dst)], sp.Start[len(p.src):]...)
 	sp.End = append(p.out2[:len(p.dst)], sp.End[len(p.src):]...)

sstable/reader_virtual.go

@@ -266,7 +266,7 @@ func (v *VirtualReader) EstimateDiskUsage(start, end []byte) (uint64, error) {
 	_, f, l := v.vState.constrainBounds(start, end, true /* endInclusive */)
 	if v.vState.prefixChange != nil {
 		if !bytes.HasPrefix(f, v.vState.prefixChange.SyntheticPrefix) || !bytes.HasPrefix(l, v.vState.prefixChange.SyntheticPrefix) {
-			return 0, errInputPrefixMismatch
+			return 0, errInputPrefixMismatch()
 		}
 		// TODO(dt): we could add a scratch buf to VirtualReader to avoid allocs on
 		// repeated calls to this.