go.mod: bump Pebble to e9a8c4ad65c5

e9a8c4ad sstable: fix error path in writer.Close
fdf055dd db: improve documentation of tableNewIters
717cbce0 db: add Options.LoggerAndTracer for tracing
7751e381 objstorage: better Writable API
78c8001e db: add missing error check in TestIngestLoad
1334233b metamorphic: add unit test for Options propagation
ed824c76 objstorage: rename ReadaheadHandle to ReadHandle

The "better Writable API" changed the interface to write to SST files.
This commit contains the required changes in Cockroach: MemFile is now
MemObject and implements objstorage.Writable. There were many test
places that were using MemFile as just an io.Writer; these are updated
to just use bytes.Buffer.

Informs cockroachdb#97350.

Release note: none
Epic: none

DEPS.bzl

@@ -1555,10 +1555,10 @@ def go_deps():
         patches = [
             "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
         ],
-        sha256 = "af71f92b799b6618a951843c7c5589cef2e095549a9f6af03153b569219219c8",
-        strip_prefix = "github.com/cockroachdb/[email protected]20230301151825-0cd1ceea645a",
+        sha256 = "8ea9b6931898da952aae5e5c84f551c6e6d2fadd72fead740d2794672a345e04",
+        strip_prefix = "github.com/cockroachdb/[email protected]20230306180248-e9a8c4ad65c5",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20230301151825-0cd1ceea645a.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20230306180248-e9a8c4ad65c5.zip",
         ],
     )
     go_repository(

build/bazelutil/distdir_files.bzl

@@ -202,7 +202,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/google-api-go-client/com_github_cockroachdb_google_api_go_client-v0.80.1-0.20221117193156-6a9f7150cb93.zip": "b3378c579f4f4340403038305907d672c86f615f8233118a8873ebe4229c4f39",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20230118201751-21c54148d20b.zip": "ca7776f47e5fecb4c495490a679036bfc29d95bd7625290cfdb9abb0baf97476",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20230301151825-0cd1ceea645a.zip": "af71f92b799b6618a951843c7c5589cef2e095549a9f6af03153b569219219c8",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20230306180248-e9a8c4ad65c5.zip": "8ea9b6931898da952aae5e5c84f551c6e6d2fadd72fead740d2794672a345e04",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.3.zip": "7778b1e4485e4f17f35e5e592d87eb99c29e173ac9507801d000ad76dd0c261e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/sentry-go/com_github_cockroachdb_sentry_go-v0.6.1-cockroachdb.2.zip": "fbb2207d02aecfdd411b1357efe1192dbb827959e36b7cab7491731ac55935c9",

go.mod

@@ -114,7 +114,7 @@ require (
 	github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
 	github.com/cockroachdb/gostdlib v1.19.0
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b
-	github.com/cockroachdb/pebble v0.0.0-20230301151825-0cd1ceea645a
+	github.com/cockroachdb/pebble v0.0.0-20230306180248-e9a8c4ad65c5
 	github.com/cockroachdb/redact v1.1.3
 	github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
 	github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b

go.sum

@@ -491,8 +491,8 @@ github.com/cockroachdb/gostdlib v1.19.0/go.mod h1:+dqqpARXbE/gRDEhCak6dm0l14AaTy
 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20230301151825-0cd1ceea645a h1:RUozu4Qb0PCexALTQJ7zr5MbSPEBOcoszQHlOqMOwtU=
-github.com/cockroachdb/pebble v0.0.0-20230301151825-0cd1ceea645a/go.mod h1:9lRMC4XN3/BLPtIp6kAKwIaHu369NOf2rMucPzipz50=
+github.com/cockroachdb/pebble v0.0.0-20230306180248-e9a8c4ad65c5 h1:rGj0g8lJ4RKw2sPqWQCcDbivT4XxXzTsEL96nSOtiBg=
+github.com/cockroachdb/pebble v0.0.0-20230306180248-e9a8c4ad65c5/go.mod h1:9lRMC4XN3/BLPtIp6kAKwIaHu369NOf2rMucPzipz50=
 github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd h1:KFOt5I9nEKZgCnOSmy8r4Oykh8BYQO8bFOTgHDS8YZA=

pkg/ccl/backupccl/restore_data_processor_test.go

@@ -9,6 +9,7 @@
 package backupccl
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	math "math"
@@ -190,8 +191,8 @@ func runTestIngest(t *testing.T, init func(*cluster.Settings)) {
 	writeSST := func(t *testing.T, offsets []int) string {
 		path := strconv.FormatInt(timeutil.Now().UnixNano(), 10)
 
-		sstFile := &storage.MemFile{}
-		sst := storage.MakeBackupSSTWriter(ctx, cs, sstFile)
+		var sstFile bytes.Buffer
+		sst := storage.MakeBackupSSTWriter(ctx, cs, &sstFile)
 		defer sst.Close()
 		ts := hlc.NewClockForTesting(nil).Now()
 		value := roachpb.MakeValueFromString("bar")
@@ -206,7 +207,7 @@ func runTestIngest(t *testing.T, init func(*cluster.Settings)) {
 		if err := sst.Finish(); err != nil {
 			t.Fatalf("%+v", err)
 		}
-		if err := os.WriteFile(filepath.Join(dir, "foo", path), sstFile.Data(), 0644); err != nil {
+		if err := os.WriteFile(filepath.Join(dir, "foo", path), sstFile.Bytes(), 0644); err != nil {
 			t.Fatalf("%+v", err)
 		}
 		return path

pkg/ccl/streamingccl/streamingest/stream_ingestion_processor.go

@@ -90,7 +90,7 @@ type rangeKeyBatcher struct {
 	// adding MVCCRangeKeyValue
 	rangeKeySSTWriterMaker func() *storage.SSTWriter
 	// In-memory SST file for flushing MVCC range keys
-	rangeKeySSTFile *storage.MemFile
+	rangeKeySSTFile *storage.MemObject
 	// curRangeKVBatch is the current batch of range KVs which will
 	// be ingested through 'flush' later.
 	curRangeKVBatch mvccRangeKeyValues
@@ -113,7 +113,7 @@ func newRangeKeyBatcher(
 		db:              db,
 		minTimestamp:    hlc.MaxTimestamp,
 		batchSummary:    kvpb.BulkOpSummary{},
-		rangeKeySSTFile: &storage.MemFile{},
+		rangeKeySSTFile: &storage.MemObject{},
 		onFlush:         onFlush,
 	}
 	batcher.rangeKeySSTWriterMaker = func() *storage.SSTWriter {

pkg/kv/bulk/sst_batcher.go

@@ -168,7 +168,7 @@ type SSTBatcher struct {
 	// The rest of the fields are per-batch and are reset via Reset() before each
 	// batch is started.
 	sstWriter         storage.SSTWriter
-	sstFile           *storage.MemFile
+	sstFile           *storage.MemObject
 	batchStartKey     []byte
 	batchEndKey       []byte
 	batchEndValue     []byte
@@ -365,7 +365,7 @@ func (b *SSTBatcher) AddMVCCKey(ctx context.Context, key storage.MVCCKey, value
 // Reset clears all state in the batcher and prepares it for reuse.
 func (b *SSTBatcher) Reset(ctx context.Context) error {
 	b.sstWriter.Close()
-	b.sstFile = &storage.MemFile{}
+	b.sstFile = &storage.MemObject{}
 	// Create sstables intended for ingestion using the newest format that all
 	// nodes can support. MakeIngestionSSTWriter will handle cluster version
 	// gating using b.settings.
@@ -912,7 +912,7 @@ func createSplitSSTable(
 	iter storage.SimpleMVCCIterator,
 	settings *cluster.Settings,
 ) (*sstSpan, *sstSpan, error) {
-	sstFile := &storage.MemFile{}
+	sstFile := &storage.MemObject{}
 	w := storage.MakeIngestionSSTWriter(ctx, settings, sstFile)
 	defer w.Close()
 
@@ -937,7 +937,7 @@ func createSplitSSTable(
 			}
 
 			left = &sstSpan{start: first, end: last.Next(), sstBytes: sstFile.Data()}
-			*sstFile = storage.MemFile{}
+			*sstFile = storage.MemObject{}
 			w = storage.MakeIngestionSSTWriter(ctx, settings, sstFile)
 			split = true
 			first = nil

pkg/kv/kvnemesis/applier_test.go

@@ -53,7 +53,7 @@ func TestApplier(t *testing.T) {
 	sstValueHeader.KVNemesisSeq.Set(1)
 	sstSpan := roachpb.Span{Key: roachpb.Key(k1), EndKey: roachpb.Key(k4)}
 	sstTS := hlc.Timestamp{WallTime: 1}
-	sstFile := &storage.MemFile{}
+	sstFile := &storage.MemObject{}
 	{
 		st := cluster.MakeTestingClusterSettings()
 		storage.ValueBlocksEnabled.Override(ctx, &st.SV, true)

pkg/kv/kvnemesis/generator.go

@@ -573,7 +573,7 @@ func randAddSSTable(g *generator, rng *rand.Rand) Operation {
 	sstTombstone := storage.MVCCValue{MVCCValueHeader: sstValueHeader}
 
 	// Write key/value pairs to the SST.
-	f := &storage.MemFile{}
+	f := &storage.MemObject{}
 	st := cluster.MakeTestingClusterSettings()
 	w := storage.MakeIngestionSSTWriter(ctx, st, f)
 	defer w.Close()

pkg/kv/kvnemesis/operations_test.go

@@ -51,7 +51,7 @@ func TestOperationsFormat(t *testing.T) {
 	sstValueHeader.KVNemesisSeq.Set(1)
 	sstSpan := roachpb.Span{Key: roachpb.Key(k1), EndKey: roachpb.Key(k4)}
 	sstTS := hlc.Timestamp{WallTime: 1}
-	sstFile := &storage.MemFile{}
+	sstFile := &storage.MemObject{}
 	{
 		st := cluster.MakeTestingClusterSettings()
 		storage.ValueBlocksEnabled.Override(ctx, &st.SV, false)

pkg/kv/kvnemesis/validator_test.go

@@ -197,7 +197,7 @@ func TestValidate(t *testing.T) {
 	}
 
 	makeAddSSTable := func(seq kvnemesisutil.Seq, kvs []sstKV) Operation {
-		f := &storage.MemFile{}
+		f := &storage.MemObject{}
 		st := cluster.MakeTestingClusterSettings()
 		storage.ValueBlocksEnabled.Override(ctx, &st.SV, true)
 		w := storage.MakeIngestionSSTWriter(ctx, st, f)

pkg/kv/kvserver/BUILD.bazel

@@ -208,6 +208,7 @@ go_library(
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_cockroachdb_logtags//:logtags",
         "@com_github_cockroachdb_pebble//:pebble",
+        "@com_github_cockroachdb_pebble//objstorage",
         "@com_github_cockroachdb_redact//:redact",
         "@com_github_gogo_protobuf//proto",
         "@com_github_google_btree//:btree",

pkg/kv/kvserver/batcheval/cmd_add_sstable_test.go

@@ -11,6 +11,7 @@
 package batcheval_test
 
 import (
+	"bytes"
 	"context"
 	"os"
 	"regexp"
@@ -1421,14 +1422,14 @@ func runTestDBAddSSTable(
 		value := roachpb.MakeValueFromString("1")
 		value.InitChecksum([]byte("foo"))
 
-		sstFile := &storage.MemFile{}
-		w := storage.MakeBackupSSTWriter(ctx, cs, sstFile)
+		var sstFile bytes.Buffer
+		w := storage.MakeBackupSSTWriter(ctx, cs, &sstFile)
 		defer w.Close()
 		require.NoError(t, w.Put(key, value.RawBytes))
 		require.NoError(t, w.Finish())
 
 		_, _, err := db.AddSSTable(
-			ctx, "b", "c", sstFile.Data(), allowConflicts, allowShadowing, allowShadowingBelow, nilStats, ingestAsSST, noTS)
+			ctx, "b", "c", sstFile.Bytes(), allowConflicts, allowShadowing, allowShadowingBelow, nilStats, ingestAsSST, noTS)
 		require.Error(t, err)
 		require.Contains(t, err.Error(), "invalid checksum")
 	}

pkg/kv/kvserver/batcheval/cmd_export.go

@@ -11,6 +11,7 @@
 package batcheval
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"time"
@@ -172,7 +173,7 @@ func evalExport(
 
 	var curSizeOfExportedSSTs int64
 	for start := args.Key; start != nil; {
-		destFile := &storage.MemFile{}
+		var destFile bytes.Buffer
 		opts := storage.MVCCExportOptions{
 			StartKey:           storage.MVCCKey{Key: start, Timestamp: resumeKeyTS},
 			EndKey:             args.EndKey,
@@ -198,7 +199,7 @@ func evalExport(
 			}
 			var hasRangeKeys bool
 			summary, resumeInfo, fingerprint, hasRangeKeys, err = storage.MVCCExportFingerprint(ctx,
-				cArgs.EvalCtx.ClusterSettings(), reader, opts, destFile)
+				cArgs.EvalCtx.ClusterSettings(), reader, opts, &destFile)
 			if err != nil {
 				return result.Result{}, maybeAnnotateExceedMaxSizeError(err)
 			}
@@ -208,16 +209,16 @@ func evalExport(
 			// part of the ExportResponse. This frees up the memory used by the empty
 			// SST file.
 			if !hasRangeKeys {
-				destFile = &storage.MemFile{}
+				destFile = bytes.Buffer{}
 			}
 		} else {
 			summary, resumeInfo, err = storage.MVCCExportToSST(ctx, cArgs.EvalCtx.ClusterSettings(), reader,
-				opts, destFile)
+				opts, &destFile)
 			if err != nil {
 				return result.Result{}, maybeAnnotateExceedMaxSizeError(err)
 			}
 		}
-		data := destFile.Data()
+		data := destFile.Bytes()
 
 		// NB: This should only happen in two cases:
 		//

pkg/kv/kvserver/batcheval/cmd_export_test.go

@@ -442,7 +442,7 @@ func exportUsingGoIterator(
 	startKey, endKey roachpb.Key,
 	reader storage.Reader,
 ) ([]byte, error) {
-	memFile := &storage.MemFile{}
+	memFile := &storage.MemObject{}
 	sst := storage.MakeIngestionSSTWriter(
 		ctx, cluster.MakeTestingClusterSettings(), memFile,
 	)
@@ -593,7 +593,7 @@ func assertEqualKVs(
 			var sst []byte
 			maxSize := uint64(0)
 			prevStart := start
-			sstFile := &storage.MemFile{}
+			var sstFile bytes.Buffer
 			summary, resumeInfo, err := storage.MVCCExportToSST(ctx, st, e, storage.MVCCExportOptions{
 				StartKey:           start,
 				EndKey:             endKey,
@@ -603,10 +603,10 @@ func assertEqualKVs(
 				TargetSize:         targetSize,
 				MaxSize:            maxSize,
 				StopMidKey:         bool(stopMidKey),
-			}, sstFile)
+			}, &sstFile)
 			require.NoError(t, err)
 			start = resumeInfo.ResumeKey
-			sst = sstFile.Data()
+			sst = sstFile.Bytes()
 			loaded := loadSST(t, sst, startKey, endKey)
 			// Ensure that the pagination worked properly.
 			if start.Key != nil {
@@ -652,7 +652,7 @@ func assertEqualKVs(
 					TargetSize:         targetSize,
 					MaxSize:            maxSize,
 					StopMidKey:         false,
-				}, &storage.MemFile{})
+				}, &bytes.Buffer{})
 				require.Regexp(t, fmt.Sprintf("export size \\(%d bytes\\) exceeds max size \\(%d bytes\\)",
 					dataSizeWhenExceeded, maxSize), err)
 			}

pkg/kv/kvserver/client_merge_test.go

@@ -3782,13 +3782,13 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) {
 
 		type sstFileWriter struct {
 			span   roachpb.Span
-			file   *storage.MemFile
+			file   *storage.MemObject
 			writer storage.SSTWriter
 		}
 		keySpans := rditer.MakeReplicatedKeySpans(inSnap.Desc)
 		sstFileWriters := map[string]sstFileWriter{}
 		for _, span := range keySpans {
-			file := &storage.MemFile{}
+			file := &storage.MemObject{}
 			writer := storage.MakeIngestionSSTWriter(ctx, st, file)
 			if err := writer.ClearRawRange(span.Key, span.EndKey, true, true); err != nil {
 				return err
@@ -3850,7 +3850,7 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) {
 		// replicas (while absorbing their user keys into the LHS).
 		for _, k := range []roachpb.Key{keyB, keyC} {
 			rangeID := rangeIds[string(k)]
-			sstFile := &storage.MemFile{}
+			sstFile := &storage.MemObject{}
 			sst := storage.MakeIngestionSSTWriter(ctx, st, sstFile)
 			defer sst.Close()
 			{
@@ -3892,7 +3892,7 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) {
 		}
 
 		// Construct an SST for the user key range of the subsumed replicas.
-		sstFile := &storage.MemFile{}
+		sstFile := &storage.MemObject{}
 		sst := storage.MakeIngestionSSTWriter(ctx, st, sstFile)
 		defer sst.Close()
 		desc := roachpb.RangeDescriptor{

pkg/kv/kvserver/client_replica_circuit_breaker_test.go

@@ -534,7 +534,7 @@ func TestReplicaCircuitBreaker_ExemptRequests(t *testing.T) {
 	exemptRequests := []func() kvpb.Request{
 		func() kvpb.Request { return &kvpb.ExportRequest{} },
 		func() kvpb.Request {
-			sstFile := &storage.MemFile{}
+			sstFile := &storage.MemObject{}
 			sst := storage.MakeIngestionSSTWriter(context.Background(), cluster.MakeTestingClusterSettings(), sstFile)
 			defer sst.Close()
 			require.NoError(t, sst.LogData([]byte("hello")))

pkg/kv/kvserver/helpers_test.go

@@ -392,7 +392,7 @@ func (r *Replica) LoadBasedSplitter() *split.Decider {
 func MakeSSTable(
 	ctx context.Context, key, value string, ts hlc.Timestamp,
 ) ([]byte, storage.MVCCKeyValue) {
-	sstFile := &storage.MemFile{}
+	sstFile := &storage.MemObject{}
 	sst := storage.MakeIngestionSSTWriter(ctx, cluster.MakeTestingClusterSettings(), sstFile)
 	defer sst.Close()
 

pkg/kv/kvserver/replica_raftstorage.go

@@ -497,7 +497,7 @@ func (r *Replica) applySnapshot(
 	}
 	if nonempty {
 		// TODO(itsbilal): Write to SST directly in unreplicatedSST rather than
-		// buffering in a MemFile first.
+		// buffering in a MemObject first.
 		if err := inSnap.SSTStorageScratch.WriteSST(ctx, unreplicatedSSTFile.Data()); err != nil {
 			return err
 		}
@@ -715,8 +715,8 @@ func writeUnreplicatedSST(
 	meta raftpb.SnapshotMetadata,
 	hs raftpb.HardState,
 	sl *logstore.StateLoader,
-) (_ *storage.MemFile, nonempty bool, _ error) {
-	unreplicatedSSTFile := &storage.MemFile{}
+) (_ *storage.MemObject, nonempty bool, _ error) {
+	unreplicatedSSTFile := &storage.MemObject{}
 	unreplicatedSST := storage.MakeIngestionSSTWriter(
 		ctx, st, unreplicatedSSTFile,
 	)
@@ -789,7 +789,7 @@ func clearSubsumedReplicaDiskData(
 	totalKeySpans := append([]roachpb.Span(nil), keySpans...)
 	for _, subDesc := range subsumedDescs {
 		// We have to create an SST for the subsumed replica's range-id local keys.
-		subsumedReplSSTFile := &storage.MemFile{}
+		subsumedReplSSTFile := &storage.MemObject{}
 		subsumedReplSST := storage.MakeIngestionSSTWriter(
 			ctx, st, subsumedReplSSTFile,
 		)
@@ -811,7 +811,7 @@ func clearSubsumedReplicaDiskData(
 		}
 		if subsumedReplSST.DataSize > 0 {
 			// TODO(itsbilal): Write to SST directly in subsumedReplSST rather than
-			// buffering in a MemFile first.
+			// buffering in a MemObject first.
 			if err := writeSST(ctx, subsumedReplSSTFile.Data()); err != nil {
 				return err
 			}
@@ -845,7 +845,7 @@ func clearSubsumedReplicaDiskData(
 	// subsume both r1 and r2 in S1.
 	for i := range keySpans {
 		if totalKeySpans[i].EndKey.Compare(keySpans[i].EndKey) > 0 {
-			subsumedReplSSTFile := &storage.MemFile{}
+			subsumedReplSSTFile := &storage.MemObject{}
 			subsumedReplSST := storage.MakeIngestionSSTWriter(
 				ctx, st, subsumedReplSSTFile,
 			)
@@ -866,7 +866,7 @@ func clearSubsumedReplicaDiskData(
 			}
 			if subsumedReplSST.DataSize > 0 {
 				// TODO(itsbilal): Write to SST directly in subsumedReplSST rather than
-				// buffering in a MemFile first.
+				// buffering in a MemObject first.
 				if err := writeSST(ctx, subsumedReplSSTFile.Data()); err != nil {
 					return err
 				}