vendor: bump Pebble to 406c1dce54c9

``` 406c1dce *: add simpleLevelIterator, reduce merging levels in external iter cd7f076e metamorphic: reposition iterator after SetOptions 5f6b4325 db: make RangeKeyMasking.Filter a constructor function ``` In addition to the vendor bump, this commit makes some small code changes in response to interface changes within Pebbe. I kept these changes minimal for the Pebble bump PR, and we can follow up with fully adapting call sites to take advantage of the new ability to pass 'levels' of sstables into the sstable iterator. Release note: None Release justification: Non-production code changes, low-risk updates to new functionality.
cockroachdb · Aug 19, 2022 · 05a9ebb · 05a9ebb
1 parent c4d4a0b
commit 05a9ebb
Show file tree

Hide file tree

Showing 9 changed files with 47 additions and 22 deletions.
diff --git a/DEPS.bzl b/DEPS.bzl
@@ -1421,10 +1421,10 @@ def go_deps():
         patches = [
             "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
         ],
-        sha256 = "6d348c25c032bcaafe9de96e2501cce07568adadf76381f2d976f3129a4ecaca",
-        strip_prefix = "github.com/cockroachdb/[email protected]20220817213320-358f750befa5",
+        sha256 = "67b1fe4b95ca0e931a091895127f9f6579fab8db84bacd6952a2db2b6dc0cabd",
+        strip_prefix = "github.com/cockroachdb/[email protected]20220819162537-406c1dce54c9",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220817213320-358f750befa5.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220819162537-406c1dce54c9.zip",
         ],
     )
     go_repository(

diff --git a/build/bazelutil/distdir_files.bzl b/build/bazelutil/distdir_files.bzl
@@ -186,7 +186,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/go-test-teamcity/com_github_cockroachdb_go_test_teamcity-v0.0.0-20191211140407-cff980ad0a55.zip": "bac30148e525b79d004da84d16453ddd2d5cd20528e9187f1d7dac708335674b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.13.0.zip": "b3d43d8f95edf65f73a5348f29e1159823cac64b148f8d3bb48340bf55d70872",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20211118104740-dabe8e521a4f.zip": "1972c3f171f118add3fd9e64bcea6cbb9959a3b7fa0ada308e8a7310813fea74",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220817213320-358f750befa5.zip": "6d348c25c032bcaafe9de96e2501cce07568adadf76381f2d976f3129a4ecaca",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220819162537-406c1dce54c9.zip": "67b1fe4b95ca0e931a091895127f9f6579fab8db84bacd6952a2db2b6dc0cabd",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.3.zip": "7778b1e4485e4f17f35e5e592d87eb99c29e173ac9507801d000ad76dd0c261e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/sentry-go/com_github_cockroachdb_sentry_go-v0.6.1-cockroachdb.2.zip": "fbb2207d02aecfdd411b1357efe1192dbb827959e36b7cab7491731ac55935c9",

diff --git a/go.mod b/go.mod
@@ -47,7 +47,7 @@ require (
 	github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
 	github.com/cockroachdb/gostdlib v1.13.0
 	github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f
-	github.com/cockroachdb/pebble v0.0.0-20220817213320-358f750befa5
+	github.com/cockroachdb/pebble v0.0.0-20220819162537-406c1dce54c9
 	github.com/cockroachdb/redact v1.1.3
 	github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
 	github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b

diff --git a/go.sum b/go.sum
@@ -470,8 +470,8 @@ github.com/cockroachdb/gostdlib v1.13.0/go.mod h1:eXX95p9QDrYwJfJ6AgeN9QnRa/lqqi
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74=
 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20220817213320-358f750befa5 h1:KwPwcpvNxdK9uX9fcaH5FQvGfwKVfVmfAS4QNgkGot0=
-github.com/cockroachdb/pebble v0.0.0-20220817213320-358f750befa5/go.mod h1:890yq1fUb9b6dGNwssgeUO5vQV9qfXnCPxAJhBQfXw0=
+github.com/cockroachdb/pebble v0.0.0-20220819162537-406c1dce54c9 h1:JoFP5bNi5eXRyBFQCb+k8Cvmxw8uRoCfNBwOplyY2gM=
+github.com/cockroachdb/pebble v0.0.0-20220819162537-406c1dce54c9/go.mod h1:890yq1fUb9b6dGNwssgeUO5vQV9qfXnCPxAJhBQfXw0=
 github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=

diff --git a/pkg/ccl/backupccl/restore_data_processor_test.go b/pkg/ccl/backupccl/restore_data_processor_test.go
@@ -73,7 +73,7 @@ func slurpSSTablesLatestKey(
 			LowerBound: keys.LocalMax,
 			UpperBound: keys.MaxKey,
 		}
-		sst, err := storage.NewPebbleSSTIterator([]sstable.ReadableFile{file}, iterOpts)
+		sst, err := storage.NewPebbleSSTIterator([][]sstable.ReadableFile{{file}}, iterOpts, false /* forwardOnly */)
 		if err != nil {
 			t.Fatal(err)
 		}

diff --git a/pkg/ccl/storageccl/external_sst_reader.go b/pkg/ccl/storageccl/external_sst_reader.go
@@ -115,12 +115,15 @@ func ExternalSSTReader(
 	encryption *roachpb.FileEncryptionOptions,
 	iterOpts storage.IterOptions,
 ) (storage.SimpleMVCCIterator, error) {
+	// TODO(jackson): Change the interface to accept a two-dimensional
+	// [][]StoreFiles slice, and propagate that structure to
+	// NewPebbleSSTIterator.
 
 	if !remoteSSTs.Get(&storeFiles[0].Store.Settings().SV) {
 		return newMemPebbleSSTReader(ctx, storeFiles, encryption, iterOpts)
 	}
 	remoteCacheSize := remoteSSTSuffixCacheSize.Get(&storeFiles[0].Store.Settings().SV)
-	readers := make([]sstable.ReadableFile, 0, len(storeFiles))
+	readerLevels := make([][]sstable.ReadableFile, 0, len(storeFiles))
 
 	for _, sf := range storeFiles {
 		// prevent capturing the loop variables by reference when defining openAt below.
@@ -160,9 +163,12 @@ func ExternalSSTReader(
 			}
 			reader = raw
 		}
-		readers = append(readers, reader)
+		readerLevels = append(readerLevels, []sstable.ReadableFile{reader})
 	}
-	return storage.NewPebbleSSTIterator(readers, iterOpts)
+	// NB: It's okay to pass forwardOnly=true, because this function returns a
+	// SimpleMVCCIterator which does not provide an interface for reverse
+	// iteration.
+	return storage.NewPebbleSSTIterator(readerLevels, iterOpts, true /* forwardOnly */)
 }
 
 type sstReader struct {

diff --git a/pkg/storage/pebble_iterator.go b/pkg/storage/pebble_iterator.go
@@ -105,13 +105,20 @@ func newPebbleIteratorByCloning(
 }
 
 // newPebbleSSTIterator creates a new Pebble iterator for the given SSTs.
-func newPebbleSSTIterator(files []sstable.ReadableFile, opts IterOptions) (*pebbleIterator, error) {
+func newPebbleSSTIterator(
+	files [][]sstable.ReadableFile, opts IterOptions, forwardOnly bool,
+) (*pebbleIterator, error) {
 	p := pebbleIterPool.Get().(*pebbleIterator)
 	p.reusable = false // defensive
 	p.init(nil, opts, StandardDurability, true /* supportsRangeKeys */)
 
+	var externalIterOpts []pebble.ExternalIterOption
+	if forwardOnly {
+		externalIterOpts = append(externalIterOpts, pebble.ExternalIterForwardOnly{})
+	}
+
 	var err error
-	if p.iter, err = pebble.NewExternalIter(DefaultPebbleOptions(), &p.options, files); err != nil {
+	if p.iter, err = pebble.NewExternalIter(DefaultPebbleOptions(), &p.options, files, externalIterOpts...); err != nil {
 		p.Close()
 		return nil, err
 	}
@@ -228,7 +235,7 @@ func (p *pebbleIterator) setOptions(opts IterOptions, durability DurabilityRequi
 			p.rangeKeyMaskingBuf, opts.RangeKeyMaskingBelow)
 		p.options.RangeKeyMasking.Suffix = p.rangeKeyMaskingBuf
 		p.maskFilter.BlockIntervalFilter.Init(mvccWallTimeIntervalCollector, 0, math.MaxUint64)
-		p.options.RangeKeyMasking.Filter = &p.maskFilter
+		p.options.RangeKeyMasking.Filter = p.getBlockPropertyFilterMask
 	}
 
 	if opts.MaxTimestampHint.IsSet() {
@@ -881,6 +888,10 @@ func (p *pebbleIterator) GetRawIter() *pebble.Iterator {
 	return p.iter
 }
 
+func (p *pebbleIterator) getBlockPropertyFilterMask() pebble.BlockPropertyFilterMask {
+	return &p.maskFilter
+}
+
 func (p *pebbleIterator) destroy() {
 	if p.inuse {
 		panic("iterator still in use")

diff --git a/pkg/storage/sst_iterator.go b/pkg/storage/sst_iterator.go
@@ -19,17 +19,25 @@ import (
 	"github.com/cockroachdb/pebble/vfs"
 )
 
-// NewPebbleSSTIterator returns an `MVCCIterator` for the provided SST files.
-// The SSTs are merged during iteration. The file slice must be sorted in
-// reverse chronological order: a key in a file at a lower index will shadow the
-// same key contained within a file at a higher index.
+// NewPebbleSSTIterator returns an `MVCCIterator` for the provided "levels" of
+// SST files.  The SSTs are merged during iteration. Each subslice's sstables
+// must have non-overlapping point keys, and be ordered by point key in
+// ascending order. Range keys may overlap arbitrarily, including within a
+// subarray. The outer slice of levels must be sorted in reverse chronological
+// order: a key in a file in a level at a lower index will shadow the same key
+// contained within a file in a level at a higher index.
+//
+// If the iterator is only going to be used for forward iteration, the caller
+// may pass forwardOnly=true for better performance.
 //
 // TODO(erikgrinaker): This currently has significant performance overhead
 // compared with sstIterator. This must be optimized, and then replace (or be
 // used in) NewSSTIterator and NewMemSSTIterator. It should also replace
 // MultiIterator.
-func NewPebbleSSTIterator(files []sstable.ReadableFile, opts IterOptions) (MVCCIterator, error) {
-	return newPebbleSSTIterator(files, opts)
+func NewPebbleSSTIterator(
+	files [][]sstable.ReadableFile, opts IterOptions, forwardOnly bool,
+) (MVCCIterator, error) {
+	return newPebbleSSTIterator(files, opts, forwardOnly)
 }
 
 // NewPebbleMemSSTIterator returns an `MVCCIterator` for the provided SST data,
@@ -47,7 +55,7 @@ func NewPebbleMultiMemSSTIterator(
 	for _, sst := range ssts {
 		files = append(files, vfs.NewMemFile(sst))
 	}
-	iter, err := NewPebbleSSTIterator(files, opts)
+	iter, err := NewPebbleSSTIterator([][]sstable.ReadableFile{files}, opts, false /* forwardOnly */)
 	if err != nil {
 		return nil, err
 	}

diff --git a/vendor b/vendor
+291 −32		github.com/cockroachdb/pebble/external_iterator.go
+8 −3		github.com/cockroachdb/pebble/iterator.go
+5 −3		github.com/cockroachdb/pebble/options.go
+9 −5		github.com/cockroachdb/pebble/range_keys.go
+1 −1		modules.txt