diff --git a/DEPS.bzl b/DEPS.bzl index 77e1c9a60df8..ddb322c68332 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -1347,10 +1347,10 @@ def go_deps(): patches = [ "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch", ], - sha256 = "dfa5ce136f7d8d40ddf24077323df27de81f0e1889c1058e4453c77cd8c2cb75", - strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20220426173801-b33d6e173cae", + sha256 = "65c359674e777445a63c2268e62d8fc740992c1aa86f042a07344371ba01e46b", + strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20220517003944-e567fec84c6e", urls = [ - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220426173801-b33d6e173cae.zip", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220517003944-e567fec84c6e.zip", ], ) go_repository( diff --git a/go.mod b/go.mod index f38b10b9db00..4a18cd6dc432 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 github.com/cockroachdb/gostdlib v1.13.0 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f - github.com/cockroachdb/pebble v0.0.0-20220426173801-b33d6e173cae + github.com/cockroachdb/pebble v0.0.0-20220517003944-e567fec84c6e github.com/cockroachdb/redact v1.1.3 github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd github.com/cockroachdb/stress v0.0.0-20220310203902-58fb4627376e diff --git a/go.sum b/go.sum index 90fb6b96e6cb..7fdf03c60f06 100644 --- a/go.sum +++ b/go.sum @@ -453,8 +453,8 @@ github.com/cockroachdb/gostdlib v1.13.0/go.mod h1:eXX95p9QDrYwJfJ6AgeN9QnRa/lqqi github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20220426173801-b33d6e173cae h1:7lGpwt2wTBh7FApXTEdTDX5OFWsEg9CCe8wMlHJZDwA= -github.com/cockroachdb/pebble v0.0.0-20220426173801-b33d6e173cae/go.mod h1:buxOO9GBtOcq1DiXDpIPYrmxY020K2A8lOrwno5FetU= +github.com/cockroachdb/pebble v0.0.0-20220517003944-e567fec84c6e h1:PU73bIcAcMerOI+xzYa4f3Grrd4I5cxO2ffT9+OcRt0= +github.com/cockroachdb/pebble v0.0.0-20220517003944-e567fec84c6e/go.mod h1:buxOO9GBtOcq1DiXDpIPYrmxY020K2A8lOrwno5FetU= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= diff --git a/pkg/storage/disk_map_test.go b/pkg/storage/disk_map_test.go index 4bc4bade791d..afde670d624d 100644 --- a/pkg/storage/disk_map_test.go +++ b/pkg/storage/disk_map_test.go @@ -179,7 +179,6 @@ func TestPebbleMap(t *testing.T) { defer e.Close() runTestForEngine(ctx, t, testutils.TestDataPath(t, "diskmap"), e) - } func TestPebbleMultiMap(t *testing.T) { diff --git a/pkg/storage/engine_test.go b/pkg/storage/engine_test.go index 82fdf731ca1d..03fec4795853 100644 --- a/pkg/storage/engine_test.go +++ b/pkg/storage/engine_test.go @@ -574,20 +574,16 @@ func TestEngineMustExist(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - test := func(errStr string) { - tempDir, dirCleanupFn := testutils.TempDir(t) - defer dirCleanupFn() + tempDir, dirCleanupFn := testutils.TempDir(t) + defer dirCleanupFn() - _, err := Open(context.Background(), Filesystem(tempDir), MustExist) - if err == nil { - t.Fatal("expected error related to missing directory") - } - if !strings.Contains(fmt.Sprint(err), errStr) { - t.Fatal(err) - } + _, err := Open(context.Background(), Filesystem(tempDir), MustExist) + if err == nil { + t.Fatal("expected error related to missing directory") + } + if !strings.Contains(fmt.Sprint(err), "no such file or directory") { + t.Fatal(err) } - - test("no such file or directory") } func TestEngineTimeBound(t *testing.T) { diff --git a/pkg/storage/open.go b/pkg/storage/open.go index e5943a0441ca..816339dac780 100644 --- a/pkg/storage/open.go +++ b/pkg/storage/open.go @@ -164,11 +164,7 @@ type Location struct { func Filesystem(dir string) Location { return Location{ dir: dir, - // fs is left nil intentionally, so that it will be left as the - // default of vfs.Default wrapped in vfs.WithDiskHealthChecks - // (initialized by DefaultPebbleOptions). - // TODO(jackson): Refactor to make it harder to accidentally remove - // disk health checks by setting your own VFS in a call to NewPebble. + fs: vfs.Default, } } @@ -196,9 +192,7 @@ func Open(ctx context.Context, loc Location, opts ...ConfigOption) (*Pebble, err var cfg engineConfig cfg.Dir = loc.dir cfg.Opts = DefaultPebbleOptions() - if loc.fs != nil { - cfg.Opts.FS = loc.fs - } + cfg.Opts.FS = loc.fs for _, opt := range opts { if err := opt(&cfg); err != nil { return nil, err diff --git a/pkg/storage/pebble.go b/pkg/storage/pebble.go index 29b97fc3eac7..89bdc5c7d16b 100644 --- a/pkg/storage/pebble.go +++ b/pkg/storage/pebble.go @@ -579,6 +579,7 @@ func DefaultPebbleOptions() *pebble.Options { opts := &pebble.Options{ Comparer: EngineComparer, + FS: vfs.Default, L0CompactionThreshold: 2, L0StopWritesThreshold: 1000, LBaseMaxBytes: 64 << 20, // 64 MB @@ -635,8 +636,15 @@ func DefaultPebbleOptions() *pebble.Options { // of the benefit of having bloom filters on every level for only 10% of the // memory cost. opts.Levels[6].FilterPolicy = nil + return opts +} - // Set disk health check interval to min(5s, maxSyncDurationDefault). This +// wrapFilesystemMiddleware wraps the Option's vfs.FS with disk-health checking +// and ENOSPC detection. It mutates the provided options to set the FS and +// returns a Closer that should be invoked when the filesystem will no longer be +// used. +func wrapFilesystemMiddleware(opts *pebble.Options) io.Closer { + // Set disk-health check interval to min(5s, maxSyncDurationDefault). This // is mostly to ease testing; the default of 5s is too infrequent to test // conveniently. See the disk-stalled roachtest for an example of how this // is used. @@ -644,9 +652,11 @@ func DefaultPebbleOptions() *pebble.Options { if diskHealthCheckInterval.Seconds() > maxSyncDurationDefault.Seconds() { diskHealthCheckInterval = maxSyncDurationDefault } - // Instantiate a file system with disk health checking enabled. This FS wraps - // vfs.Default, and can be wrapped for encryption-at-rest. - opts.FS = vfs.WithDiskHealthChecks(vfs.Default, diskHealthCheckInterval, + // Instantiate a file system with disk health checking enabled. This FS + // wraps the filesystem with a layer that times all write-oriented + // operations. + var closer io.Closer + opts.FS, closer = vfs.WithDiskHealthChecks(opts.FS, diskHealthCheckInterval, func(name string, duration time.Duration) { opts.EventListener.DiskSlow(pebble.DiskSlowInfo{ Path: name, @@ -657,7 +667,7 @@ func DefaultPebbleOptions() *pebble.Options { opts.FS = vfs.OnDiskFull(opts.FS, func() { exit.WithCode(exit.DiskFull()) }) - return opts + return closer } type pebbleLogger struct { @@ -733,6 +743,9 @@ type Pebble struct { syncutil.Mutex flushCompletedCallback func() } + // closer is populated when the database is opened. The closer is associated + // with the filesyetem + closer io.Closer wrappedIntentWriter intentDemuxWriter @@ -825,13 +838,23 @@ func ResolveEncryptedEnvOptions(cfg *PebbleConfig) (*PebbleFileRegistry, *Encryp } // NewPebble creates a new Pebble instance, at the specified path. -func NewPebble(ctx context.Context, cfg PebbleConfig) (*Pebble, error) { +func NewPebble(ctx context.Context, cfg PebbleConfig) (p *Pebble, err error) { // pebble.Open also calls EnsureDefaults, but only after doing a clone. Call // EnsureDefaults beforehand so we have a matching cfg here for when we save // cfg.FS and cfg.ReadOnly later on. if cfg.Opts == nil { cfg.Opts = DefaultPebbleOptions() } + + // Initialize the FS, wrapping it with disk health-checking and + // ENOSPC-detection. + filesystemCloser := wrapFilesystemMiddleware(cfg.Opts) + defer func() { + if err != nil { + filesystemCloser.Close() + } + }() + cfg.Opts.EnsureDefaults() cfg.Opts.ErrorIfNotExists = cfg.MustExist if settings := cfg.Settings; settings != nil { @@ -853,8 +876,6 @@ func NewPebble(ctx context.Context, cfg PebbleConfig) (*Pebble, error) { // FS for those that need it. Some call sites need the unencrypted // FS for the purpose of atomic renames. unencryptedFS := cfg.Opts.FS - // TODO(jackson): Assert that unencryptedFS provides atomic renames. - fileRegistry, env, err := ResolveEncryptedEnvOptions(&cfg) if err != nil { return nil, err @@ -899,7 +920,7 @@ func NewPebble(ctx context.Context, cfg PebbleConfig) (*Pebble, error) { storeProps := computeStoreProperties(ctx, cfg.Dir, cfg.Opts.ReadOnly, env != nil /* encryptionEnabled */) - p := &Pebble{ + p = &Pebble{ readOnly: cfg.Opts.ReadOnly, path: cfg.Dir, auxDir: auxDir, @@ -915,6 +936,7 @@ func NewPebble(ctx context.Context, cfg PebbleConfig) (*Pebble, error) { unencryptedFS: unencryptedFS, logger: cfg.Opts.Logger, storeIDPebbleLog: storeIDContainer, + closer: filesystemCloser, } cfg.Opts.EventListener = pebble.TeeEventListener( pebble.MakeLoggingEventListener(pebbleLogger{ @@ -1030,6 +1052,9 @@ func (p *Pebble) Close() { if p.encryption != nil { _ = p.encryption.Closer.Close() } + if p.closer != nil { + _ = p.closer.Close() + } } // Closed implements the Engine interface. diff --git a/pkg/storage/temp_engine.go b/pkg/storage/temp_engine.go index e5fadfada634..bbdae5baf9ca 100644 --- a/pkg/storage/temp_engine.go +++ b/pkg/storage/temp_engine.go @@ -12,6 +12,7 @@ package storage import ( "context" + "io" "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/diskmap" @@ -29,13 +30,16 @@ func NewTempEngine( } type pebbleTempEngine struct { - db *pebble.DB + db *pebble.DB + closer io.Closer } // Close implements the diskmap.Factory interface. func (r *pebbleTempEngine) Close() { - err := r.db.Close() - if err != nil { + if err := r.db.Close(); err != nil { + log.Fatalf(context.TODO(), "%v", err) + } + if err := r.closer.Close(); err != nil { log.Fatalf(context.TODO(), "%v", err) } } @@ -93,6 +97,8 @@ func newPebbleTempEngine( // Set store ID for the pebble engine. p.SetStoreID(ctx, base.TempStoreID) - - return &pebbleTempEngine{db: p.db}, p, nil + return &pebbleTempEngine{ + db: p.db, + closer: p.closer, + }, p, nil } diff --git a/vendor b/vendor index 73c75fbaf35e..c404bd45895f 160000 --- a/vendor +++ b/vendor @@ -1 +1 @@ -Subproject commit 73c75fbaf35e68220c2f5d3a2acfed24dcd9fbef +Subproject commit c404bd45895f1a010fb547a4f6c833d392fc4062