From 13874c1860412f010166deaa6eb381ba61d234bd Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Tue, 19 Mar 2024 12:56:23 -0400 Subject: [PATCH] db: surface OPTIONS filename in pebble.Peek The pebble.Peek function allows external users to retrieve metadata about an Engine without opening it. This commit adds the filename of the latest OPTIONS file to the output, so the caller may parse the options used by the most recent process. Additionally, this commit adds new methods for locating atomicfs.Marker that use an existing directory listing to avoid repeated listing of the data directory during Open and Peek. --- format_major_version.go | 4 +-- ingest_test.go | 1 + open.go | 56 +++++++++++++++++++++++++++++++++-------- open_test.go | 22 ++++++++++++++++ testdata/peek | 31 +++++++++++++++++++++++ version_set.go | 4 +-- vfs/atomicfs/marker.go | 27 ++++++++++++++------ 7 files changed, 123 insertions(+), 22 deletions(-) create mode 100644 testdata/peek diff --git a/format_major_version.go b/format_major_version.go index f84a5d57a4..b795fa7996 100644 --- a/format_major_version.go +++ b/format_major_version.go @@ -282,9 +282,9 @@ const formatVersionMarkerName = `format-version` // only acceptable if we are creating a new store (we no longer support // FormatMostCompatible which is the only one with no version marker file). func lookupFormatMajorVersion( - fs vfs.FS, dirname string, + fs vfs.FS, dirname string, ls []string, ) (FormatMajorVersion, *atomicfs.Marker, error) { - m, versString, err := atomicfs.LocateMarker(fs, dirname, formatVersionMarkerName) + m, versString, err := atomicfs.LocateMarkerInListing(fs, dirname, formatVersionMarkerName, ls) if err != nil { return 0, nil, err } diff --git a/ingest_test.go b/ingest_test.go index f38edc89ff..512ea295f1 100644 --- a/ingest_test.go +++ b/ingest_test.go @@ -3038,6 +3038,7 @@ func TestIngestMemtableOverlapRace(t *testing.T) { dbDesc, err := Peek("", mem) require.NoError(t, err) require.True(t, dbDesc.Exists) + require.Greater(t, len(dbDesc.OptionsFilename), 0) f, err = mem.Open(dbDesc.ManifestFilename) require.NoError(t, err) defer f.Close() diff --git a/open.go b/open.go index b9843f7243..b4bd52aef5 100644 --- a/open.go +++ b/open.go @@ -124,8 +124,16 @@ func Open(dirname string, opts *Options) (db *DB, err error) { } }() + // List the directory contents. This also happens to include WAL log files, if + // they are in the same dir, but we will ignore those below. The provider is + // also given this list, but it ignores non sstable files. + ls, err := opts.FS.List(dirname) + if err != nil { + return nil, err + } + // Establish the format major version. - formatVersion, formatVersionMarker, err := lookupFormatMajorVersion(opts.FS, dirname) + formatVersion, formatVersionMarker, err := lookupFormatMajorVersion(opts.FS, dirname, ls) if err != nil { return nil, err } @@ -169,7 +177,7 @@ func Open(dirname string, opts *Options) (db *DB, err error) { } // Find the currently active manifest, if there is one. - manifestMarker, manifestFileNum, manifestExists, err := findCurrentManifest(opts.FS, dirname) + manifestMarker, manifestFileNum, manifestExists, err := findCurrentManifest(opts.FS, dirname, ls) if err != nil { return nil, errors.Wrapf(err, "pebble: database %q", dirname) } @@ -281,13 +289,6 @@ func Open(dirname string, opts *Options) (db *DB, err error) { jobID := d.mu.nextJobID d.mu.nextJobID++ - // List the objects. This also happens to include WAL log files, if they are - // in the same dir, but we will ignore those below. The provider is also - // given this list, but it ignores non sstable files. - ls, err := opts.FS.List(d.dirname) - if err != nil { - return nil, err - } providerSettings := objstorageprovider.Settings{ Logger: opts.Logger, FS: opts.FS, @@ -1076,13 +1077,33 @@ type DBDesc struct { // ManifestFilename is the filename of the current active manifest, // if the database exists. ManifestFilename string + // OptionsFilename is the filename of the most recent OPTIONS file, if it + // exists. + OptionsFilename string +} + +// String implements fmt.Stringer. +func (d *DBDesc) String() string { + if !d.Exists { + return "uninitialized" + } + var buf bytes.Buffer + fmt.Fprintf(&buf, "initialized at format major version %s\n", d.FormatMajorVersion) + fmt.Fprintf(&buf, "manifest: %s\n", d.ManifestFilename) + fmt.Fprintf(&buf, "options: %s", d.OptionsFilename) + return buf.String() } // Peek looks for an existing database in dirname on the provided FS. It // returns a brief description of the database. Peek is read-only and // does not open the database func Peek(dirname string, fs vfs.FS) (*DBDesc, error) { - vers, versMarker, err := lookupFormatMajorVersion(fs, dirname) + ls, err := fs.List(dirname) + if err != nil { + return nil, err + } + + vers, versMarker, err := lookupFormatMajorVersion(fs, dirname, ls) if err != nil { return nil, err } @@ -1093,7 +1114,7 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) { } // Find the currently active manifest, if there is one. - manifestMarker, manifestFileNum, exists, err := findCurrentManifest(fs, dirname) + manifestMarker, manifestFileNum, exists, err := findCurrentManifest(fs, dirname, ls) if err != nil { return nil, err } @@ -1107,6 +1128,19 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) { Exists: exists, FormatMajorVersion: vers, } + + // Find the OPTIONS file with the highest file number within the list of + // directory entries. + var previousOptionsFileNum base.DiskFileNum + for _, filename := range ls { + ft, fn, ok := base.ParseFilename(fs, filename) + if !ok || ft != fileTypeOptions || fn < previousOptionsFileNum { + continue + } + previousOptionsFileNum = fn + desc.OptionsFilename = fs.PathJoin(dirname, filename) + } + if exists { desc.ManifestFilename = base.MakeFilepath(fs, dirname, fileTypeManifest, manifestFileNum) } diff --git a/open_test.go b/open_test.go index bc55df9bf6..1875a73b8e 100644 --- a/open_test.go +++ b/open_test.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "runtime/debug" "slices" "sort" @@ -1189,6 +1190,27 @@ func TestOpenWALReplayMemtableGrowth(t *testing.T) { db.Close() } +func TestPeek(t *testing.T) { + // The file paths are UNIX-oriented. To avoid duplicating the test fixtures + // just for Windows, just skip the tests on Windows. + if runtime.GOOS == "windows" { + t.Skip() + } + + datadriven.RunTest(t, "testdata/peek", func(t *testing.T, td *datadriven.TestData) string { + switch td.Cmd { + case "peek": + desc, err := Peek(td.CmdArgs[0].String(), vfs.Default) + if err != nil { + return fmt.Sprintf("err=%v", err) + } + return desc.String() + default: + return fmt.Sprintf("unrecognized command %q\n", td.Cmd) + } + }) +} + func TestGetVersion(t *testing.T) { mem := vfs.NewMem() opts := &Options{ diff --git a/testdata/peek b/testdata/peek new file mode 100644 index 0000000000..c7b6fce739 --- /dev/null +++ b/testdata/peek @@ -0,0 +1,31 @@ +peek testdata/db-stage-1 +---- +initialized at format major version 013 +manifest: testdata/db-stage-1/MANIFEST-000001 +options: testdata/db-stage-1/OPTIONS-000003 + +peek testdata/db-stage-2 +---- +initialized at format major version 013 +manifest: testdata/db-stage-2/MANIFEST-000001 +options: testdata/db-stage-2/OPTIONS-000003 + +peek testdata/db-stage-3 +---- +initialized at format major version 013 +manifest: testdata/db-stage-3/MANIFEST-000006 +options: testdata/db-stage-3/OPTIONS-000007 + +peek testdata/db-stage-4 +---- +initialized at format major version 013 +manifest: testdata/db-stage-4/MANIFEST-000006 +options: testdata/db-stage-4/OPTIONS-000007 + +peek testdata/db-stage-5 +---- +err=open testdata/db-stage-5: no such file or directory + +peek testdata +---- +uninitialized diff --git a/version_set.go b/version_set.go index 093725dd56..d6a371f23e 100644 --- a/version_set.go +++ b/version_set.go @@ -1070,11 +1070,11 @@ func (vs *versionSet) updateObsoleteTableMetricsLocked() { } func findCurrentManifest( - fs vfs.FS, dirname string, + fs vfs.FS, dirname string, ls []string, ) (marker *atomicfs.Marker, manifestNum base.DiskFileNum, exists bool, err error) { // Locating a marker should succeed even if the marker has never been placed. var filename string - marker, filename, err = atomicfs.LocateMarker(fs, dirname, manifestMarkerName) + marker, filename, err = atomicfs.LocateMarkerInListing(fs, dirname, manifestMarkerName, ls) if err != nil { return nil, 0, false, err } diff --git a/vfs/atomicfs/marker.go b/vfs/atomicfs/marker.go index 6bdc506db0..c640754b9d 100644 --- a/vfs/atomicfs/marker.go +++ b/vfs/atomicfs/marker.go @@ -18,7 +18,11 @@ import ( // current value of the marker. Callers that may need to move the marker // to a new value should use LocateMarker. func ReadMarker(fs vfs.FS, dir, markerName string) (string, error) { - state, err := scanForMarker(fs, dir, markerName) + ls, err := fs.List(dir) + if err != nil { + return "", err + } + state, err := scanForMarker(fs, ls, markerName) if err != nil { return "", err } @@ -29,7 +33,20 @@ func ReadMarker(fs vfs.FS, dir, markerName string) (string, error) { // to the Marker that may be used to move the marker and the // current value of the marker. func LocateMarker(fs vfs.FS, dir, markerName string) (*Marker, string, error) { - state, err := scanForMarker(fs, dir, markerName) + ls, err := fs.List(dir) + if err != nil { + return nil, "", err + } + return LocateMarkerInListing(fs, dir, markerName, ls) +} + +// LocateMarkerInListing is like LocateMarker but takes a listing of the files +// contained within dir. It's useful when the caller has already listed the +// directory entries of dir for its own purposes. +func LocateMarkerInListing( + fs vfs.FS, dir, markerName string, ls []string, +) (*Marker, string, error) { + state, err := scanForMarker(fs, ls, markerName) if err != nil { return nil, "", err } @@ -57,11 +74,7 @@ type scannedState struct { obsolete []string } -func scanForMarker(fs vfs.FS, dir, markerName string) (scannedState, error) { - ls, err := fs.List(dir) - if err != nil { - return scannedState{}, err - } +func scanForMarker(fs vfs.FS, ls []string, markerName string) (scannedState, error) { var state scannedState for _, filename := range ls { if !strings.HasPrefix(filename, `marker.`) {