*: Add vfs.SharedStorage interface for storing sstables

This change adds a vfs.SharedStorage interface that can be
implemented by blob storage drivers. The objstorage.Provider
coming in cockroachdb#2267 will largely be responsible for storing
state of files' locations (i.e. shared or local), and calling into
vfs.SharedStorage as necessary.

options.go

@@ -615,6 +615,18 @@ type Options struct {
 		// Any change in exclusion behavior takes effect only on future written
 		// sstables, and does not start rewriting existing sstables.
 		RequiredInPlaceValueBound UserKeyPrefixBound
+
+		// SharedStorage is a second FS-like storage medium that can be shared
+		// between multiple Pebble instances. It is used to store sstables only, and
+		// is managed by objstorage.Provider. Each sstable might only be written to
+		// by one Pebble instance, but other Pebble instances can possibly read the
+		// same files if they have the path to get to them. The pebble instance that
+		// wrote a file should not delete it if other Pebble instances are known to
+		// be reading this file. This FS is expected to have slower read/write
+		// performance than the default FS above.
+		//
+		// TODO(bilal): Uncomment this once it's in use.
+		// SharedStorage vfs.SharedStorage
 	}
 
 	// Filters is a map from filter policy name to filter policy. It is used for

vfs/vfs.go

@@ -171,6 +171,44 @@ type FS interface {
 	GetDiskUsage(path string) (DiskUsage, error)
 }
 
+// SharedStorage is an interface for a blob storage driver. This is lower-level
+// than an FS-like interface, however FS/File-like abstractions can be built on
+// top of these methods.
+//
+// TODO(bilal): Consider reworking this interface to return File-like objects,
+// and maybe push shared file obsoletion as well as path generation behind this
+// interface.
+type SharedStorage interface {
+	io.Closer
+
+	// ReadFile is shorthand for ReadFileAt with offset 0.
+	ReadFile(basename string) (io.ReadCloser, error)
+
+	// ReadFileAt returns a Reader for requested name reading at offset.
+	ReadFileAt(basename string, offset int64) (io.ReadCloser, int64, error)
+
+	// Writer returns a writer for the requested name.
+	//
+	// A Writer *must* be closed via either Close, and if closing returns a
+	// non-nil error, that error should be handled or reported to the user -- an
+	// implementation may buffer written data until Close and only then return
+	// an error, or Write may return an opaque io.EOF with the underlying cause
+	// returned by the subsequent Close().
+	Writer(basename string) (io.WriteCloser, error)
+
+	// List enumerates files within the supplied prefix, returning a list of
+	// objects within that prefix. If delimiter is non-empty names which have the
+	// same prefix, prior to the delimiter, are grouped into a single result which
+	// is that prefix. The order that results are returned is undefined.
+	List(prefix, delimiter string) []string
+
+	// Delete removes the named file from the store.
+	Delete(basename string) error
+
+	// Size returns the length of the named file in bytes.
+	Size(basename string) (int64, error)
+}
+
 // DiskUsage summarizes disk space usage on a filesystem.
 type DiskUsage struct {
 	// Total disk space available to the current process in bytes.