Skip to content

Commit

Permalink
*: Add objstorage.shared.Storage interface for storing sstables
Browse files Browse the repository at this point in the history
This change adds a objstorage.shared.Storage interface that can be
implemented by blob storage drivers. The objstorage.Provider
coming in cockroachdb#2267 will largely be responsible for storing
state of files' locations (i.e. shared or local), and calling into
Storage as necessary.
  • Loading branch information
itsbilal committed Feb 6, 2023
1 parent 4eba71e commit 8a77fa3
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 0 deletions.
52 changes: 52 additions & 0 deletions objstorage/shared/storage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package shared

import "io"

// Storage is an interface for a blob storage driver. This is lower-level
// than an FS-like interface, however FS/File-like abstractions can be built on
// top of these methods.
//
// TODO(bilal): Consider pushing shared file obsoletion as well as path
// generation behind this interface.
type Storage interface {
io.Closer

// ReadObjectAt returns a Reader for reading the object at the requested name
// and offset.
ReadObjectAt(basename string, offset int64) (io.ReadCloser, int64, error)

// CreateObject returns a writer for the object at the request name. A new
// empty object is created if CreateObject is called on an existing object.
//
// A Writer *must* be closed via either Close, and if closing returns a
// non-nil error, that error should be handled or reported to the user -- an
// implementation may buffer written data until Close and only then return
// an error, or Write may return an opaque io.EOF with the underlying cause
// returned by the subsequent Close().
CreateObject(basename string) (io.WriteCloser, error)

// List enumerates files within the supplied prefix, returning a list of
// objects within that prefix. If delimiter is non-empty, names which have the
// same prefix, prior to the delimiter but after the prefix, are grouped into a
// single result which is that prefix. The order that results are returned is
// undefined. If a prefix is specified, the prefix is trimmed from the result
// list.
//
// An example would be, if the storage contains objects a, b/4, b/5 and b/6,
// these would be the return values:
// List("", "") -> ["a", "b/4", "b/5", "b/6"]
// List("", "/") -> ["a", "b"]
// List("b", "/") -> ["4", "5", "6"]
// List("b", "") -> ["4", "5", "6"]
List(prefix, delimiter string) []string

// Delete removes the named object from the store.
Delete(basename string) error

// Size returns the length of the named object in bytes.
Size(basename string) (int64, error)
}
12 changes: 12 additions & 0 deletions options.go
Original file line number Diff line number Diff line change
Expand Up @@ -624,6 +624,18 @@ type Options struct {
// https://github.com/cockroachdb/pebble/issues/2292 and
// https://github.com/cockroachdb/pebble/issues/2266 are closed.
IngestSSTablesAsFlushable bool

// SharedStorage is a second FS-like storage medium that can be shared
// between multiple Pebble instances. It is used to store sstables only, and
// is managed by objstorage.Provider. Each sstable might only be written to
// by one Pebble instance, but other Pebble instances can possibly read the
// same files if they have the path to get to them. The pebble instance that
// wrote a file should not delete it if other Pebble instances are known to
// be reading this file. This FS is expected to have slower read/write
// performance than the default FS above.
//
// TODO(bilal): Uncomment this once it's in use.
// SharedStorage shared.Storage
}

// Filters is a map from filter policy name to filter policy. It is used for
Expand Down

0 comments on commit 8a77fa3

Please sign in to comment.