tool: better errors in case of encrypted store

This commit improves the output of the pebble tool when it is run
against an encrypted store and the encryption key is not set up
correctly.

Until now, we get a very obscure "invalid key=value syntax" error from
inside the options parsing code. What's worse the error contains
unescaped non-printable characters which can end up erasing the
message prefix altogether.

Finally, the errors show up on stdout instead of stderr.

This commit
 - escapes the non-printable characters and limits the length in the
   above error message
 - adds more context as to where the error is coming from
 - marks the error as a corruption error
 - adds an option to the tool to allow augmenting open errors. From
   the cockroach side, we can mention encryption in corruption cases.
 - switches to using stderr for all `pebble db` subcommand errors.

Before:
```
$ go run ./cmd/pebble db scan /home/radu/local/1/data
<invalid UTF-8 garbage>
```

After:
```
$ go run ./cmd/pebble db scan /home/radu/local/1/data
error loading options: invalid key=value syntax: "\x96N\xa71$D\x81p\x1cƻ\xc1U\xee6\x88\x80\xd1\xdf\xf7R\x9c\xe2\xe5\xa2\xd2H\xb4\xd1\r+('W\xfeu\x9b\xbd1\xcafC\xd6\x01r\x80c..."
```

open.go

@@ -89,7 +89,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 	// Open the database and WAL directories first.
 	walDirname, dataDir, walDir, err := prepareAndOpenDirs(dirname, opts)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "error opening database at %q", dirname)
 	}
 	defer func() {
 		if db == nil {
@@ -1106,6 +1106,12 @@ var ErrDBAlreadyExists = errors.New("pebble: database already exists")
 // Note that errors can be wrapped with more details; use errors.Is().
 var ErrDBNotPristine = errors.New("pebble: database already exists and is not pristine")
 
+// IsCorruptionError returns true if the given error indicates database
+// corruption.
+func IsCorruptionError(err error) bool {
+	return errors.Is(err, base.ErrCorruption)
+}
+
 func checkConsistency(v *manifest.Version, dirname string, objProvider objstorage.Provider) error {
 	var buf bytes.Buffer
 	var args []interface{}

options.go

@@ -1283,7 +1283,11 @@ func parseOptions(s string, fn func(section, key, value string) error) error {
 
 		pos := strings.Index(line, "=")
 		if pos < 0 {
-			return errors.Errorf("pebble: invalid key=value syntax: %s", errors.Safe(line))
+			const maxLen = 50
+			if len(line) > maxLen {
+				line = line[:maxLen-3] + "..."
+			}
+			return base.CorruptionErrorf("invalid key=value syntax: %q", errors.Safe(line))
 		}
 
 		key := strings.TrimSpace(line[:pos])

tool/data_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/datadriven"
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/testkeys"
 	"github.com/cockroachdb/pebble/vfs"
@@ -105,12 +106,19 @@ func runTests(t *testing.T, path string) {
 					m.Name = "test-merger"
 					return &m
 				}()
+				openErrEnhancer := func(err error) error {
+					if errors.Is(err, base.ErrCorruption) {
+						return base.CorruptionErrorf("%v\nCustom message in case of corruption error.", err)
+					}
+					return err
+				}
 
 				tool := New(
 					DefaultComparer(comparer),
 					Comparers(altComparer, testkeys.Comparer),
 					Mergers(merger),
 					FS(fs),
+					OpenErrEnhancer(openErrEnhancer),
 				)
 
 				c := &cobra.Command{}

tool/db.go

@@ -40,9 +40,10 @@ type dbT struct {
 	IOBench    *cobra.Command
 
 	// Configuration.
-	opts      *pebble.Options
-	comparers sstable.Comparers
-	mergers   sstable.Mergers
+	opts            *pebble.Options
+	comparers       sstable.Comparers
+	mergers         sstable.Mergers
+	openErrEnhancer func(error) error
 
 	// Flags.
 	comparerName  string
@@ -59,11 +60,17 @@ type dbT struct {
 	verbose       bool
 }
 
-func newDB(opts *pebble.Options, comparers sstable.Comparers, mergers sstable.Mergers) *dbT {
+func newDB(
+	opts *pebble.Options,
+	comparers sstable.Comparers,
+	mergers sstable.Mergers,
+	openErrEnhancer func(error) error,
+) *dbT {
 	d := &dbT{
-		opts:      opts,
-		comparers: comparers,
-		mergers:   mergers,
+		opts:            opts,
+		comparers:       comparers,
+		mergers:         mergers,
+		openErrEnhancer: openErrEnhancer,
 	}
 	d.fmtKey.mustSet("quoted")
 	d.fmtValue.mustSet("[%x]")
@@ -281,9 +288,20 @@ type openOption interface {
 }
 
 func (d *dbT) openDB(dir string, openOptions ...openOption) (*pebble.DB, error) {
-	if err := d.loadOptions(dir); err != nil {
+	db, err := d.openDBInternal(dir, openOptions...)
+	if err != nil {
+		if d.openErrEnhancer != nil {
+			err = d.openErrEnhancer(err)
+		}
 		return nil, err
 	}
+	return db, nil
+}
+
+func (d *dbT) openDBInternal(dir string, openOptions ...openOption) (*pebble.DB, error) {
+	if err := d.loadOptions(dir); err != nil {
+		return nil, errors.Wrap(err, "error loading options")
+	}
 	if d.comparerName != "" {
 		d.opts.Comparer = d.comparers[d.comparerName]
 		if d.opts.Comparer == nil {
@@ -305,24 +323,24 @@ func (d *dbT) openDB(dir string, openOptions ...openOption) (*pebble.DB, error)
 	return pebble.Open(dir, &opts)
 }
 
-func (d *dbT) closeDB(stdout io.Writer, db *pebble.DB) {
+func (d *dbT) closeDB(stderr io.Writer, db *pebble.DB) {
 	if err := db.Close(); err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 	}
 }
 
 func (d *dbT) runCheck(cmd *cobra.Command, args []string) {
-	stdout := cmd.OutOrStdout()
+	stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
 	db, err := d.openDB(args[0])
 	if err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 		return
 	}
-	defer d.closeDB(stdout, db)
+	defer d.closeDB(stderr, db)
 
 	var stats pebble.CheckLevelsStats
 	if err := db.CheckLevels(&stats); err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 	}
 	fmt.Fprintf(stdout, "checked %d %s and %d %s\n",
 		stats.NumPoints, makePlural("point", stats.NumPoints), stats.NumTombstones, makePlural("tombstone", int64(stats.NumTombstones)))
@@ -338,37 +356,37 @@ func (n nonReadOnly) apply(opts *pebble.Options) {
 }
 
 func (d *dbT) runCheckpoint(cmd *cobra.Command, args []string) {
-	stdout := cmd.OutOrStdout()
+	stderr := cmd.ErrOrStderr()
 	db, err := d.openDB(args[0], nonReadOnly{})
 	if err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 		return
 	}
-	defer d.closeDB(stdout, db)
+	defer d.closeDB(stderr, db)
 	destDir := args[1]
 
 	if err := db.Checkpoint(destDir); err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 	}
 }
 
 func (d *dbT) runGet(cmd *cobra.Command, args []string) {
-	stdout := cmd.OutOrStdout()
+	stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
 	db, err := d.openDB(args[0])
 	if err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 		return
 	}
-	defer d.closeDB(stdout, db)
+	defer d.closeDB(stderr, db)
 	var k key
 	if err := k.Set(args[1]); err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 		return
 	}
 
 	val, closer, err := db.Get(k)
 	if err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 		return
 	}
 	defer func() {
@@ -382,25 +400,25 @@ func (d *dbT) runGet(cmd *cobra.Command, args []string) {
 }
 
 func (d *dbT) runLSM(cmd *cobra.Command, args []string) {
-	stdout := cmd.OutOrStdout()
+	stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
 	db, err := d.openDB(args[0])
 	if err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 		return
 	}
-	defer d.closeDB(stdout, db)
+	defer d.closeDB(stderr, db)
 
 	fmt.Fprintf(stdout, "%s", db.Metrics())
 }
 
 func (d *dbT) runScan(cmd *cobra.Command, args []string) {
-	stdout := cmd.OutOrStdout()
+	stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
 	db, err := d.openDB(args[0])
 	if err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 		return
 	}
-	defer d.closeDB(stdout, db)
+	defer d.closeDB(stderr, db)
 
 	// Update the internal formatter if this comparator has one specified.
 	if d.opts.Comparer != nil {
@@ -439,7 +457,7 @@ func (d *dbT) runScan(cmd *cobra.Command, args []string) {
 	}
 
 	if err := iter.Close(); err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 	}
 
 	elapsed := timeNow().Sub(start)
@@ -449,7 +467,7 @@ func (d *dbT) runScan(cmd *cobra.Command, args []string) {
 }
 
 func (d *dbT) runSpace(cmd *cobra.Command, args []string) {
-	stdout, stderr := cmd.OutOrStdout(), cmd.OutOrStderr()
+	stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
 	db, err := d.openDB(args[0])
 	if err != nil {
 		fmt.Fprintf(stderr, "%s\n", err)
@@ -466,7 +484,7 @@ func (d *dbT) runSpace(cmd *cobra.Command, args []string) {
 }
 
 func (d *dbT) runProperties(cmd *cobra.Command, args []string) {
-	stdout, stderr := cmd.OutOrStdout(), cmd.OutOrStderr()
+	stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
 	dirname := args[0]
 	err := func() error {
 		desc, err := pebble.Peek(dirname, d.opts.FS)
@@ -620,25 +638,25 @@ func (d *dbT) runProperties(cmd *cobra.Command, args []string) {
 }
 
 func (d *dbT) runSet(cmd *cobra.Command, args []string) {
-	stdout := cmd.OutOrStdout()
+	stderr := cmd.ErrOrStderr()
 	db, err := d.openDB(args[0], nonReadOnly{})
 	if err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 		return
 	}
-	defer d.closeDB(stdout, db)
+	defer d.closeDB(stderr, db)
 	var k, v key
 	if err := k.Set(args[1]); err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 		return
 	}
 	if err := v.Set(args[2]); err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 		return
 	}
 
 	if err := db.Set(k, v, nil); err != nil {
-		fmt.Fprintf(stdout, "%s\n", err)
+		fmt.Fprintf(stderr, "%s\n", err)
 	}
 }
 

tool/testdata/corrupt-options-db/OPTIONS-000002

@@ -0,0 +1 @@
+blargle

tool/testdata/db_check

@@ -5,7 +5,14 @@ accepts 1 arg(s), received 0
 db check
 non-existent
 ----
-pebble: database "non-existent" does not exist
+error opening database at "non-existent": pebble: database "non-existent" does not exist
+
+db check
+./testdata/corrupt-options-db
+----
+error loading options: invalid key=value syntax: "blargle"
+Custom message in case of corruption error.
+
 
 db check
 ../testdata/db-stage-4

tool/testdata/db_lsm

@@ -5,7 +5,7 @@ accepts 1 arg(s), received 0
 db lsm
 non-existent
 ----
-pebble: database "non-existent" does not exist
+error opening database at "non-existent": pebble: database "non-existent" does not exist
 
 db lsm
 ../testdata/db-stage-4

tool/testdata/db_scan

@@ -5,7 +5,13 @@ accepts 1 arg(s), received 0
 db scan
 non-existent
 ----
-pebble: database "non-existent" does not exist
+error opening database at "non-existent": pebble: database "non-existent" does not exist
+
+db scan
+./testdata/corrupt-options-db
+----
+error loading options: invalid key=value syntax: "blargle"
+Custom message in case of corruption error.
 
 db scan
 ../testdata/db-stage-4

tool/tool.go

@@ -37,6 +37,7 @@ type T struct {
 	comparers       sstable.Comparers
 	mergers         sstable.Mergers
 	defaultComparer string
+	openErrEnhancer func(error) error
 }
 
 // A Option configures the Pebble introspection tool.
@@ -88,6 +89,16 @@ func FS(fs vfs.FS) Option {
 	}
 }
 
+// OpenErrEnhancer sets a function that enhances an error encountered when the
+// tool opens a database; used to provide the user additional context, for
+// example that a corruption error might be caused by encryption at rest not
+// being configured properly.
+func OpenErrEnhancer(fn func(error) error) Option {
+	return func(t *T) {
+		t.openErrEnhancer = fn
+	}
+}
+
 // New creates a new introspection tool.
 func New(opts ...Option) *T {
 	t := &T{
@@ -110,7 +121,7 @@ func New(opts ...Option) *T {
 		opt(t)
 	}
 
-	t.db = newDB(&t.opts, t.comparers, t.mergers)
+	t.db = newDB(&t.opts, t.comparers, t.mergers, t.openErrEnhancer)
 	t.find = newFind(&t.opts, t.comparers, t.defaultComparer, t.mergers)
 	t.lsm = newLSM(&t.opts, t.comparers)
 	t.manifest = newManifest(&t.opts, t.comparers)