cliccl: add encryption-registry-list command

The existing `enc_util` package contains a tool that could be used to
dump the files in an encryption registry. This command has been broken
since the file registry format was updated.

Add the `(*PebbleFileRegistry).List` function, that returns a map of
files in the registry. Adapt existing test cases.

Add a `debug encryption-registry-list` command that will print all files
contained in the registry of an encrypted store. This is useful for
debugging which store / data key was used to encrypt each file,
replacing the equivalent functionality in `enc_util`.

Touches: cockroachdb#89095.
Epic: None.

Release note (ops change): Adds a new command that can be used by an
operator to list the files present in the Encryption-At-Rest file
registry.

pkg/ccl/cliccl/BUILD.bazel

@@ -30,6 +30,7 @@ go_library(
         "//pkg/util/timeutil",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_cockroachdb_errors//oserror",
+        "@com_github_cockroachdb_pebble//vfs",
         "@com_github_spf13_cobra//:cobra",
     ],
 )
@@ -52,8 +53,10 @@ go_test(
         "//pkg/server",
         "//pkg/storage",
         "//pkg/testutils/serverutils",
+        "//pkg/util/envutil",
         "//pkg/util/leaktest",
         "//pkg/util/log",
+        "//pkg/util/randutil",
         "@com_github_spf13_cobra//:cobra",
         "@com_github_stretchr_testify//require",
     ],

pkg/ccl/cliccl/debug.go

@@ -92,11 +92,22 @@ stdout.
 		RunE: clierrorplus.MaybeDecorateError(runDecrypt),
 	}
 
+	encryptionRegistryList := &cobra.Command{
+		Use:   "encryption-registry-list <directory>",
+		Short: "list files in the encryption-at-rest file registry",
+		Long: `Prints a list of files in an Encryption At Rest file registry, along
+with their env type and encryption settings (if applicable).
+`,
+		Args: cobra.MinimumNArgs(1),
+		RunE: clierrorplus.MaybeDecorateError(runList),
+	}
+
 	// Add commands to the root debug command.
 	// We can't add them to the lists of commands (eg: DebugCmdsForPebble) as cli init() is called before us.
 	cli.DebugCmd.AddCommand(encryptionStatusCmd)
 	cli.DebugCmd.AddCommand(encryptionActiveKeyCmd)
 	cli.DebugCmd.AddCommand(encryptionDecryptCmd)
+	cli.DebugCmd.AddCommand(encryptionRegistryList)
 
 	// Add the encryption flag to commands that need it.
 	// For the encryption-status command.
@@ -108,6 +119,9 @@ stdout.
 	// For the encryption-decrypt command.
 	f = encryptionDecryptCmd.Flags()
 	cliflagcfg.VarFlag(f, &storeEncryptionSpecs, cliflagsccl.EnterpriseEncryption)
+	// For the encryption-registry-list command.
+	f = encryptionRegistryList.Flags()
+	cliflagcfg.VarFlag(f, &storeEncryptionSpecs, cliflagsccl.EnterpriseEncryption)
 
 	// Add encryption flag to all OSS debug commands that want it.
 	for _, cmd := range cli.DebugCommandsRequiringEncryption {

pkg/ccl/cliccl/ear.go

@@ -9,14 +9,21 @@
 package cliccl
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"os"
+	"sort"
 
+	"github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl"
 	"github.com/cockroachdb/cockroach/pkg/cli"
 	"github.com/cockroachdb/cockroach/pkg/storage"
+	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
+	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
 	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/vfs"
 	"github.com/spf13/cobra"
 )
 
@@ -57,3 +64,61 @@ func runDecrypt(_ *cobra.Command, args []string) (returnErr error) {
 
 	return nil
 }
+
+type fileEntry struct {
+	name     string
+	envType  enginepb.EnvType
+	settings enginepbccl.EncryptionSettings
+}
+
+func (f fileEntry) String() string {
+	var b bytes.Buffer
+	_, _ = fmt.Fprintf(
+		&b, "%s:\n  env type: %s, %s\n",
+		f.name, f.envType, f.settings.EncryptionType,
+	)
+	if f.settings.EncryptionType != enginepbccl.EncryptionType_Plaintext {
+		_, _ = fmt.Fprintf(
+			&b, "  keyID: %s\n  nonce: % x\n  counter: %d",
+			f.settings.KeyId, f.settings.Nonce, f.settings.Counter,
+		)
+	}
+	return b.String()
+}
+
+func runList(cmd *cobra.Command, args []string) error {
+	dir := args[0]
+
+	fr := &storage.PebbleFileRegistry{
+		FS:       vfs.Default,
+		DBDir:    dir,
+		ReadOnly: true,
+	}
+	if err := fr.Load(); err != nil {
+		return errors.Wrapf(err, "could not load file registry")
+	}
+	defer func() { _ = fr.Close() }()
+
+	// List files and print to stdout.
+	var entries []fileEntry
+	for name, entry := range fr.List() {
+		var encSettings enginepbccl.EncryptionSettings
+		settings := entry.EncryptionSettings
+		if err := protoutil.Unmarshal(settings, &encSettings); err != nil {
+			return errors.Wrapf(err, "could not unmarshal encryption settings for %s", name)
+		}
+		entries = append(entries, fileEntry{
+			name:     name,
+			envType:  entry.EnvType,
+			settings: encSettings,
+		})
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].name < entries[j].name
+	})
+	for _, e := range entries {
+		_, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s\n", e)
+	}
+
+	return nil
+}

pkg/ccl/cliccl/ear_test.go

@@ -11,6 +11,7 @@ package cliccl
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"fmt"
 	"path/filepath"
 	"strings"
@@ -22,8 +23,10 @@ import (
 	_ "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl"
 	"github.com/cockroachdb/cockroach/pkg/cli"
 	"github.com/cockroachdb/cockroach/pkg/storage"
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/randutil"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/require"
 )
@@ -97,6 +100,97 @@ func TestDecrypt(t *testing.T) {
 	require.Contains(t, out, checkStr)
 }
 
+func TestList(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	// Pin the random generator to use a fixed seed. This also requires overriding
+	// the PRNG for the duration of the test. This ensures that all the keys
+	// generated by the key manager are deterministic.
+	reset := envutil.TestSetEnv(t, "COCKROACH_RANDOM_SEED", "1665612120123601000")
+	defer reset()
+	randBefore := rand.Reader
+	randOverride, _ := randutil.NewPseudoRand()
+	rand.Reader = randOverride
+	defer func() { rand.Reader = randBefore }()
+
+	ctx := context.Background()
+	dir := t.TempDir()
+
+	// Generate a new encryption key to use.
+	keyPath := filepath.Join(dir, "aes.key")
+	err := cli.GenEncryptionKeyCmd.RunE(nil, []string{keyPath})
+	require.NoError(t, err)
+
+	// Spin up a new encrypted store.
+	encSpecStr := fmt.Sprintf("path=%s,key=%s,old-key=plain", dir, keyPath)
+	encSpec, err := baseccl.NewStoreEncryptionSpec(encSpecStr)
+	require.NoError(t, err)
+	encOpts, err := encSpec.ToEncryptionOptions()
+	require.NoError(t, err)
+	p, err := storage.Open(ctx, storage.Filesystem(dir), storage.EncryptionAtRest(encOpts))
+	require.NoError(t, err)
+
+	// Write a key and flush, to create a table in the store.
+	err = p.PutUnversioned([]byte("foo"), nil)
+	require.NoError(t, err)
+	err = p.Flush()
+	require.NoError(t, err)
+	p.Close()
+
+	// List the files in the registry.
+	cmd := getTool(cli.DebugCmd, []string{"debug", "encryption-registry-list"})
+	require.NotNil(t, cmd)
+	var b bytes.Buffer
+	cmd.SetOut(&b)
+	cmd.SetErr(&b)
+	err = runList(cmd, []string{dir})
+	require.NoError(t, err)
+
+	const want = `000002.log:
+  env type: Data, AES128_CTR
+  keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
+  nonce: 06 c2 26 f9 68 f0 fc ff b9 e7 82 8f
+  counter: 914487965
+000004.log:
+  env type: Data, AES128_CTR
+  keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
+  nonce: 80 18 c0 79 61 c7 cf ef b4 25 4e 78
+  counter: 1483615076
+000005.sst:
+  env type: Data, AES128_CTR
+  keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
+  nonce: 71 12 f7 22 9a fb 90 24 4e 58 27 01
+  counter: 3082989236
+COCKROACHDB_DATA_KEYS_000001_monolith:
+  env type: Store, AES128_CTR
+  keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867
+  nonce: 8f 4c ba 1a a3 4f db 3c db 84 cf f5
+  counter: 2436226951
+CURRENT:
+  env type: Data, AES128_CTR
+  keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
+  nonce: 18 c2 a6 23 cc 6e 2e 7c 8e bf 84 77
+  counter: 3159373900
+MANIFEST-000001:
+  env type: Data, AES128_CTR
+  keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
+  nonce: 2e fd 49 2f 5f c5 53 0a e8 8f 78 cc
+  counter: 110434741
+OPTIONS-000003:
+  env type: Data, AES128_CTR
+  keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
+  nonce: d3 97 11 b3 1a ed 22 2b 74 fb 02 0c
+  counter: 1229228536
+marker.datakeys.000001.COCKROACHDB_DATA_KEYS_000001_monolith:
+  env type: Store, AES128_CTR
+  keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867
+  nonce: 55 d7 d4 27 6c 97 9b dd f1 5d 40 c8
+  counter: 467030050
+`
+	require.Equal(t, want, b.String())
+}
+
 // getTool traverses the given cobra.Command recursively, searching for a tool
 // matching the given command.
 func getTool(cmd *cobra.Command, want []string) *cobra.Command {

pkg/storage/pebble_file_registry.go

@@ -532,6 +532,19 @@ func (r *PebbleFileRegistry) getRegistryCopy() *enginepb.FileRegistry {
 	return rv
 }
 
+// List returns a mapping of file in the registry to their enginepb.FileEntry.
+func (r *PebbleFileRegistry) List() map[string]*enginepb.FileEntry {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	// Perform a defensive deep-copy of the internal map here, as there may be
+	// modifications to it after it has been returned to the caller.
+	m := make(map[string]*enginepb.FileEntry, len(r.mu.entries))
+	for k, v := range r.mu.entries {
+		m[k] = v
+	}
+	return m
+}
+
 // Close closes the record writer and record file used for the registry.
 // It should be called when a Pebble instance is closed.
 func (r *PebbleFileRegistry) Close() error {

pkg/storage/pebble_file_registry_test.go

@@ -16,6 +16,7 @@ import (
 	"io"
 	"os"
 	"runtime/debug"
+	"sort"
 	"strings"
 	"testing"
 
@@ -383,6 +384,29 @@ func TestFileRegistry(t *testing.T) {
 				require.NoError(t, f.Close())
 			}
 			return buf.String()
+		case "list":
+			type fileEntry struct {
+				name  string
+				entry *enginepb.FileEntry
+			}
+			var fileEntries []fileEntry
+			for name, entry := range registry.List() {
+				fileEntries = append(fileEntries, fileEntry{
+					name:  name,
+					entry: entry,
+				})
+			}
+			sort.Slice(fileEntries, func(i, j int) bool {
+				return fileEntries[i].name < fileEntries[j].name
+			})
+			var b bytes.Buffer
+			for _, fe := range fileEntries {
+				b.WriteString(fmt.Sprintf(
+					"name=%s,type=%s,settings=%s\n",
+					fe.name, fe.entry.EnvType.String(), string(fe.entry.EncryptionSettings),
+				))
+			}
+			return b.String()
 		default:
 			panic("unrecognized command " + d.Cmd)
 		}