From 0d467be76c47de5d7e98280ed87c3cb5025dabe8 Mon Sep 17 00:00:00 2001 From: Nick Travers Date: Wed, 12 Oct 2022 15:27:03 -0700 Subject: [PATCH] cliccl: add `encryption-registry-list` command The existing `enc_util` package contains a tool that could be used to dump the files in an encryption registry. This command has been broken since the file registry format was updated. Add the `(*PebbleFileRegistry).List` function, that returns a map of files in the registry. Adapt existing test cases. Add a `debug encryption-registry-list` command that will print all files contained in the registry of an encrypted store. This is useful for debugging which store / data key was used to encrypt each file, replacing the equivalent functionality in `enc_util`. Touches: #89095. Epic: None. Release note (ops change): Adds a new command that can be used by an operator to list the files present in the Encryption-At-Rest file registry. --- pkg/ccl/cliccl/BUILD.bazel | 3 + pkg/ccl/cliccl/debug.go | 14 ++++ pkg/ccl/cliccl/ear.go | 65 ++++++++++++++++ pkg/ccl/cliccl/ear_test.go | 94 ++++++++++++++++++++++++ pkg/storage/pebble_file_registry.go | 13 ++++ pkg/storage/pebble_file_registry_test.go | 24 ++++++ pkg/storage/testdata/file_registry | 42 +++++++++++ 7 files changed, 255 insertions(+) diff --git a/pkg/ccl/cliccl/BUILD.bazel b/pkg/ccl/cliccl/BUILD.bazel index 5a88f2e73fdb..473033a01e40 100644 --- a/pkg/ccl/cliccl/BUILD.bazel +++ b/pkg/ccl/cliccl/BUILD.bazel @@ -30,6 +30,7 @@ go_library( "//pkg/util/timeutil", "@com_github_cockroachdb_errors//:errors", "@com_github_cockroachdb_errors//oserror", + "@com_github_cockroachdb_pebble//vfs", "@com_github_spf13_cobra//:cobra", ], ) @@ -52,8 +53,10 @@ go_test( "//pkg/server", "//pkg/storage", "//pkg/testutils/serverutils", + "//pkg/util/envutil", "//pkg/util/leaktest", "//pkg/util/log", + "//pkg/util/randutil", "@com_github_spf13_cobra//:cobra", "@com_github_stretchr_testify//require", ], diff --git a/pkg/ccl/cliccl/debug.go b/pkg/ccl/cliccl/debug.go index dd346aa8f8b1..a30de500cec6 100644 --- a/pkg/ccl/cliccl/debug.go +++ b/pkg/ccl/cliccl/debug.go @@ -92,11 +92,22 @@ stdout. RunE: clierrorplus.MaybeDecorateError(runDecrypt), } + encryptionRegistryList := &cobra.Command{ + Use: "encryption-registry-list ", + Short: "list files in the encryption-at-rest file registry", + Long: `Prints a list of files in an Encryption At Rest file registry, along +with their env type and encryption settings (if applicable). +`, + Args: cobra.MinimumNArgs(1), + RunE: clierrorplus.MaybeDecorateError(runList), + } + // Add commands to the root debug command. // We can't add them to the lists of commands (eg: DebugCmdsForPebble) as cli init() is called before us. cli.DebugCmd.AddCommand(encryptionStatusCmd) cli.DebugCmd.AddCommand(encryptionActiveKeyCmd) cli.DebugCmd.AddCommand(encryptionDecryptCmd) + cli.DebugCmd.AddCommand(encryptionRegistryList) // Add the encryption flag to commands that need it. // For the encryption-status command. @@ -108,6 +119,9 @@ stdout. // For the encryption-decrypt command. f = encryptionDecryptCmd.Flags() cliflagcfg.VarFlag(f, &storeEncryptionSpecs, cliflagsccl.EnterpriseEncryption) + // For the encryption-registry-list command. + f = encryptionRegistryList.Flags() + cliflagcfg.VarFlag(f, &storeEncryptionSpecs, cliflagsccl.EnterpriseEncryption) // Add encryption flag to all OSS debug commands that want it. for _, cmd := range cli.DebugCommandsRequiringEncryption { diff --git a/pkg/ccl/cliccl/ear.go b/pkg/ccl/cliccl/ear.go index c00dc068ee6c..722a902e5672 100644 --- a/pkg/ccl/cliccl/ear.go +++ b/pkg/ccl/cliccl/ear.go @@ -9,14 +9,21 @@ package cliccl import ( + "bytes" "context" + "fmt" "io" "os" + "sort" + "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl" "github.com/cockroachdb/cockroach/pkg/cli" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/vfs" "github.com/spf13/cobra" ) @@ -57,3 +64,61 @@ func runDecrypt(_ *cobra.Command, args []string) (returnErr error) { return nil } + +type fileEntry struct { + name string + envType enginepb.EnvType + settings enginepbccl.EncryptionSettings +} + +func (f fileEntry) String() string { + var b bytes.Buffer + _, _ = fmt.Fprintf( + &b, "%s:\n env type: %s, %s\n", + f.name, f.envType, f.settings.EncryptionType, + ) + if f.settings.EncryptionType != enginepbccl.EncryptionType_Plaintext { + _, _ = fmt.Fprintf( + &b, " keyID: %s\n nonce: % x\n counter: %d", + f.settings.KeyId, f.settings.Nonce, f.settings.Counter, + ) + } + return b.String() +} + +func runList(cmd *cobra.Command, args []string) error { + dir := args[0] + + fr := &storage.PebbleFileRegistry{ + FS: vfs.Default, + DBDir: dir, + ReadOnly: true, + } + if err := fr.Load(); err != nil { + return errors.Wrapf(err, "could not load file registry") + } + defer func() { _ = fr.Close() }() + + // List files and print to stdout. + var entries []fileEntry + for name, entry := range fr.List() { + var encSettings enginepbccl.EncryptionSettings + settings := entry.EncryptionSettings + if err := protoutil.Unmarshal(settings, &encSettings); err != nil { + return errors.Wrapf(err, "could not unmarshal encryption settings for %s", name) + } + entries = append(entries, fileEntry{ + name: name, + envType: entry.EnvType, + settings: encSettings, + }) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].name < entries[j].name + }) + for _, e := range entries { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s\n", e) + } + + return nil +} diff --git a/pkg/ccl/cliccl/ear_test.go b/pkg/ccl/cliccl/ear_test.go index 1fb77826e431..64353c04a5e9 100644 --- a/pkg/ccl/cliccl/ear_test.go +++ b/pkg/ccl/cliccl/ear_test.go @@ -11,6 +11,7 @@ package cliccl import ( "bytes" "context" + "crypto/rand" "fmt" "path/filepath" "strings" @@ -22,8 +23,10 @@ import ( _ "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl" "github.com/cockroachdb/cockroach/pkg/cli" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/randutil" "github.com/spf13/cobra" "github.com/stretchr/testify/require" ) @@ -97,6 +100,97 @@ func TestDecrypt(t *testing.T) { require.Contains(t, out, checkStr) } +func TestList(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + // Pin the random generator to use a fixed seed. This also requires overriding + // the PRNG for the duration of the test. This ensures that all the keys + // generated by the key manager are deterministic. + reset := envutil.TestSetEnv(t, "COCKROACH_RANDOM_SEED", "1665612120123601000") + defer reset() + randBefore := rand.Reader + randOverride, _ := randutil.NewPseudoRand() + rand.Reader = randOverride + defer func() { rand.Reader = randBefore }() + + ctx := context.Background() + dir := t.TempDir() + + // Generate a new encryption key to use. + keyPath := filepath.Join(dir, "aes.key") + err := cli.GenEncryptionKeyCmd.RunE(nil, []string{keyPath}) + require.NoError(t, err) + + // Spin up a new encrypted store. + encSpecStr := fmt.Sprintf("path=%s,key=%s,old-key=plain", dir, keyPath) + encSpec, err := baseccl.NewStoreEncryptionSpec(encSpecStr) + require.NoError(t, err) + encOpts, err := encSpec.ToEncryptionOptions() + require.NoError(t, err) + p, err := storage.Open(ctx, storage.Filesystem(dir), storage.EncryptionAtRest(encOpts)) + require.NoError(t, err) + + // Write a key and flush, to create a table in the store. + err = p.PutUnversioned([]byte("foo"), nil) + require.NoError(t, err) + err = p.Flush() + require.NoError(t, err) + p.Close() + + // List the files in the registry. + cmd := getTool(cli.DebugCmd, []string{"debug", "encryption-registry-list"}) + require.NotNil(t, cmd) + var b bytes.Buffer + cmd.SetOut(&b) + cmd.SetErr(&b) + err = runList(cmd, []string{dir}) + require.NoError(t, err) + + const want = `000002.log: + env type: Data, AES128_CTR + keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef + nonce: 06 c2 26 f9 68 f0 fc ff b9 e7 82 8f + counter: 914487965 +000004.log: + env type: Data, AES128_CTR + keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef + nonce: 80 18 c0 79 61 c7 cf ef b4 25 4e 78 + counter: 1483615076 +000005.sst: + env type: Data, AES128_CTR + keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef + nonce: 71 12 f7 22 9a fb 90 24 4e 58 27 01 + counter: 3082989236 +COCKROACHDB_DATA_KEYS_000001_monolith: + env type: Store, AES128_CTR + keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867 + nonce: 8f 4c ba 1a a3 4f db 3c db 84 cf f5 + counter: 2436226951 +CURRENT: + env type: Data, AES128_CTR + keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef + nonce: 18 c2 a6 23 cc 6e 2e 7c 8e bf 84 77 + counter: 3159373900 +MANIFEST-000001: + env type: Data, AES128_CTR + keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef + nonce: 2e fd 49 2f 5f c5 53 0a e8 8f 78 cc + counter: 110434741 +OPTIONS-000003: + env type: Data, AES128_CTR + keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef + nonce: d3 97 11 b3 1a ed 22 2b 74 fb 02 0c + counter: 1229228536 +marker.datakeys.000001.COCKROACHDB_DATA_KEYS_000001_monolith: + env type: Store, AES128_CTR + keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867 + nonce: 55 d7 d4 27 6c 97 9b dd f1 5d 40 c8 + counter: 467030050 +` + require.Equal(t, want, b.String()) +} + // getTool traverses the given cobra.Command recursively, searching for a tool // matching the given command. func getTool(cmd *cobra.Command, want []string) *cobra.Command { diff --git a/pkg/storage/pebble_file_registry.go b/pkg/storage/pebble_file_registry.go index 708e48c63e21..89f26a7ba936 100644 --- a/pkg/storage/pebble_file_registry.go +++ b/pkg/storage/pebble_file_registry.go @@ -532,6 +532,19 @@ func (r *PebbleFileRegistry) getRegistryCopy() *enginepb.FileRegistry { return rv } +// List returns a mapping of file in the registry to their enginepb.FileEntry. +func (r *PebbleFileRegistry) List() map[string]*enginepb.FileEntry { + r.mu.Lock() + defer r.mu.Unlock() + // Perform a defensive deep-copy of the internal map here, as there may be + // modifications to it after it has been returned to the caller. + m := make(map[string]*enginepb.FileEntry, len(r.mu.entries)) + for k, v := range r.mu.entries { + m[k] = v + } + return m +} + // Close closes the record writer and record file used for the registry. // It should be called when a Pebble instance is closed. func (r *PebbleFileRegistry) Close() error { diff --git a/pkg/storage/pebble_file_registry_test.go b/pkg/storage/pebble_file_registry_test.go index b2e6ca9aec20..33d79c35de3c 100644 --- a/pkg/storage/pebble_file_registry_test.go +++ b/pkg/storage/pebble_file_registry_test.go @@ -16,6 +16,7 @@ import ( "io" "os" "runtime/debug" + "sort" "strings" "testing" @@ -383,6 +384,29 @@ func TestFileRegistry(t *testing.T) { require.NoError(t, f.Close()) } return buf.String() + case "list": + type fileEntry struct { + name string + entry *enginepb.FileEntry + } + var fileEntries []fileEntry + for name, entry := range registry.List() { + fileEntries = append(fileEntries, fileEntry{ + name: name, + entry: entry, + }) + } + sort.Slice(fileEntries, func(i, j int) bool { + return fileEntries[i].name < fileEntries[j].name + }) + var b bytes.Buffer + for _, fe := range fileEntries { + b.WriteString(fmt.Sprintf( + "name=%s,type=%s,settings=%s\n", + fe.name, fe.entry.EnvType.String(), string(fe.entry.EncryptionSettings), + )) + } + return b.String() default: panic("unrecognized command " + d.Cmd) } diff --git a/pkg/storage/testdata/file_registry b/pkg/storage/testdata/file_registry index 71938ac1587a..a1bced6319e8 100644 --- a/pkg/storage/testdata/file_registry +++ b/pkg/storage/testdata/file_registry @@ -9,6 +9,9 @@ load ---- open-dir("") +list +---- + close ---- close("") @@ -24,6 +27,9 @@ load ---- open-dir("") +list +---- + set filename=foo settings=bar ---- create("COCKROACHDB_REGISTRY_000001") @@ -35,6 +41,10 @@ sync("") write("COCKROACHDB_REGISTRY_000001", <...23 bytes...>) sync("COCKROACHDB_REGISTRY_000001") +list +---- +name=foo,type=Data,settings=bar + close ---- write("COCKROACHDB_REGISTRY_000001", <...0 bytes...>) @@ -65,6 +75,9 @@ remove("COCKROACHDB_REGISTRY_000001") write("COCKROACHDB_REGISTRY_000002", <...14 bytes...>) sync("COCKROACHDB_REGISTRY_000002") +list +---- + close ---- write("COCKROACHDB_REGISTRY_000002", <...0 bytes...>) @@ -85,6 +98,9 @@ open-dir("") open("COCKROACHDB_REGISTRY_000002") close("COCKROACHDB_REGISTRY_000002") +list +---- + set filename=foo settings=bar ---- create("COCKROACHDB_REGISTRY_000003") @@ -98,6 +114,10 @@ remove("COCKROACHDB_REGISTRY_000002") write("COCKROACHDB_REGISTRY_000003", <...23 bytes...>) sync("COCKROACHDB_REGISTRY_000003") +list +---- +name=foo,type=Data,settings=bar + get filename=foo ---- bar @@ -138,6 +158,9 @@ remove("COCKROACHDB_REGISTRY_000003") write("COCKROACHDB_REGISTRY_000004", <...14 bytes...>) sync("COCKROACHDB_REGISTRY_000004") +list +---- + get filename=foo ---- @@ -160,6 +183,9 @@ load ---- open-dir("") +list +---- + set filename=foo settings=helloworld ---- create("COCKROACHDB_REGISTRY_000001") @@ -179,6 +205,11 @@ set filename=bar settings=hi write("COCKROACHDB_REGISTRY_000001", <...22 bytes...>) sync("COCKROACHDB_REGISTRY_000001") +list +---- +name=bar,type=Data,settings=hi +name=foo,type=Data,settings=helloworld + close ---- write("COCKROACHDB_REGISTRY_000001", <...0 bytes...>) @@ -209,6 +240,11 @@ close("COCKROACHDB_REGISTRY_000001") stat("bar") stat("foo") +list +---- +name=bar,type=Data,settings=hi +name=foo,type=Data,settings=helloworld + get filename=bar ---- hi @@ -226,6 +262,12 @@ remove("COCKROACHDB_REGISTRY_000001") write("COCKROACHDB_REGISTRY_000002", <...25 bytes...>) sync("COCKROACHDB_REGISTRY_000002") +list +---- +name=bar,type=Data,settings=hi +name=bax,type=Data,settings=hello +name=foo,type=Data,settings=helloworld + close ---- write("COCKROACHDB_REGISTRY_000002", <...0 bytes...>)