From 23f28fa3f1225cb037fa4a0a3632c15e1857fd5d Mon Sep 17 00:00:00 2001 From: Nick Travers Date: Thu, 6 Oct 2022 11:51:48 -0700 Subject: [PATCH] ccl/cclcli: add `debug encryption-decrypt` command During storage-level L2 investigations, files from problematic stores are often requested (e.g. the MANIFEST file(s), SSTables, etc.). In cases where the store is using encryption-at-rest, the debug artifacts are useless unless they have been decrypted. Add a new debug command that can be used to decrypt a file in-situ, given the encryption spec for the store, and a path to an encrypted file in the store. For example: ```bash $ cockroach encryption-decrypt \ /path/to/store \ /path/to/encrypted/file \ /path/to/decrypted/output/file ``` Touches #89095. Release note (ops change): A new debug tool was added to allow for decrypting files in a store using encryption-at-rest. This tool is intended for use while debugging, or for providing debug artifacts to Cockroach Labs to aid with support investigations. It is intended to be run "in-situ" (i.e. on site), as it prevents having to move sensitive key material. --- pkg/ccl/baseccl/encryption_spec.go | 8 +- pkg/ccl/cliccl/BUILD.bazel | 15 +++- pkg/ccl/cliccl/debug.go | 18 +++++ pkg/ccl/cliccl/ear.go | 59 +++++++++++++++ pkg/ccl/cliccl/ear_test.go | 118 +++++++++++++++++++++++++++++ pkg/cli/gen.go | 11 ++- 6 files changed, 220 insertions(+), 9 deletions(-) create mode 100644 pkg/ccl/cliccl/ear.go create mode 100644 pkg/ccl/cliccl/ear_test.go diff --git a/pkg/ccl/baseccl/encryption_spec.go b/pkg/ccl/baseccl/encryption_spec.go index 45f794381eba..bf5aa70c951b 100644 --- a/pkg/ccl/baseccl/encryption_spec.go +++ b/pkg/ccl/baseccl/encryption_spec.go @@ -37,8 +37,8 @@ type StoreEncryptionSpec struct { RotationPeriod time.Duration } -// Convert to a serialized EncryptionOptions protobuf. -func (es StoreEncryptionSpec) toEncryptionOptions() ([]byte, error) { +// ToEncryptionOptions convert to a serialized EncryptionOptions protobuf. +func (es StoreEncryptionSpec) ToEncryptionOptions() ([]byte, error) { opts := EncryptionOptions{ KeySource: EncryptionKeySource_KeyFiles, KeyFiles: &EncryptionKeyFiles{ @@ -206,7 +206,7 @@ func PopulateStoreSpecWithEncryption( // Tell the store we absolutely need the file registry. storeSpecs.Specs[i].UseFileRegistry = true - opts, err := es.toEncryptionOptions() + opts, err := es.ToEncryptionOptions() if err != nil { return err } @@ -235,7 +235,7 @@ func EncryptionOptionsForStore( for _, es := range encryptionSpecs.Specs { if es.Path == path { - return es.toEncryptionOptions() + return es.ToEncryptionOptions() } } diff --git a/pkg/ccl/cliccl/BUILD.bazel b/pkg/ccl/cliccl/BUILD.bazel index eff3040a618e..5a88f2e73fdb 100644 --- a/pkg/ccl/cliccl/BUILD.bazel +++ b/pkg/ccl/cliccl/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "cliccl.go", "debug.go", "demo.go", + "ear.go", "start.go", ], importpath = "github.com/cockroachdb/cockroach/pkg/ccl/cliccl", @@ -36,13 +37,25 @@ go_library( go_test( name = "cliccl_test", size = "medium", - srcs = ["main_test.go"], + srcs = [ + "ear_test.go", + "main_test.go", + ], args = ["-test.timeout=295s"], + embed = [":cliccl"], deps = [ "//pkg/build", + "//pkg/ccl/baseccl", + "//pkg/ccl/storageccl/engineccl", "//pkg/ccl/utilccl", + "//pkg/cli", "//pkg/server", + "//pkg/storage", "//pkg/testutils/serverutils", + "//pkg/util/leaktest", + "//pkg/util/log", + "@com_github_spf13_cobra//:cobra", + "@com_github_stretchr_testify//require", ], ) diff --git a/pkg/ccl/cliccl/debug.go b/pkg/ccl/cliccl/debug.go index 2192ab893b10..dd346aa8f8b1 100644 --- a/pkg/ccl/cliccl/debug.go +++ b/pkg/ccl/cliccl/debug.go @@ -79,17 +79,35 @@ AES128_CTR:be235... # AES-128 encryption with store key ID RunE: clierrorplus.MaybeDecorateError(runEncryptionActiveKey), } + encryptionDecryptCmd := &cobra.Command{ + Use: "encryption-decrypt [out-file]", + Short: "decrypt a file from an encrypted store", + Long: `Decrypts a file from an encrypted store, and outputs it to the +specified path. + +If out-file is not specified, the command will output the decrypted contents to +stdout. +`, + Args: cobra.MinimumNArgs(2), + RunE: clierrorplus.MaybeDecorateError(runDecrypt), + } + // Add commands to the root debug command. // We can't add them to the lists of commands (eg: DebugCmdsForPebble) as cli init() is called before us. cli.DebugCmd.AddCommand(encryptionStatusCmd) cli.DebugCmd.AddCommand(encryptionActiveKeyCmd) + cli.DebugCmd.AddCommand(encryptionDecryptCmd) // Add the encryption flag to commands that need it. + // For the encryption-status command. f := encryptionStatusCmd.Flags() cliflagcfg.VarFlag(f, &storeEncryptionSpecs, cliflagsccl.EnterpriseEncryption) // And other flags. f.BoolVar(&encryptionStatusOpts.activeStoreIDOnly, "active-store-key-id-only", false, "print active store key ID and exit") + // For the encryption-decrypt command. + f = encryptionDecryptCmd.Flags() + cliflagcfg.VarFlag(f, &storeEncryptionSpecs, cliflagsccl.EnterpriseEncryption) // Add encryption flag to all OSS debug commands that want it. for _, cmd := range cli.DebugCommandsRequiringEncryption { diff --git a/pkg/ccl/cliccl/ear.go b/pkg/ccl/cliccl/ear.go new file mode 100644 index 000000000000..c00dc068ee6c --- /dev/null +++ b/pkg/ccl/cliccl/ear.go @@ -0,0 +1,59 @@ +// Copyright 2022 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt + +package cliccl + +import ( + "context" + "io" + "os" + + "github.com/cockroachdb/cockroach/pkg/cli" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/util/stop" + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +func runDecrypt(_ *cobra.Command, args []string) (returnErr error) { + dir, inPath := args[0], args[1] + var outPath string + if len(args) > 2 { + outPath = args[2] + } + + stopper := stop.NewStopper() + defer stopper.Stop(context.Background()) + + db, err := cli.OpenEngine(dir, stopper, storage.MustExist, storage.ReadOnly) + if err != nil { + return errors.Wrap(err, "could not open store") + } + + // Open the specified file through the FS, decrypting it. + f, err := db.Open(inPath) + if err != nil { + return errors.Wrapf(err, "could not open input file %s", inPath) + } + defer f.Close() + + // Copy the raw bytes into the destination file. + outFile := os.Stdout + if outPath != "" { + outFile, err = os.OpenFile(outPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return errors.Wrapf(err, "could not open output file %s", outPath) + } + defer outFile.Close() + } + if _, err = io.Copy(outFile, f); err != nil { + return errors.Wrapf(err, "could not write to output file") + } + + return nil +} diff --git a/pkg/ccl/cliccl/ear_test.go b/pkg/ccl/cliccl/ear_test.go new file mode 100644 index 000000000000..1fb77826e431 --- /dev/null +++ b/pkg/ccl/cliccl/ear_test.go @@ -0,0 +1,118 @@ +// Copyright 2022 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt + +package cliccl + +import ( + "bytes" + "context" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/cockroachdb/cockroach/pkg/ccl/baseccl" + // The following import is required for the hook that populates + // NewEncryptedEnvFunc in `pkg/storage`. + _ "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl" + "github.com/cockroachdb/cockroach/pkg/cli" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" +) + +func TestDecrypt(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + ctx := context.Background() + dir := t.TempDir() + + // Generate a new encryption key to use. + keyPath := filepath.Join(dir, "aes.key") + err := cli.GenEncryptionKeyCmd.RunE(nil, []string{keyPath}) + require.NoError(t, err) + + // Spin up a new encrypted store. + encSpecStr := fmt.Sprintf("path=%s,key=%s,old-key=plain", dir, keyPath) + encSpec, err := baseccl.NewStoreEncryptionSpec(encSpecStr) + require.NoError(t, err) + encOpts, err := encSpec.ToEncryptionOptions() + require.NoError(t, err) + p, err := storage.Open(ctx, storage.Filesystem(dir), storage.EncryptionAtRest(encOpts)) + require.NoError(t, err) + + // Find a manifest file to check. + files, err := p.List(dir) + require.NoError(t, err) + var manifestPath string + for _, basename := range files { + if strings.HasPrefix(basename, "MANIFEST-") { + manifestPath = filepath.Join(dir, basename) + break + } + } + // Should have found a manifest file. + require.NotEmpty(t, manifestPath) + + // Close the DB. + p.Close() + + // Pluck the `pebble manifest dump` command out of the debug command. + dumpCmd := getTool(cli.DebugPebbleCmd, []string{"pebble", "manifest", "dump"}) + require.NotNil(t, dumpCmd) + + dumpManifest := func(cmd *cobra.Command, path string) string { + var b bytes.Buffer + dumpCmd.SetOut(&b) + dumpCmd.SetErr(&b) + dumpCmd.Run(cmd, []string{path}) + return b.String() + } + out := dumpManifest(dumpCmd, manifestPath) + // Check for the presence of the comparator line in the manifest dump, as a + // litmus test for the manifest file being readable. This line should only + // appear once the file has been decrypted. + const checkStr = "comparer: cockroach_comparator" + require.NotContains(t, out, checkStr) + + // Decrypt the manifest file. + outPath := filepath.Join(dir, "manifest.plain") + decryptCmd := getTool(cli.DebugCmd, []string{"debug", "encryption-decrypt"}) + require.NotNil(t, decryptCmd) + err = decryptCmd.Flags().Set("enterprise-encryption", encSpecStr) + require.NoError(t, err) + err = decryptCmd.RunE(decryptCmd, []string{dir, manifestPath, outPath}) + require.NoError(t, err) + + // Check that the decrypted manifest file can now be read. + out = dumpManifest(dumpCmd, outPath) + require.Contains(t, out, checkStr) +} + +// getTool traverses the given cobra.Command recursively, searching for a tool +// matching the given command. +func getTool(cmd *cobra.Command, want []string) *cobra.Command { + // Base cases. + if cmd.Name() != want[0] { + return nil + } + if len(want) == 1 { + return cmd + } + // Recursive case. + for _, subCmd := range cmd.Commands() { + found := getTool(subCmd, want[1:]) + if found != nil { + return found + } + } + return nil +} diff --git a/pkg/cli/gen.go b/pkg/cli/gen.go index 379842938551..448d72f22ccf 100644 --- a/pkg/cli/gen.go +++ b/pkg/cli/gen.go @@ -139,7 +139,10 @@ func runGenAutocompleteCmd(cmd *cobra.Command, args []string) error { var aesSize int var overwriteKey bool -var genEncryptionKeyCmd = &cobra.Command{ +// GenEncryptionKeyCmd is a command to generate a store key for Encryption At +// Rest. +// Exported to allow use by CCL code. +var GenEncryptionKeyCmd = &cobra.Command{ Use: "encryption-key ", Short: "generate store key for encryption at rest", Long: `Generate store key for encryption at rest. @@ -274,7 +277,7 @@ var genCmds = []*cobra.Command{ genExamplesCmd, genHAProxyCmd, genSettingsListCmd, - genEncryptionKeyCmd, + GenEncryptionKeyCmd, } func init() { @@ -285,9 +288,9 @@ func init() { genHAProxyCmd.PersistentFlags().StringVar(&haProxyPath, "out", "haproxy.cfg", "path to generated haproxy configuration file") cliflagcfg.VarFlag(genHAProxyCmd.Flags(), &haProxyLocality, cliflags.Locality) - genEncryptionKeyCmd.PersistentFlags().IntVarP(&aesSize, "size", "s", 128, + GenEncryptionKeyCmd.PersistentFlags().IntVarP(&aesSize, "size", "s", 128, "AES key size for encryption at rest (one of: 128, 192, 256)") - genEncryptionKeyCmd.PersistentFlags().BoolVar(&overwriteKey, "overwrite", false, + GenEncryptionKeyCmd.PersistentFlags().BoolVar(&overwriteKey, "overwrite", false, "Overwrite key if it exists") genSettingsListCmd.PersistentFlags().BoolVar(&includeReservedSettings, "include-reserved", false, "include undocumented 'reserved' settings")