storage: remove obsolete encryption-at-rest code

Remove encryption-at-rest code that handles the upgrade process to the 21.2
encryption-at-rest formats and all code that reads the 21.1 and earlier
formats.

Fix #72520.

Release note: None

pkg/ccl/migrationccl/migrationsccl/BUILD.bazel

@@ -4,12 +4,10 @@ go_test(
     name = "migrationsccl_test",
     srcs = [
         "main_test.go",
-        "records_based_registry_external_test.go",
         "seed_tenant_span_configs_external_test.go",
     ],
     deps = [
         "//pkg/base",
-        "//pkg/ccl/baseccl",
         "//pkg/ccl/kvccl/kvtenantccl",
         "//pkg/clusterversion",
         "//pkg/keys",
@@ -23,7 +21,6 @@ go_test(
         "//pkg/testutils/testcluster",
         "//pkg/util/leaktest",
         "//pkg/util/log",
-        "//pkg/util/protoutil",
         "@com_github_stretchr_testify//require",
     ],
 )

pkg/ccl/storageccl/engineccl/BUILD.bazel

@@ -15,7 +15,6 @@ go_library(
         "//pkg/ccl/storageccl/engineccl/enginepbccl:enginepbccl_go_proto",
         "//pkg/storage",
         "//pkg/storage/enginepb",
-        "//pkg/storage/fs",
         "//pkg/util/log",
         "//pkg/util/protoutil",
         "//pkg/util/syncutil",
@@ -60,6 +59,7 @@ go_test(
         "@com_github_cockroachdb_errors//oserror",
         "@com_github_cockroachdb_pebble//:pebble",
         "@com_github_cockroachdb_pebble//vfs",
+        "@com_github_cockroachdb_pebble//vfs/atomicfs",
         "@com_github_gogo_protobuf//proto",
         "@com_github_kr_pretty//:pretty",
         "@com_github_stretchr_testify//require",

pkg/ccl/storageccl/engineccl/encrypted_fs.go

@@ -364,7 +364,6 @@ func newEncryptedEnv(
 			storeKM: storeKeyManager,
 			dataKM:  dataKeyManager,
 		},
-		UpgradeVersion: dataKeyManager.UseMarker,
 	}, nil
 }
 

pkg/ccl/storageccl/engineccl/pebble_key_manager.go

@@ -17,7 +17,6 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl"
-	"github.com/cockroachdb/cockroach/pkg/storage/fs"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
@@ -178,12 +177,6 @@ type DataKeyManager struct {
 		// Transitions to true when SetActiveStoreKeyInfo() is called for the
 		// first time.
 		rotationEnabled bool
-		// useMarker indicates whether or not the data key registry
-		// should write new files and use an atomic marker to mark the
-		// active file. This is set to true once the 21.2 version is
-		// finalized.
-		// TODO(jackson): Remove after v21.2.
-		useMarker bool
 		// marker is an atomic file marker used to denote which of the
 		// data keys registry files is the current one. When we rotate
 		// files, the marker is atomically moved to the new file. It's
@@ -214,23 +207,10 @@ func (m *DataKeyManager) Load(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	useMarker := filename != ""
-
-	// If the marker doesn't exist, filename is the empty string.  In
-	// this case, we fall back to looking for the file at the fixed
-	// `DATA_KEYS_REGISTRY` path.
-	if filename == "" {
-		filename = keyRegistryFilename
-		_, err = m.fs.Stat(m.fs.PathJoin(m.dbDir, filename))
-		if err != nil && !oserror.IsNotExist(err) {
-			return err
-		}
-	}
 
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	m.mu.marker = marker
-	m.mu.useMarker = useMarker
 	if oserror.IsNotExist(err) {
 		// First run.
 		m.mu.keyRegistry = makeRegistryProto()
@@ -239,21 +219,23 @@ func (m *DataKeyManager) Load(ctx context.Context) error {
 
 	// Load the existing state from the file named by `filename`.
 	m.mu.filename = filename
-	f, err := m.fs.Open(m.fs.PathJoin(m.dbDir, filename))
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	b, err := ioutil.ReadAll(f)
-	if err != nil {
-		return err
-	}
 	m.mu.keyRegistry = makeRegistryProto()
-	if err = protoutil.Unmarshal(b, m.mu.keyRegistry); err != nil {
-		return err
-	}
-	if err = validateRegistry(m.mu.keyRegistry); err != nil {
-		return err
+	if filename != "" {
+		f, err := m.fs.Open(m.fs.PathJoin(m.dbDir, filename))
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		b, err := ioutil.ReadAll(f)
+		if err != nil {
+			return err
+		}
+		if err = protoutil.Unmarshal(b, m.mu.keyRegistry); err != nil {
+			return err
+		}
+		if err = validateRegistry(m.mu.keyRegistry); err != nil {
+			return err
+		}
 	}
 	if m.mu.keyRegistry.ActiveDataKeyId != "" {
 		key, found := m.mu.keyRegistry.DataKeys[m.mu.keyRegistry.ActiveDataKeyId]
@@ -269,31 +251,6 @@ func (m *DataKeyManager) Load(ctx context.Context) error {
 	return nil
 }
 
-// UseMarker informs the data key manager that it should begin using the
-// marker file to denote which file is active.
-//
-// TODO(jackson): Remove this in 22.1. In 22.1 we can unconditionally
-// use the marker file.
-func (m *DataKeyManager) UseMarker() error {
-	if m.readOnly {
-		return nil
-	}
-
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	m.mu.useMarker = true
-
-	// If there is no filename, the data keys registry has never been
-	// written. There's no file to mark yet. The first rotation will set
-	// the marker.
-	if m.mu.filename == "" {
-		return nil
-	}
-	// NB: This may move the marker to mark the file with the previous
-	// static filename "COCKROACHDB_DATA_KEYS".
-	return m.mu.marker.Move(m.mu.filename)
-}
-
 // ActiveKey implements PebbleKeyManager.ActiveKey.
 //
 // TODO(sbhola): do rotation via a background activity instead of in this function so that we don't
@@ -473,26 +430,6 @@ func (m *DataKeyManager) rotateDataKeyAndWrite(
 		return err
 	}
 
-	if !m.mu.useMarker {
-		// If the v21.2 version hasn't been finalized yet, write the
-		// registry to the static filename `COCKROACHDB_DATA_KEYS`. If
-		// there's a crash mid-Rename, it's possible that we'll be left
-		// with a corrupt data key registry.
-		// TODO(jackson): Remove this for 22.1.
-		path := m.fs.PathJoin(m.dbDir, keyRegistryFilename)
-		if err = fs.SafeWriteToFile(m.fs, m.dbDir, path, bytes); err != nil {
-			return
-		}
-		m.mu.filename = keyRegistryFilename
-		m.mu.keyRegistry = keyRegistry
-		m.mu.activeKey = newKey
-		return err
-	}
-
-	// The v21.2 version has been finalized. Write a new file
-	// containing the updated state, and move the atomic marker to
-	// point to the new file.
-
 	// Write the current registry state to a new file and sync it.
 	// The new file's filename incorporates the marker's iteration
 	// number to ensure we're not overwriting the existing registry.

pkg/ccl/storageccl/engineccl/pebble_key_manager_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/datadriven"
 	"github.com/cockroachdb/pebble/vfs"
+	"github.com/cockroachdb/pebble/vfs/atomicfs"
 	"github.com/gogo/protobuf/proto"
 	"github.com/stretchr/testify/require"
 )
@@ -290,6 +291,10 @@ func TestDataKeyManager(t *testing.T) {
 						return err.Error()
 					}
 					writeToFile(t, memFS, memFS.PathJoin(data[0], keyRegistryFilename), b)
+					marker, _, err := atomicfs.LocateMarker(memFS, data[0], keysRegistryMarkerName)
+					require.NoError(t, err)
+					require.NoError(t, marker.Move(keyRegistryFilename))
+					require.NoError(t, marker.Close())
 				}
 				return ""
 			case "load":
@@ -446,9 +451,6 @@ func TestDataKeyManagerIO(t *testing.T) {
 				d.ScanArgs(t, "id", &id)
 				fmt.Fprintf(&buf, "%s", setActiveStoreKey(dkm, id, enginepbccl.EncryptionType_AES128_CTR))
 				return buf.String()
-			case "use-marker":
-				appendError(dkm.UseMarker())
-				return buf.String()
 			default:
 				return fmt.Sprintf("unknown command: %s\n", d.Cmd)
 			}