Merge #58622

58622: roachtest: Enable encryption-at-rest in many storage-heavy non-bench tests r=sumeerbhola a=itsbilal


Currently, encryption-at-rest is only used in roachtests that either have
`enc=true`, `encryption` or `encrypted` in their name. In addition, the other
roachtest to use encryption-at-rest is `clearrange/*`, and only on some random
runs.

This change updates many more roachtests to use encryption-at-rest on about
half of all runs (chosen by a random var):
 * `backup/2TB/*`
 * `acceptance/many-splits`
 * `import/tpc{c,h}/*`
 * `tpcc/*, tpcc-nowait/*, schemachange/*tpcc*, scrub/*tpcc*` (NOT tpccbench/*)
 * `restore2TB/*`

Fixes #57997.

Release note: None

Co-authored-by: Bilal Akhtar <[email protected]>

pkg/cmd/roachtest/backup.go

@@ -37,6 +37,8 @@ const (
 func registerBackup(r *testRegistry) {
 	importBankData := func(ctx context.Context, rows int, t *test, c *cluster) string {
 		dest := c.name
+		// Randomize starting with encryption-at-rest enabled.
+		c.encryptAtRandom = true
 
 		if local {
 			rows = 100
@@ -205,6 +207,8 @@ func registerBackup(r *testRegistry) {
 		Cluster: makeClusterSpec(3),
 		Timeout: 1 * time.Hour,
 		Run: func(ctx context.Context, t *test, c *cluster) {
+			// Randomize starting with encryption-at-rest enabled.
+			c.encryptAtRandom = true
 			c.Put(ctx, cockroach, "./cockroach")
 			c.Put(ctx, workload, "./workload")
 			c.Start(ctx, t)

pkg/cmd/roachtest/clearrange.go

@@ -13,7 +13,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"math/rand"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
@@ -39,17 +38,12 @@ func registerClearRange(r *testRegistry) {
 }
 
 func runClearRange(ctx context.Context, t *test, c *cluster, aggressiveChecks bool) {
+	// Randomize starting with encryption-at-rest enabled.
+	c.encryptAtRandom = true
 	c.Put(ctx, cockroach, "./cockroach")
 
 	t.Status("restoring fixture")
-	// Randomize starting with encryption-at-rest enabled.
-	rng := rand.New(rand.NewSource(timeutil.Now().UnixNano()))
-	var opts []option
-	if rng.Intn(2) == 1 {
-		c.l.Printf("starting with encryption at rest enabled")
-		opts = append(opts, startArgs("--encrypt"))
-	}
-	c.Start(ctx, t, opts...)
+	c.Start(ctx, t)
 
 	// NB: on a 10 node cluster, this should take well below 3h.
 	tBegin := timeutil.Now()
@@ -59,6 +53,7 @@ func runClearRange(ctx context.Context, t *test, c *cluster, aggressiveChecks bo
 	c.Stop(ctx)
 	t.Status()
 
+	var opts []option
 	if aggressiveChecks {
 		// Run with an env var that runs a synchronous consistency check after each rebalance and merge.
 		// This slows down merges, so it might hide some races.

pkg/cmd/roachtest/cluster.go

@@ -1085,6 +1085,11 @@ type cluster struct {
 	// at rest enabled. The default only applies if encryption is not explicitly
 	// enabled or disabled by options passed to Start.
 	encryptDefault bool
+	// encryptAtRandom is true if the cluster should enable encryption-at-rest
+	// on about half of all runs. Only valid if encryptDefault is false. Only
+	// applies if encryption is not explicitly enabled or disabled by options
+	// passed to Start. For use in roachtests.
+	encryptAtRandom bool
 
 	// destroyState contains state related to the cluster's destruction.
 	destroyState destroyState
@@ -2132,8 +2137,16 @@ func (c *cluster) StartE(ctx context.Context, opts ...option) error {
 	}
 	args = append(args, roachprodArgs(opts)...)
 	args = append(args, c.makeNodes(opts...))
-	if !argExists(args, "--encrypt") && c.encryptDefault {
-		args = append(args, "--encrypt")
+	if !argExists(args, "--encrypt") {
+		if c.encryptDefault {
+			args = append(args, "--encrypt")
+		} else if c.encryptAtRandom {
+			rng := rand.New(rand.NewSource(timeutil.Now().UnixNano()))
+			if rng.Intn(2) == 1 {
+				c.l.Printf("starting with encryption at rest enabled")
+				args = append(args, "--encrypt")
+			}
+		}
 	}
 	return execCmd(ctx, c.l, args...)
 }

pkg/cmd/roachtest/import.go

@@ -22,6 +22,8 @@ import (
 
 func registerImportTPCC(r *testRegistry) {
 	runImportTPCC := func(ctx context.Context, t *test, c *cluster, warehouses int) {
+		// Randomize starting with encryption-at-rest enabled.
+		c.encryptAtRandom = true
 		c.Put(ctx, cockroach, "./cockroach")
 		c.Put(ctx, workload, "./workload")
 		t.Status("starting csv servers")
@@ -95,6 +97,8 @@ func registerImportTPCH(r *testRegistry) {
 			Cluster: makeClusterSpec(item.nodes),
 			Timeout: item.timeout,
 			Run: func(ctx context.Context, t *test, c *cluster) {
+				// Randomize starting with encryption-at-rest enabled.
+				c.encryptAtRandom = true
 				c.Put(ctx, cockroach, "./cockroach")
 				c.Start(ctx, t)
 				conn := c.Conn(ctx, 1)

pkg/cmd/roachtest/many_splits.go

@@ -18,6 +18,8 @@ import (
 // runManySplits attempts to create 2000 tiny ranges on a 4-node cluster using
 // left-to-right splits and check the cluster is still live afterwards.
 func runManySplits(ctx context.Context, t *test, c *cluster) {
+	// Randomize starting with encryption-at-rest enabled.
+	c.encryptAtRandom = true
 	args := startArgs("--env=COCKROACH_SCAN_MAX_IDLE_TIME=5ms")
 	c.Put(ctx, cockroach, "./cockroach")
 	c.Start(ctx, t, args)

pkg/cmd/roachtest/restore.go

@@ -223,6 +223,8 @@ func registerRestore(r *testRegistry) {
 			Cluster: makeClusterSpec(item.nodes),
 			Timeout: item.timeout,
 			Run: func(ctx context.Context, t *test, c *cluster) {
+				// Randomize starting with encryption-at-rest enabled.
+				c.encryptAtRandom = true
 				c.Put(ctx, cockroach, "./cockroach")
 				c.Start(ctx, t)
 				m := newMonitor(ctx, c)

pkg/cmd/roachtest/tpcc.go

@@ -77,6 +77,8 @@ func tpccImportCmd(warehouses int, extraArgs ...string) string {
 func setupTPCC(
 	ctx context.Context, t *test, c *cluster, opts tpccOptions,
 ) (crdbNodes, workloadNode nodeListOption) {
+	// Randomize starting with encryption-at-rest enabled.
+	c.encryptAtRandom = true
 	crdbNodes = c.Range(1, c.spec.NodeCount-1)
 	workloadNode = c.Node(c.spec.NodeCount)
 	if c.isLocal() {
@@ -110,7 +112,7 @@ func setupTPCC(
 	func() {
 		db := c.Conn(ctx, 1)
 		defer db.Close()
-		c.Start(ctx, t, crdbNodes, startArgsDontEncrypt)
+		c.Start(ctx, t, crdbNodes)
 		waitForFullReplication(t, c.Conn(ctx, crdbNodes[0]))
 		switch opts.SetupType {
 		case usingImport: