roachtest: enable metamorphic expiration leases in most tests

This patch uses metamorphic expiration leases in most non-benchmark
roachtests. There will likely be some fallout from this, in particular
for timing/latency-sensitive tests, but we can deal with that as it
happens.

Epic: none
Release note: None

pkg/cmd/roachtest/tests/acceptance.go

@@ -27,6 +27,7 @@ func registerAcceptance(r registry.Registry) {
 		numNodes          int
 		timeout           time.Duration
 		encryptionSupport registry.EncryptionSupport
+		defaultLeases     bool
 	}{
 		registry.OwnerKV: {
 			{name: "decommission-self", fn: runDecommissionSelf},
@@ -63,9 +64,10 @@ func registerAcceptance(r registry.Registry) {
 		},
 		registry.OwnerTestEng: {
 			{
-				name:    "version-upgrade",
-				fn:      runVersionUpgrade,
-				timeout: 30 * time.Minute,
+				name:          "version-upgrade",
+				fn:            runVersionUpgrade,
+				timeout:       30 * time.Minute,
+				defaultLeases: true,
 			},
 		},
 		registry.OwnerDisasterRecovery: {
@@ -105,6 +107,9 @@ func registerAcceptance(r registry.Registry) {
 				spec.Timeout = tc.timeout
 			}
 			spec.EncryptionSupport = tc.encryptionSupport
+			if !tc.defaultLeases {
+				spec.Leases = registry.MetamorphicLeases
+			}
 			spec.Run = func(ctx context.Context, t test.Test, c cluster.Cluster) {
 				tc.fn(ctx, t, c)
 			}

pkg/cmd/roachtest/tests/activerecord.go

@@ -249,5 +249,6 @@ func registerActiveRecord(r registry.Registry) {
 		NativeLibs: registry.LibGEOS,
 		Tags:       registry.Tags(`default`, `orm`),
 		Run:        runActiveRecord,
+		Leases:     registry.MetamorphicLeases,
 	})
 }

pkg/cmd/roachtest/tests/admission_control_elastic_backup.go

@@ -41,6 +41,7 @@ func registerElasticControlForBackups(r registry.Registry) {
 		Owner:   registry.OwnerAdmissionControl,
 		Tags:    registry.Tags(`weekly`),
 		Cluster: r.MakeClusterSpec(4, spec.CPU(8)),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			if c.Spec().NodeCount < 4 {
 				t.Fatalf("expected at least 4 nodes, found %d", c.Spec().NodeCount)

pkg/cmd/roachtest/tests/admission_control_elastic_cdc.go

@@ -35,6 +35,7 @@ func registerElasticControlForCDC(r registry.Registry) {
 		Tags:            registry.Tags(`weekly`),
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(8)),
 		RequiresLicense: true,
+		Leases:          registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			if c.Spec().NodeCount < 4 {
 				t.Fatalf("expected at least 4 nodes, found %d", c.Spec().NodeCount)

pkg/cmd/roachtest/tests/admission_control_index_overload.go

@@ -37,6 +37,7 @@ func registerIndexOverload(r registry.Registry) {
 		Owner:   registry.OwnerAdmissionControl,
 		Tags:    registry.Tags("weekly"),
 		Cluster: r.MakeClusterSpec(4, spec.CPU(8)),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			crdbNodes := c.Spec().NodeCount - 1
 			workloadNode := c.Spec().NodeCount

pkg/cmd/roachtest/tests/admission_control_multi_store_overload.go

@@ -96,6 +96,7 @@ func registerMultiStoreOverload(r registry.Registry) {
 		Owner:   registry.OwnerAdmissionControl,
 		Tags:    registry.Tags(`weekly`),
 		Cluster: r.MakeClusterSpec(2, spec.CPU(8), spec.SSD(2)),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runKV(ctx, t, c)
 		},

pkg/cmd/roachtest/tests/admission_control_multitenant_fairness.go

@@ -90,6 +90,7 @@ func registerMultiTenantFairness(r registry.Registry) {
 			Name:              fmt.Sprintf("admission-control/multitenant-fairness/%s", s.name),
 			Cluster:           r.MakeClusterSpec(5),
 			Owner:             registry.OwnerAdmissionControl,
+			Leases:            registry.MetamorphicLeases,
 			NonReleaseBlocker: false,
 			Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 				runMultiTenantFairness(ctx, t, c, s)

pkg/cmd/roachtest/tests/admission_control_snapshot_overload.go

@@ -43,6 +43,7 @@ func registerSnapshotOverload(r registry.Registry) {
 		Owner:   registry.OwnerAdmissionControl,
 		Tags:    registry.Tags(`weekly`),
 		Cluster: r.MakeClusterSpec(4, spec.CPU(8)),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			if c.Spec().NodeCount < 4 {
 				t.Fatalf("expected at least 4 nodes, found %d", c.Spec().NodeCount)

pkg/cmd/roachtest/tests/admission_control_tpcc_overload.go

@@ -168,6 +168,7 @@ func registerTPCCOverload(r registry.Registry) {
 			Cluster:           r.MakeClusterSpec(s.Nodes+1, spec.CPU(s.CPUs)),
 			Run:               s.run,
 			EncryptionSupport: registry.EncryptionMetamorphic,
+			Leases:            registry.MetamorphicLeases,
 			Timeout:           20 * time.Minute,
 		})
 	}

pkg/cmd/roachtest/tests/allocator.go

@@ -151,6 +151,7 @@ func registerAllocator(r registry.Registry) {
 		Name:    `replicate/up/1to3`,
 		Owner:   registry.OwnerKV,
 		Cluster: r.MakeClusterSpec(4),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runAllocator(ctx, t, c, 1, 10.0)
 		},
@@ -159,6 +160,7 @@ func registerAllocator(r registry.Registry) {
 		Name:    `replicate/rebalance/3to5`,
 		Owner:   registry.OwnerKV,
 		Cluster: r.MakeClusterSpec(6),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runAllocator(ctx, t, c, 3, 42.0)
 		},
@@ -168,6 +170,7 @@ func registerAllocator(r registry.Registry) {
 		Owner:   registry.OwnerKV,
 		Timeout: 10 * time.Minute,
 		Cluster: r.MakeClusterSpec(9, spec.CPU(1)),
+		Leases:  registry.MetamorphicLeases,
 		Run:     runWideReplication,
 	})
 }

pkg/cmd/roachtest/tests/alterpk.go

@@ -183,6 +183,7 @@ func registerAlterPK(r registry.Registry) {
 		// Use a 4 node cluster -- 3 nodes will run cockroach, and the last will be the
 		// workload driver node.
 		Cluster: r.MakeClusterSpec(4),
+		Leases:  registry.MetamorphicLeases,
 		Run:     runAlterPKBank,
 	})
 	r.Add(registry.TestSpec{
@@ -191,6 +192,7 @@ func registerAlterPK(r registry.Registry) {
 		// Use a 4 node cluster -- 3 nodes will run cockroach, and the last will be the
 		// workload driver node.
 		Cluster: r.MakeClusterSpec(4, spec.CPU(32)),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runAlterPKTPCC(ctx, t, c, 250 /* warehouses */, true /* expensiveChecks */)
 		},
@@ -201,6 +203,7 @@ func registerAlterPK(r registry.Registry) {
 		// Use a 4 node cluster -- 3 nodes will run cockroach, and the last will be the
 		// workload driver node.
 		Cluster: r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runAlterPKTPCC(ctx, t, c, 500 /* warehouses */, false /* expensiveChecks */)
 		},

pkg/cmd/roachtest/tests/asyncpg.go

@@ -142,6 +142,7 @@ func registerAsyncpg(r registry.Registry) {
 		Owner:   registry.OwnerSQLSessions,
 		Cluster: r.MakeClusterSpec(1, spec.CPU(16)),
 		Tags:    registry.Tags(`default`, `orm`),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runAsyncpg(ctx, t, c)
 		},

pkg/cmd/roachtest/tests/awsdms.go

@@ -191,6 +191,7 @@ func registerAWSDMS(r registry.Registry) {
 		Name:    "awsdms",
 		Owner:   registry.OwnerSQLSessions, // TODO(otan): add a migrations OWNERS team
 		Cluster: r.MakeClusterSpec(1),
+		Leases:  registry.MetamorphicLeases,
 		Tags:    registry.Tags(`default`, `awsdms`, `aws`),
 		Run:     runAWSDMS,
 	})

pkg/cmd/roachtest/tests/backup.go

@@ -216,6 +216,7 @@ func registerBackupNodeShutdown(r registry.Registry) {
 		Owner:             registry.OwnerDisasterRecovery,
 		Cluster:           backupNodeRestartSpec,
 		EncryptionSupport: registry.EncryptionMetamorphic,
+		Leases:            registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			gatewayNode := 2
 			nodeToShutdown := 3
@@ -237,6 +238,7 @@ func registerBackupNodeShutdown(r registry.Registry) {
 		Owner:             registry.OwnerDisasterRecovery,
 		Cluster:           backupNodeRestartSpec,
 		EncryptionSupport: registry.EncryptionMetamorphic,
+		Leases:            registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			gatewayNode := 2
 			nodeToShutdown := 2
@@ -432,6 +434,7 @@ func registerBackup(r registry.Registry) {
 			Owner:             registry.OwnerDisasterRecovery,
 			Cluster:           r.MakeClusterSpec(3),
 			EncryptionSupport: registry.EncryptionMetamorphic,
+			Leases:            registry.MetamorphicLeases,
 			Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 				if c.Spec().Cloud != item.machine {
 					t.Skip("backup assumeRole is only configured to run on "+item.machine, "")
@@ -538,6 +541,7 @@ func registerBackup(r registry.Registry) {
 			Owner:             registry.OwnerDisasterRecovery,
 			Cluster:           KMSSpec,
 			EncryptionSupport: registry.EncryptionMetamorphic,
+			Leases:            registry.MetamorphicLeases,
 			Tags:              item.tags,
 			Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 				if c.Spec().Cloud != item.machine {
@@ -671,6 +675,7 @@ func registerBackup(r registry.Registry) {
 		Name:              `backupTPCC`,
 		Owner:             registry.OwnerDisasterRecovery,
 		Cluster:           r.MakeClusterSpec(3),
+		Leases:            registry.MetamorphicLeases,
 		Timeout:           1 * time.Hour,
 		EncryptionSupport: registry.EncryptionMetamorphic,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
@@ -877,6 +882,7 @@ func registerBackup(r registry.Registry) {
 		Owner:             registry.OwnerDisasterRecovery,
 		Timeout:           4 * time.Hour,
 		Cluster:           r.MakeClusterSpec(3, spec.CPU(8)),
+		Leases:            registry.MetamorphicLeases,
 		EncryptionSupport: registry.EncryptionMetamorphic,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runBackupMVCCRangeTombstones(ctx, t, c, mvccRangeTombstoneConfig{})

pkg/cmd/roachtest/tests/cancel.go

@@ -139,6 +139,7 @@ func registerCancel(r registry.Registry) {
 		Name:    fmt.Sprintf("cancel/tpch/distsql/queries=%s,nodes=%d", queries, numNodes),
 		Owner:   registry.OwnerSQLQueries,
 		Cluster: r.MakeClusterSpec(numNodes),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runCancel(ctx, t, c, tpchQueriesToRun, true /* useDistsql */)
 		},
@@ -148,6 +149,7 @@ func registerCancel(r registry.Registry) {
 		Name:    fmt.Sprintf("cancel/tpch/local/queries=%s,nodes=%d", queries, numNodes),
 		Owner:   registry.OwnerSQLQueries,
 		Cluster: r.MakeClusterSpec(numNodes),
+		Leases:  registry.MetamorphicLeases,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runCancel(ctx, t, c, tpchQueriesToRun, false /* useDistsql */)
 		},

pkg/cmd/roachtest/tests/cdc.go

@@ -966,6 +966,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/tpcc-1000",
 		Owner:           registry.OwnerCDC,
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -989,6 +990,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/tpcc-1000/sink=null",
 		Owner:           registry.OwnerCDC,
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		Tags:            registry.Tags("manual"),
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
@@ -1013,6 +1015,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/initial-scan",
 		Owner:           registry.OwnerCDC,
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -1032,6 +1035,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/sink-chaos",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -1056,6 +1060,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/crdb-chaos",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -1085,6 +1090,7 @@ func registerCDC(r registry.Registry) {
 		// but this cannot be allocated without some sort of configuration outside
 		// of this test. Look into it.
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -1116,6 +1122,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/cloud-sink-gcs/rangefeed=true",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -1142,6 +1149,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/pubsub-sink",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -1177,6 +1185,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/pubsub-sink/assume-role",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -1209,6 +1218,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/cloud-sink-gcs/assume-role",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -1232,6 +1242,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/webhook-sink",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(4, spec.CPU(16)),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -1264,6 +1275,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/kafka-auth",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(1),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runCDCKafkaAuth(ctx, t, c)
@@ -1273,6 +1285,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/kafka-oauth",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(4),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			ct := newCDCTester(ctx, t, c)
@@ -1307,6 +1320,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/bank",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(4),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Timeout:         30 * time.Minute,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
@@ -1317,6 +1331,7 @@ func registerCDC(r registry.Registry) {
 		Name:            "cdc/schemareg",
 		Owner:           `cdc`,
 		Cluster:         r.MakeClusterSpec(1),
+		Leases:          registry.MetamorphicLeases,
 		RequiresLicense: true,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runCDCSchemaRegistry(ctx, t, c)

pkg/cmd/roachtest/tests/clearrange.go

@@ -37,6 +37,7 @@ func registerClearRange(r registry.Registry) {
 				// to <3:30h but it varies.
 				Timeout: 5*time.Hour + 90*time.Minute,
 				Cluster: r.MakeClusterSpec(10, spec.CPU(16)),
+				Leases:  registry.MetamorphicLeases,
 				Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 					runClearRange(ctx, t, c, checks, rangeTombstones)
 				},
@@ -54,6 +55,7 @@ func registerClearRange(r registry.Registry) {
 				Timeout:           5*time.Hour + 120*time.Minute,
 				Cluster:           r.MakeClusterSpec(10, spec.CPU(16), spec.SetFileSystem(spec.Zfs)),
 				EncryptionSupport: registry.EncryptionMetamorphic,
+				Leases:            registry.MetamorphicLeases,
 				Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 					runClearRange(ctx, t, c, checks, rangeTombstones)
 				},

pkg/cmd/roachtest/tests/clock_jump_crash.go

@@ -139,6 +139,7 @@ func registerClockJumpTests(r registry.Registry) {
 			// These tests muck with NTP, therefore we don't want the cluster reused
 			// by others.
 			Cluster: r.MakeClusterSpec(1, spec.ReuseTagged("offset-injector"), spec.TerminateOnMigration()),
+			Leases:  registry.MetamorphicLeases,
 			Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 				runClockJump(ctx, t, c, tc)
 			},

pkg/cmd/roachtest/tests/clock_monotonic.go

@@ -141,6 +141,7 @@ func registerClockMonotonicTests(r registry.Registry) {
 			// These tests muck with NTP, therefor we don't want the cluster reused by
 			// others.
 			Cluster: r.MakeClusterSpec(1, spec.ReuseTagged("offset-injector"), spec.TerminateOnMigration()),
+			Leases:  registry.MetamorphicLeases,
 			Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 				runClockMonotonicity(ctx, t, c, tc)
 			},

pkg/cmd/roachtest/tests/cluster_to_cluster.go

@@ -714,6 +714,7 @@ func c2cRegisterWrapper(
 		Name:            sp.name,
 		Owner:           registry.OwnerDisasterRecovery,
 		Cluster:         r.MakeClusterSpec(sp.dstNodes+sp.srcNodes+1, clusterOps...),
+		Leases:          registry.MetamorphicLeases,
 		Timeout:         sp.timeout,
 		Skip:            sp.skip,
 		Tags:            sp.tags,

pkg/cmd/roachtest/tests/copy.go

@@ -182,6 +182,7 @@ func registerCopy(r registry.Registry) {
 			Name:    fmt.Sprintf("copy/bank/rows=%d,nodes=%d,txn=%t", tc.rows, tc.nodes, tc.txn),
 			Owner:   registry.OwnerKV,
 			Cluster: r.MakeClusterSpec(tc.nodes),
+			Leases:  registry.MetamorphicLeases,
 			Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 				runCopy(ctx, t, c, tc.rows, tc.txn)
 			},