Merge cockroachdb#59369

59369: roachtest: add cold data to tpcc/mixed-headroom r=irfansharif a=tbg

This test mainly targets the version upgrade mechanism, which may do
work proportional to the amount of data in the cluster. By adding a
large dataset, we add an opportunity to shake out inefficiencies or
bugs.

Closes cockroachdb#58455.

Release note: None


Co-authored-by: Tobias Grieger <[email protected]>

pkg/cmd/roachtest/mixed_version_jobs.go

@@ -58,7 +58,7 @@ func (s *backgroundStepper) launch(ctx context.Context, _ *test, u *versionUpgra
 	})
 }
 
-func (s *backgroundStepper) stop(ctx context.Context, t *test, u *versionUpgradeTest) {
+func (s *backgroundStepper) wait(ctx context.Context, t *test, u *versionUpgradeTest) {
 	s.m.cancel()
 	// We don't care about the workload failing since we only use it to produce a
 	// few `RESTORE` jobs. And indeed workload will fail because it does not
@@ -312,7 +312,7 @@ func runJobsMixedVersions(
 		allowAutoUpgradeStep(1),
 		waitForUpgradeStep(roachNodes),
 		resumeAllJobsAndWaitStep,
-		backgroundTPCC.stop,
+		backgroundTPCC.wait,
 		checkForFailedJobsStep,
 	)
 	u.run(ctx, t)

pkg/cmd/roachtest/tpcc.go

@@ -244,16 +244,15 @@ func registerTPCC(r *testRegistry) {
 		Tags:    []string{`default`},
 		Cluster: mixedHeadroomSpec,
 		Run: func(ctx context.Context, t *test, c *cluster) {
+			crdbNodes := c.Range(1, 4)
+			workloadNode := c.Node(5)
+
 			maxWarehouses := maxSupportedTPCCWarehouses(r.buildVersion, cloud, t.spec.Cluster)
 			headroomWarehouses := int(float64(maxWarehouses) * 0.7)
-			oldV, err := PredecessorVersion(r.buildVersion)
-			if err != nil {
-				t.Fatal(err)
+			if local {
+				headroomWarehouses = 10
 			}
 
-			crdbNodes := c.Range(1, 4)
-			workloadNode := c.Node(5)
-
 			// We'll need this below.
 			tpccBackgroundStepper := backgroundStepper{
 				nodes: crdbNodes,
@@ -275,6 +274,20 @@ func registerTPCC(r *testRegistry) {
 				n1         = 1
 			)
 
+			// NB: this results in ~100GB of (actual) disk usage per node once things
+			// have settled down, and ~7.5k ranges. The import takes ~40 minutes.
+			// The full 6.5m import ran into out of disk errors (on 250gb machines),
+			// hence division by two.
+			bankRows := 65104166 / 2
+			if local {
+				bankRows = 1000
+			}
+
+			oldV, err := PredecessorVersion(r.buildVersion)
+			if err != nil {
+				t.Fatal(err)
+			}
+
 			newVersionUpgradeTest(c,
 				uploadAndStartFromCheckpointFixture(crdbNodes, oldV),
 				waitForUpgradeStep(crdbNodes), // let predecessor version settle (gossip etc)
@@ -283,6 +296,10 @@ func registerTPCC(r *testRegistry) {
 				// version to load some data and hopefully create some state that will
 				// need work by long-running migrations.
 				importTPCCStep(oldV, headroomWarehouses, crdbNodes),
+				// Add a lot of cold data to this cluster. This further stresses the version
+				// upgrade machinery, in which a) all ranges are touched and b) work proportional
+				// to the amount data may be carried out.
+				importLargeBankStep(oldV, bankRows, crdbNodes),
 				// Upload and restart cluster into the new
 				// binary (stays at old cluster version).
 				binaryUpgradeStep(crdbNodes, mainBinary),
@@ -297,7 +314,7 @@ func registerTPCC(r *testRegistry) {
 				setClusterSettingVersionStep,
 				// Wait until TPCC background run terminates
 				// and fail if it reports an error.
-				tpccBackgroundStepper.stop,
+				tpccBackgroundStepper.wait,
 			).run(ctx, t)
 		},
 	})

pkg/cmd/roachtest/versionupgrade.go

@@ -569,3 +569,23 @@ func importTPCCStep(oldV string, headroomWarehouses int, crdbNodes nodeListOptio
 		m.Wait()
 	}
 }
+
+func importLargeBankStep(oldV string, rows int, crdbNodes nodeListOption) versionStep {
+	return func(ctx context.Context, t *test, u *versionUpgradeTest) {
+		// Use the predecessor binary to load into the predecessor
+		// cluster to avoid random breakage due to flag changes, etc.
+		binary := "./cockroach"
+		if oldV != "" {
+			binary = filepath.Join("v"+oldV, "cockroach")
+		}
+
+		// Use a monitor so that we fail cleanly if the cluster crashes
+		// during import.
+		m := newMonitor(ctx, u.c, crdbNodes)
+		m.Go(func(ctx context.Context) error {
+			return u.c.RunE(ctx, u.c.Node(crdbNodes[0]), binary, "workload", "fixtures", "import", "bank",
+				"--payload-bytes=10240", "--rows="+fmt.Sprint(rows), "--seed=4", "--db=bigbank")
+		})
+		m.Wait()
+	}
+}