roachtest: introduce multi tenant TPCH test

This commit introduces a new roachtest that runs TPCH benchmark on
a single node in a single-tenant deployment first followed by another
run in a multi-tenant deployment with a single SQL instance. It
refactors the tpchvec roachtest a bit to extract a couple of helper
methods for performing the latency analysis. In particular, it removes
`queriesToRun` parameter since all variants now run all 22 queries.

I ran into some difficulties around managing the certificates in
different deployment models, so the test is structured that first the
single-node cluster is used in a single-tenant deployment model, and
then - after running all 22 queries - a tenant with a single SQL
instance is created and all queries are run within the tenant.

Here is the comparison of averages over 10 runs of single-tenant vs
multi-tenant:
```
Q1: 6.34s	13.35s	110.39%
Q2: 0.22s	0.49s	122.07%
Q3: 4.88s	8.04s	64.67%
Q4: 1.51s	4.75s	213.67%
Q5: 2.33s	11.69s	400.90%
Q6: 5.51s	35.49s	543.89%
Q7: 5.75s	24.08s	318.42%
Q8: 0.85s	3.82s	348.47%
Q9: 7.34s	28.37s	286.25%
Q10: 1.99s	5.00s	150.93%
Q11: 0.55s	1.92s	249.00%
Q12: 6.02s	34.76s	477.05%
Q13: 1.88s	3.76s	100.00%
Q14: 0.64s	1.10s	73.11%
Q15: 3.33s	16.80s	404.23%
Q16: 0.88s	1.29s	47.66%
Q17: 0.24s	0.60s	145.49%
Q18: 7.75s	30.13s	288.48%
Q19: 5.20s	13.08s	151.69%
Q20: 12.66s	55.30s	336.92%
Q21: 6.98s	24.50s	250.77%
Q22: 0.62s	1.17s	90.26%
```

Release justification: test-only change.

Release note: None

pkg/cmd/roachtest/tests/BUILD.bazel

@@ -79,6 +79,7 @@ go_library(
         "mixed_version_schemachange.go",
         "multitenant.go",
         "multitenant_fairness.go",
+        "multitenant_tpch.go",
         "multitenant_upgrade.go",
         "multitenant_utils.go",
         "network.go",

pkg/cmd/roachtest/tests/cancel.go

@@ -48,16 +48,16 @@ func registerCancel(r registry.Registry) {
 
 		m := c.NewMonitor(ctx, c.All())
 		m.Go(func(ctx context.Context) error {
+			conn := c.Conn(ctx, t.L(), 1)
+			defer conn.Close()
+
 			t.Status("restoring TPCH dataset for Scale Factor 1")
 			if err := loadTPCHDataset(
-				ctx, t, c, 1 /* sf */, c.NewMonitor(ctx), c.All(), false, /* disableMergeQueue */
+				ctx, t, c, conn, 1 /* sf */, c.NewMonitor(ctx), c.All(), false, /* disableMergeQueue */
 			); err != nil {
 				t.Fatal(err)
 			}
 
-			conn := c.Conn(ctx, t.L(), 1)
-			defer conn.Close()
-
 			queryPrefix := "USE tpch; "
 			if !useDistsql {
 				queryPrefix += "SET distsql = off; "

pkg/cmd/roachtest/tests/multitenant_tpch.go

@@ -0,0 +1,102 @@
+// Copyright 2022 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package tests
+
+import (
+	"context"
+	gosql "database/sql"
+	"fmt"
+
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
+	"github.com/cockroachdb/cockroach/pkg/workload/tpch"
+)
+
+// runMultiTenantTPCH runs TPCH queries on a cluster that is first used as a
+// single-tenant deployment followed by a run of all queries in a multi-tenant
+// deployment with a single SQL instance.
+func runMultiTenantTPCH(ctx context.Context, t test.Test, c cluster.Cluster) {
+	c.Put(ctx, t.Cockroach(), "./cockroach", c.All())
+	c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(1))
+	c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(install.SecureOption(true)), c.All())
+
+	setupNames := []string{"single-tenant", "multi-tenant"}
+	const numRunsPerQuery = 3
+	perfHelper := newTpchVecPerfHelper(setupNames)
+
+	// runTPCH runs all TPCH queries on a single setup. It first restores the
+	// TPCH dataset using the provided connection and then runs each TPCH query
+	// one at a time (using the given url as a parameter to the 'workload run'
+	// command). The runtimes are accumulated in the perf helper.
+	runTPCH := func(conn *gosql.DB, url string, setupIdx int) {
+		t.Status("restoring TPCH dataset for Scale Factor 1 in %s", setupNames[setupIdx])
+		if err := loadTPCHDataset(
+			ctx, t, c, conn, 1 /* sf */, c.NewMonitor(ctx), c.All(), false, /* disableMergeQueue */
+		); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := conn.Exec("USE tpch;"); err != nil {
+			t.Fatal(err)
+		}
+		createStatsFromTables(t, conn, tpchTables)
+		for queryNum := 1; queryNum <= tpch.NumQueries; queryNum++ {
+			cmd := fmt.Sprintf("./workload run tpch %s --secure "+
+				"--concurrency=1 --db=tpch --max-ops=%d --queries=%d",
+				url, numRunsPerQuery, queryNum)
+			result, err := c.RunWithDetailsSingleNode(ctx, t.L(), c.Node(1), cmd)
+			workloadOutput := result.Stdout + result.Stderr
+			t.L().Printf(workloadOutput)
+			if err != nil {
+				t.Fatal(err)
+			}
+			perfHelper.parseQueryOutput(t, []byte(workloadOutput), setupIdx)
+		}
+	}
+
+	// First, use the cluster as a single tenant deployment. It is important to
+	// not create the tenant yet so that the certs directory is not overwritten.
+	singleTenantConn := c.Conn(ctx, t.L(), 1)
+	runTPCH(singleTenantConn, "" /* url */, 0 /* setupIdx */)
+
+	// Now we create a tenant and run all TPCH queries within it.
+	const (
+		tenantID       = 123
+		tenantHTTPPort = 8081
+		tenantSQLPort  = 30258
+		tenantNode     = 1
+	)
+	_, err := singleTenantConn.Exec(`SELECT crdb_internal.create_tenant($1)`, tenantID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tenant := createTenantNode(ctx, t, c, c.All(), tenantID, tenantNode, tenantHTTPPort, tenantSQLPort)
+	tenant.start(ctx, t, c, "./cockroach")
+	multiTenantConn, err := gosql.Open("postgres", tenant.pgURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	runTPCH(multiTenantConn, "'"+tenant.secureURL()+"'", 1 /* setupIdx */)
+
+	// Analyze the runtimes of both setups.
+	perfHelper.compareSetups(t, numRunsPerQuery, nil /* timesCallback */)
+}
+
+func registerMultiTenantTPCH(r registry.Registry) {
+	r.Add(registry.TestSpec{
+		Name:    "multitenant/tpch",
+		Owner:   registry.OwnerSQLQueries,
+		Cluster: r.MakeClusterSpec(1 /* nodeCount */),
+		Run:     runMultiTenantTPCH,
+	})
+}

pkg/cmd/roachtest/tests/registry.go

@@ -78,6 +78,7 @@ func RegisterTests(r registry.Registry) {
 	registerLiquibase(r)
 	registerLoadSplits(r)
 	registerMultiTenantFairness(r)
+	registerMultiTenantTPCH(r)
 	registerMultiTenantUpgrade(r)
 	registerNetwork(r)
 	registerNodeJSPostgres(r)

pkg/cmd/roachtest/tests/tpc_utils.go

@@ -33,14 +33,12 @@ func loadTPCHDataset(
 	ctx context.Context,
 	t test.Test,
 	c cluster.Cluster,
+	db *gosql.DB,
 	sf int,
 	m cluster.Monitor,
 	roachNodes option.NodeListOption,
 	disableMergeQueue bool,
 ) error {
-	db := c.Conn(ctx, t.L(), roachNodes[0])
-	defer db.Close()
-
 	if disableMergeQueue {
 		if _, err := db.Exec("SET CLUSTER SETTING kv.range_merge.queue_enabled = false;"); err != nil {
 			t.Fatal(err)

pkg/cmd/roachtest/tests/tpch_concurrency.go

@@ -55,7 +55,7 @@ func registerTPCHConcurrency(r registry.Registry) {
 		}
 
 		if err := loadTPCHDataset(
-			ctx, t, c, 1 /* sf */, c.NewMonitor(ctx, c.Range(1, numNodes-1)),
+			ctx, t, c, conn, 1 /* sf */, c.NewMonitor(ctx, c.Range(1, numNodes-1)),
 			c.Range(1, numNodes-1), true, /* disableMergeQueue */
 		); err != nil {
 			t.Fatal(err)

pkg/cmd/roachtest/tests/tpchbench.go

@@ -71,9 +71,12 @@ func runTPCHBench(ctx context.Context, t test.Test, c cluster.Cluster, b tpchBen
 
 	m := c.NewMonitor(ctx, roachNodes)
 	m.Go(func(ctx context.Context) error {
+		conn := c.Conn(ctx, t.L(), 1)
+		defer conn.Close()
+
 		t.Status("setting up dataset")
 		err := loadTPCHDataset(
-			ctx, t, c, b.ScaleFactor, m, roachNodes, true, /* disableMergeQueue */
+			ctx, t, c, conn, b.ScaleFactor, m, roachNodes, true, /* disableMergeQueue */
 		)
 		if err != nil {
 			return err