roachtest: compile cockroach-short with --crdb_test and use in sqlsmith

This commit adds another step to the nightly roachtest invocation to
compile `cockroach-short` binary with `--crdb_test` build tag. Then it
also adds plumbing throughout the roachtest infrastructure to expose
that newly-compiled binary which is now used in 50% cases in the sqlsmith
roachtest.

Release justification: testing-only change.

Release note: None

build/teamcity/cockroach/nightlies/roachtest_compile_bits.sh

@@ -5,14 +5,19 @@
 bazel build --config crosslinux --config ci --config with_ui -c opt --config force_build_cdeps \
       //pkg/cmd/cockroach //pkg/cmd/workload //pkg/cmd/roachtest \
       //c-deps:libgeos
+bazel build --config crosslinux --config ci -c opt //pkg/cmd/cockroach-short --crdb_test
 BAZEL_BIN=$(bazel info bazel-bin --config crosslinux --config ci --config with_ui -c opt)
+# Rename the binary with enabled assertions ("-ea" suffix stands for "enabled
+# assertions").
+mv $BAZEL_BIN/pkg/cmd/cockroach-short/cockroach-short_/cockroach-short $BAZEL_BIN/pkg/cmd/cockroach-short/cockroach-short_/cockroach-short-ea
 # Move this stuff to bin for simplicity.
 mkdir -p bin
 chmod o+rwx bin
 cp $BAZEL_BIN/pkg/cmd/cockroach/cockroach_/cockroach bin
+cp $BAZEL_BIN/pkg/cmd/cockroach-short/cockroach-short_/cockroach-short-ea bin
 cp $BAZEL_BIN/pkg/cmd/roachtest/roachtest_/roachtest bin
 cp $BAZEL_BIN/pkg/cmd/workload/workload_/workload    bin
-chmod a+w bin/cockroach bin/roachtest bin/workload
+chmod a+w bin/cockroach bin/cockroach-short-ea bin/roachtest bin/workload
 # Stage the geos libs in the appropriate spot.
 mkdir -p lib.docker_amd64
 chmod o+rwx lib.docker_amd64

build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

@@ -24,6 +24,7 @@ build/teamcity-roachtest-invoke.sh \
   --cluster-id="${TC_BUILD_ID}" \
   --build-tag="${BUILD_TAG}" \
   --cockroach="${PWD}/bin/cockroach" \
+  --cockroach-short="${PWD}/bin/cockroach-short-ea" \
   --artifacts=/artifacts \
   --artifacts-literal="${LITERAL_ARTIFACTS_DIR:-}" \
   --slack-token="${SLACK_TOKEN}" \

pkg/cmd/roachtest/cluster.go

@@ -63,6 +63,7 @@ var (
 	local bool
 
 	cockroach        string
+	cockroachShort   string
 	libraryFilePaths []string
 	cloud            = spec.GCE
 	// encryptionProbability controls when encryption-at-rest is enabled
@@ -207,6 +208,14 @@ func initBinariesAndLibraries() {
 		os.Exit(1)
 	}
 
+	cockroachShort, err = findBinary(cockroachShort, cockroach)
+	if errors.As(err, &errBinaryOrLibraryNotFound{}) {
+		fmt.Fprintln(os.Stderr, "cockroach-short binary not provided, proceeding anyway")
+	} else if err != nil {
+		fmt.Fprintf(os.Stderr, "%+v\n", err)
+		os.Exit(1)
+	}
+
 	workload, err = findBinary(workload, "workload")
 	if errors.As(err, &errBinaryOrLibraryNotFound{}) {
 		fmt.Fprintln(os.Stderr, "workload binary not provided, proceeding anyway")

pkg/cmd/roachtest/cluster_test.go

@@ -64,6 +64,10 @@ func (t testWrapper) Cockroach() string {
 	return "./dummy-path/to/cockroach"
 }
 
+func (t testWrapper) CockroachShort() string {
+	return "./dummy-path/to/cockroach-short"
+}
+
 func (t testWrapper) DeprecatedWorkload() string {
 	return "./dummy-path/to/workload"
 }

pkg/cmd/roachtest/main.go

@@ -131,6 +131,8 @@ func main() {
 			"If blank, the current OS user is detected and specified.")
 	rootCmd.PersistentFlags().StringVar(
 		&cockroach, "cockroach", "", "path to cockroach binary to use")
+	rootCmd.PersistentFlags().StringVar(
+		&cockroachShort, "cockroach-short", "", "path to cockroach-short binary (compiled with crdb_test build tag) to use")
 	rootCmd.PersistentFlags().StringVar(
 		&workload, "workload", "", "path to workload binary to use")
 	rootCmd.PersistentFlags().Float64Var(

pkg/cmd/roachtest/test/test_interface.go

@@ -18,7 +18,8 @@ import (
 // Test is the interface through which roachtests interact with the
 // test harness.
 type Test interface {
-	Cockroach() string // path to main cockroach binary
+	Cockroach() string      // path to main cockroach binary
+	CockroachShort() string // path to cockroach-short binary compiled with --crdb_test build tag
 	Name() string
 	BuildVersion() *version.Version
 	IsBuildVersion(string) bool // "vXX.YY"

pkg/cmd/roachtest/test_impl.go

@@ -45,6 +45,7 @@ type testImpl struct {
 	spec *registry.TestSpec
 
 	cockroach          string // path to main cockroach binary
+	cockroachShort     string // path to cockroach-short binary compiled with --crdb_test build tag
 	deprecatedWorkload string // path to workload binary
 	debug              bool   // whether the test is in debug mode.
 	// buildVersion is the version of the Cockroach binary that the test will run
@@ -108,6 +109,10 @@ func (t *testImpl) Cockroach() string {
 	return t.cockroach
 }
 
+func (t *testImpl) CockroachShort() string {
+	return t.cockroachShort
+}
+
 func (t *testImpl) DeprecatedWorkload() string {
 	return t.deprecatedWorkload
 }

pkg/cmd/roachtest/test_runner.go

@@ -580,6 +580,7 @@ func (r *testRunner) runWorker(
 		t := &testImpl{
 			spec:                   &testToRun.spec,
 			cockroach:              cockroach,
+			cockroachShort:         cockroachShort,
 			deprecatedWorkload:     workload,
 			buildVersion:           r.buildVersion,
 			artifactsDir:           artifactsDir,

pkg/cmd/roachtest/tests/sqlsmith.go

@@ -94,7 +94,14 @@ func registerSQLSmith(r registry.Registry) {
 		rng, seed := randutil.NewTestRand()
 		t.L().Printf("seed: %d", seed)
 
-		c.Put(ctx, t.Cockroach(), "./cockroach")
+		// With 50% chance use the cockroach-short binary that was compiled with
+		// --crdb_test build tag.
+		if rng.Float64() < 0.5 {
+			c.Put(ctx, t.Cockroach(), "./cockroach")
+		} else {
+			t.Status("using cockroach-short binary compiled with --crdb_test build tag")
+			c.Put(ctx, t.CockroachShort(), "./cockroach")
+		}
 		if err := c.PutLibraries(ctx, "./lib"); err != nil {
 			t.Fatalf("could not initialize libraries: %v", err)
 		}