roachtest: compile cockroach-short with --crdb_test and use in sqlsmith

This commit adds another step to the nightly roachtest invocation to
compile `cockroach-short` binary with `--crdb_test` build tag. Then it
also adds plumbing throughout the roachtest infrastructure to expose
that newly-compiled binary which is now used in 50% cases in the sqlsmith
roachtest.

Release justification: testing-only change.

Release note: None

build/teamcity/cockroach/nightlies/roachtest_compile_bits.sh

@@ -5,14 +5,18 @@
 bazel build --config crosslinux --config ci --config with_ui -c opt --config force_build_cdeps \
       //pkg/cmd/cockroach //pkg/cmd/workload //pkg/cmd/roachtest \
       //c-deps:libgeos
+bazel build --config crosslinux --config ci -c opt //pkg/cmd/cockroach-short --crdb_test
 BAZEL_BIN=$(bazel info bazel-bin --config crosslinux --config ci --config with_ui -c opt)
 # Move this stuff to bin for simplicity.
 mkdir -p bin
 chmod o+rwx bin
 cp $BAZEL_BIN/pkg/cmd/cockroach/cockroach_/cockroach bin
+# Copy the binary with enabled assertions while adding "-ea" suffix (which
+# stands for "enabled assertions").
+cp $BAZEL_BIN/pkg/cmd/cockroach-short/cockroach-short_/cockroach-short bin/cockroach-short-ea
 cp $BAZEL_BIN/pkg/cmd/roachtest/roachtest_/roachtest bin
 cp $BAZEL_BIN/pkg/cmd/workload/workload_/workload    bin
-chmod a+w bin/cockroach bin/roachtest bin/workload
+chmod a+w bin/cockroach bin/cockroach-short-ea bin/roachtest bin/workload
 # Stage the geos libs in the appropriate spot.
 mkdir -p lib.docker_amd64
 chmod o+rwx lib.docker_amd64

build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

@@ -24,6 +24,7 @@ build/teamcity-roachtest-invoke.sh \
   --cluster-id="${TC_BUILD_ID}" \
   --build-tag="${BUILD_TAG}" \
   --cockroach="${PWD}/bin/cockroach" \
+  --cockroach-short="${PWD}/bin/cockroach-short-ea" \
   --artifacts=/artifacts \
   --artifacts-literal="${LITERAL_ARTIFACTS_DIR:-}" \
   --slack-token="${SLACK_TOKEN}" \

pkg/cmd/roachtest/cluster.go

@@ -63,6 +63,7 @@ var (
 	local bool
 
 	cockroach        string
+	cockroachShort   string
 	libraryFilePaths []string
 	cloud            = spec.GCE
 	// encryptionProbability controls when encryption-at-rest is enabled
@@ -207,6 +208,13 @@ func initBinariesAndLibraries() {
 		os.Exit(1)
 	}
 
+	if cockroachShort != "" {
+		// defValue doesn't matter since cockroachShort is a non-empty string.
+		cockroachShort, err = findBinary(cockroachShort, "" /* defValue */)
+		fmt.Fprintf(os.Stderr, "%+v\n", err)
+		os.Exit(1)
+	}
+
 	workload, err = findBinary(workload, "workload")
 	if errors.As(err, &errBinaryOrLibraryNotFound{}) {
 		fmt.Fprintln(os.Stderr, "workload binary not provided, proceeding anyway")

pkg/cmd/roachtest/cluster_test.go

@@ -64,6 +64,10 @@ func (t testWrapper) Cockroach() string {
 	return "./dummy-path/to/cockroach"
 }
 
+func (t testWrapper) CockroachShort() string {
+	return "./dummy-path/to/cockroach-short"
+}
+
 func (t testWrapper) DeprecatedWorkload() string {
 	return "./dummy-path/to/workload"
 }

pkg/cmd/roachtest/main.go

@@ -131,6 +131,8 @@ func main() {
 			"If blank, the current OS user is detected and specified.")
 	rootCmd.PersistentFlags().StringVar(
 		&cockroach, "cockroach", "", "path to cockroach binary to use")
+	rootCmd.PersistentFlags().StringVar(
+		&cockroachShort, "cockroach-short", "", "path to cockroach-short binary (compiled with crdb_test build tag) to use")
 	rootCmd.PersistentFlags().StringVar(
 		&workload, "workload", "", "path to workload binary to use")
 	rootCmd.PersistentFlags().Float64Var(

pkg/cmd/roachtest/test/test_interface.go

@@ -19,6 +19,9 @@ import (
 // test harness.
 type Test interface {
 	Cockroach() string // path to main cockroach binary
+	// CockroachShort returns the path to cockroach-short binary compiled with
+	// --crdb_test build tag, or an empty string if no such binary was given.
+	CockroachShort() string
 	Name() string
 	BuildVersion() *version.Version
 	IsBuildVersion(string) bool // "vXX.YY"

pkg/cmd/roachtest/test_impl.go

@@ -45,6 +45,7 @@ type testImpl struct {
 	spec *registry.TestSpec
 
 	cockroach          string // path to main cockroach binary
+	cockroachShort     string // path to cockroach-short binary compiled with --crdb_test build tag
 	deprecatedWorkload string // path to workload binary
 	debug              bool   // whether the test is in debug mode.
 	// buildVersion is the version of the Cockroach binary that the test will run
@@ -108,6 +109,10 @@ func (t *testImpl) Cockroach() string {
 	return t.cockroach
 }
 
+func (t *testImpl) CockroachShort() string {
+	return t.cockroachShort
+}
+
 func (t *testImpl) DeprecatedWorkload() string {
 	return t.deprecatedWorkload
 }

pkg/cmd/roachtest/test_runner.go

@@ -580,6 +580,7 @@ func (r *testRunner) runWorker(
 		t := &testImpl{
 			spec:                   &testToRun.spec,
 			cockroach:              cockroach,
+			cockroachShort:         cockroachShort,
 			deprecatedWorkload:     workload,
 			buildVersion:           r.buildVersion,
 			artifactsDir:           artifactsDir,

pkg/cmd/roachtest/tests/sqlsmith.go

@@ -94,7 +94,9 @@ func registerSQLSmith(r registry.Registry) {
 		rng, seed := randutil.NewTestRand()
 		t.L().Printf("seed: %d", seed)
 
-		c.Put(ctx, t.Cockroach(), "./cockroach")
+		// With 50% chance use the cockroach-short binary that was compiled with
+		// --crdb_test build tag.
+		maybeUseBuildWithEnabledAssertions(ctx, t, c, rng, 0.5)
 		if err := c.PutLibraries(ctx, "./lib"); err != nil {
 			t.Fatalf("could not initialize libraries: %v", err)
 		}

pkg/cmd/roachtest/tests/util.go

@@ -14,6 +14,7 @@ import (
 	"context"
 	gosql "database/sql"
 	"fmt"
+	"math/rand"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
@@ -125,3 +126,20 @@ func SetAdmissionControl(ctx context.Context, t test.Test, c cluster.Cluster, en
 		}
 	}
 }
+
+// maybeUseBuildWithEnabledAssertions stages the cockroach-short binary with
+// enabled assertions with eaProb probability if that binary is available,
+// otherwise stages the regular cockroach binary.
+func maybeUseBuildWithEnabledAssertions(
+	ctx context.Context, t test.Test, c cluster.Cluster, rng *rand.Rand, eaProb float64,
+) {
+	if rng.Float64() < eaProb {
+		// Check whether the cockroach-short binary is available.
+		if t.CockroachShort() != "" {
+			t.Status("using cockroach-short binary compiled with --crdb_test build tag")
+			c.Put(ctx, t.CockroachShort(), "./cockroach")
+			return
+		}
+	}
+	c.Put(ctx, t.Cockroach(), "./cockroach")
+}