roachtest: Allow roachtests to opt out of failing in post validation …

.github/CODEOWNERS

@@ -343,6 +343,7 @@
 /pkg/cmd/generate-binary/    @cockroachdb/sql-sessions
 /pkg/cmd/generate-distdir/ @cockroachdb/dev-inf
 /pkg/cmd/generate-logictest/       @cockroachdb/dev-inf
+/pkg/cmd/generate-acceptance-tests/       @cockroachdb/dev-inf
 /pkg/cmd/generate-metadata-tables/ @cockroachdb/sql-sessions
 /pkg/cmd/generate-spatial-ref-sys/ @cockroachdb/spatial
 /pkg/cmd/generate-bazel-extra/ @cockroachdb/dev-inf

build/bazelutil/bazel-generate.sh

@@ -63,6 +63,12 @@ else
   bazel run pkg/cmd/generate-logictest -- -out-dir="$PWD"
 fi
 
+if files_unchanged_from_upstream WORKSPACE $(find_relevant ./pkg/acceptance -name '*') $(find_relevant ./pkg/cli/interactive_tests -name '*') $(find_relevant ./pkg/cmd/generate-acceptance-tests -name '*'); then
+  echo "Skipping //pkg/cmd/generate-acceptance-tests (relevant files are unchanged from upstream)"
+else
+  bazel run pkg/cmd/generate-acceptance-tests -- -out-dir="$PWD"
+fi
+
 if files_unchanged_from_upstream c-deps/archived.bzl c-deps/REPOSITORIES.bzl DEPS.bzl WORKSPACE $(find_relevant ./pkg/cmd/generate-distdir -name BUILD.bazel -or -name '*.go') $(find_relevant ./pkg/build/bazel -name BUILD.bazel -or -name '*.go') $(find_relevant pkg/build/starlarkutil -name BUILD.bazel -or -name '*.go'); then
     echo "Skipping //pkg/cmd/generate-distdir (relevant files are unchanged from upstream)."
 else

build/teamcity/cockroach/ci/tests/acceptance.sh

@@ -34,6 +34,7 @@ $BAZCI --artifacts_dir=$PWD/artifacts -- \
   --test_arg=-l="$ARTIFACTSDIR" \
   --test_arg=-b=$PWD/artifacts/cockroach \
   --test_env=TZ=America/New_York \
+  --profile=$PWD/artifacts/profile.gz \
   --test_timeout=1800 || status=$?
 
 # Some unit tests test automatic ballast creation. These ballasts can be

pkg/BUILD.bazel

@@ -980,6 +980,8 @@ GO_TARGETS = [
     "//pkg/cmd/docs-issue-generation:docs-issue-generation_test",
     "//pkg/cmd/fuzz:fuzz",
     "//pkg/cmd/fuzz:fuzz_lib",
+    "//pkg/cmd/generate-acceptance-tests:generate-acceptance-tests",
+    "//pkg/cmd/generate-acceptance-tests:generate-acceptance-tests_lib",
     "//pkg/cmd/generate-bazel-extra:generate-bazel-extra",
     "//pkg/cmd/generate-bazel-extra:generate-bazel-extra_lib",
     "//pkg/cmd/generate-bazel-extra:generate-bazel-extra_test",
@@ -2578,6 +2580,7 @@ GET_X_DATA_TARGETS = [
     "//pkg/cmd/docgen/extract:get_x_data",
     "//pkg/cmd/docs-issue-generation:get_x_data",
     "//pkg/cmd/fuzz:get_x_data",
+    "//pkg/cmd/generate-acceptance-tests:get_x_data",
     "//pkg/cmd/generate-bazel-extra:get_x_data",
     "//pkg/cmd/generate-binary:get_x_data",
     "//pkg/cmd/generate-distdir:get_x_data",

pkg/acceptance/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "//pkg/server",  # keep
         "//pkg/testutils",
         "//pkg/testutils/serverutils",  # keep
+        "//pkg/testutils/skip",
         "//pkg/testutils/testcluster",  # keep
         "//pkg/util/log",
         "//pkg/util/randutil",  # keep
@@ -40,6 +41,7 @@ go_test(
         "cli_test.go",
         "compose_test.go",
         "debug_remote_test.go",
+        "generated_cli_test.go",
         "main_test.go",
     ],
     # Note: we overide bazel timeout to 1800s using test_timeout.
@@ -52,6 +54,7 @@ go_test(
     ],
     embed = [":acceptance"],
     gotags = ["acceptance"],
+    shard_count = 16,
     tags = ["integration"],
     deps = [
         "//pkg/acceptance/cluster",

pkg/acceptance/cli_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The Cockroach Authors.
+// Copyright 2023 The Cockroach Authors.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt.
@@ -13,87 +13,14 @@ package acceptance
 import (
 	"context"
 	"fmt"
-	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/cockroachdb/cockroach/pkg/acceptance/cluster"
-	"github.com/cockroachdb/cockroach/pkg/build/bazel"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 )
 
-var cmdBase = []string{
-	"/usr/bin/env",
-	"COCKROACH_SKIP_UPDATE_CHECK=1",
-	"COCKROACH_CRASH_REPORTS=",
-	"/bin/bash",
-	"-c",
-}
-
-func TestDockerCLI(t *testing.T) {
-	s := log.Scope(t)
-	defer s.Close(t)
-
-	containerConfig := defaultContainerConfig()
-	containerConfig.Cmd = []string{"stat", cluster.CockroachBinaryInContainer}
-	containerConfig.Env = []string{
-		"CI=1", // Disables the initial color query by the termenv library.
-		fmt.Sprintf("PGUSER=%s", username.RootUser),
-	}
-	ctx := context.Background()
-	if err := testDockerOneShot(ctx, t, "cli_test", containerConfig); err != nil {
-		skip.IgnoreLintf(t, `TODO(dt): No binary in one-shot container, see #6086: %s`, err)
-	}
-
-	testGlob := "../cli/interactive_tests/test*.tcl"
-	paths, err := filepath.Glob(testGlob)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(paths) == 0 {
-		t.Fatalf("no testfiles found (%v)", testGlob)
-	}
-
-	containerPath := "/go/src/github.com/cockroachdb/cockroach/cli/interactive_tests"
-	if bazel.BuiltWithBazel() {
-		containerPath = "/mnt/interactive_tests"
-	}
-	for _, p := range paths {
-		testFile := filepath.Base(p)
-		testPath := filepath.Join(containerPath, testFile)
-		if strings.Contains(testPath, "disabled") {
-			t.Logf("Skipping explicitly disabled test %s", testFile)
-			continue
-		}
-		t.Run(testFile, func(t *testing.T) {
-			log.Infof(ctx, "-- starting tests from: %s", testFile)
-
-			// Symlink the logs directory to /logs, which is visible outside of the
-			// container and preserved if the test fails. (They don't write to /logs
-			// directly because they are often run manually outside of Docker, where
-			// /logs is unlikely to exist.)
-			cmd := "ln -s /logs logs"
-
-			// We run the expect command using `bash -c "(expect ...)"`.
-			//
-			// We cannot run `expect` directly, nor `bash -c "expect ..."`,
-			// because both cause Expect to become the PID 1 process inside
-			// the container. On Unix, orphan processes need to be wait()ed
-			// upon by the PID 1 process when they terminate, lest they
-			// remain forever in the zombie state. Unfortunately, Expect
-			// does not contain code to do this. Bash does.
-			cmd += "; (expect -d -f " + testPath + " " + cluster.CockroachBinaryInContainer + ")"
-			containerConfig.Cmd = append(cmdBase, cmd)
-
-			if err := testDockerOneShot(ctx, t, "cli_test", containerConfig); err != nil {
-				t.Error(err)
-			}
-		})
-	}
-}
-
 // TestDockerUnixSocket verifies that CockroachDB initializes a unix
 // socket useable by 'psql', even when the server runs insecurely.
 // TODO(knz): Replace this with a roachtest when roachtest/roachprod
@@ -107,7 +34,7 @@ func TestDockerUnixSocket(t *testing.T) {
 	ctx := context.Background()
 
 	if err := testDockerOneShot(ctx, t, "cli_test", containerConfig); err != nil {
-		skip.IgnoreLintf(t, `TODO(dt): No binary in one-shot container, see #6086: %s`, err)
+		skip.IgnoreLintf(t, "TODO(dt): No binary in one-shot container, see #6086: %s", err)
 	}
 
 	containerConfig.Env = []string{fmt.Sprintf("PGUSER=%s", username.RootUser)}
@@ -131,7 +58,7 @@ func TestSQLWithoutTLS(t *testing.T) {
 	ctx := context.Background()
 
 	if err := testDockerOneShot(ctx, t, "cli_test", containerConfig); err != nil {
-		skip.IgnoreLintf(t, `TODO(dt): No binary in one-shot container, see #6086: %s`, err)
+		skip.IgnoreLintf(t, "TODO(dt): No binary in one-shot container, see #6086: %s", err)
 	}
 
 	containerConfig.Env = []string{fmt.Sprintf("PGUSER=%s", username.RootUser)}

pkg/acceptance/cluster/dockercluster.go

@@ -13,15 +13,18 @@ package cluster
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	gosql "database/sql"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
+	"math/big"
 	"net"
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -517,21 +520,28 @@ func (l *DockerCluster) startNode(ctx context.Context, node *testNode) {
 // automatically, but exposed for tests that use INIT_NONE. nodeIdx
 // may designate any node in the cluster as the target of the command.
 func (l *DockerCluster) RunInitCommand(ctx context.Context, nodeIdx int) {
+	// Add a randomID to the container name to avoid overlap between tests running on
+	// different shards.
+	nBig, err := rand.Int(rand.Reader, big.NewInt(10000000))
+	if err != nil {
+		panic(err)
+	}
+	randomID := strconv.Itoa(int(nBig.Int64()))
 	containerConfig := container.Config{
 		Image:      *cockroachImage,
 		Entrypoint: cockroachEntrypoint(),
 		Cmd: []string{
 			"init",
 			"--certs-dir=/certs/",
 			"--host=" + l.Nodes[nodeIdx].nodeStr,
-			"--log-dir=/logs/init-command",
+			"--log-dir=/logs/init-command-" + randomID,
 			"--logtostderr=NONE",
 		},
 	}
 
 	log.Infof(ctx, "trying to initialize via %v", containerConfig.Cmd)
 	maybePanic(l.OneShot(ctx, defaultImage, types.ImagePullOptions{},
-		containerConfig, container.HostConfig{}, platforms.DefaultSpec(), "init-command"))
+		containerConfig, container.HostConfig{}, platforms.DefaultSpec(), "init-command-"+randomID))
 	log.Info(ctx, "cluster successfully initialized")
 }
 

pkg/acceptance/compose_test.go

@@ -25,16 +25,16 @@ import (
 
 const composeDir = "compose"
 
-func TestComposeGSS(t *testing.T) {
-	testCompose(t, filepath.Join("gss", "docker-compose.yml"), "psql")
-}
-
-func TestComposeGSSPython(t *testing.T) {
-	testCompose(t, filepath.Join("gss", "docker-compose-python.yml"), "python")
-}
-
-func TestComposeFlyway(t *testing.T) {
-	testCompose(t, filepath.Join("flyway", "docker-compose.yml"), "flyway")
+func TestCompose(t *testing.T) {
+	t.Run("ComposeGSS", func(t *testing.T) {
+		testCompose(t, filepath.Join("gss", "docker-compose.yml"), "psql")
+	})
+	t.Run("ComposeGSSPython", func(t *testing.T) {
+		testCompose(t, filepath.Join("gss", "docker-compose-python.yml"), "python")
+	})
+	t.Run("ComposeGSSFlyway", func(t *testing.T) {
+		testCompose(t, filepath.Join("flyway", "docker-compose.yml"), "flyway")
+	})
 }
 
 func testCompose(t *testing.T, path string, exitCodeFrom string) {