acceptance: add reference test for cockroachdb#7389

Also straightened out some kinks with the existing reference tests, namely that
they were creating two single-node clusters per test which were never used.
Instead, we can now create zero-node clusters and run one-shot containers on
it (one would argue that that's still weird, but it seems acceptable for the
time being).

The newly added reference test will crash when run for commits between cockroachdb#7310
and cockroachdb#7429. See cockroachdb/postgres-test#13 for the initial state used in the
test.

acceptance/cluster/localcluster.go

@@ -225,7 +225,13 @@ func (l *LocalCluster) OneShot(
 	if err := l.oneshot.Start(); err != nil {
 		return err
 	}
-	return l.oneshot.Wait()
+	err = l.oneshot.Wait()
+	if err != nil {
+		var b bytes.Buffer
+		l.oneshot.Logs(&b)
+		return fmt.Errorf("%s:\n%s", err, b.String())
+	}
+	return nil
 }
 
 // stopOnPanic is invoked as a deferred function in Start in order to attempt

acceptance/reference_test.go

@@ -21,14 +21,26 @@ import (
 	"testing"
 )
 
+func runReferenceTestWithScript(
+	t *testing.T,
+	script string,
+) {
+	if err := testDockerOneShot(t, "reference", []string{"/cockroach", "version"}); err != nil {
+		t.Skipf(`TODO(dt): No /cockroach binary in one-shot container, see #6086: %s`, err)
+	}
+
+	err := testDockerOneShot(t, "reference", []string{"/bin/bash", "-c", script})
+	if err != nil {
+		t.Errorf("expected success: %s", err)
+	}
+
+}
+
 func runReadWriteReferenceTest(
 	t *testing.T,
 	referenceBinPath string,
 	backwardReferenceTest string,
 ) {
-	if err := testDockerSingleNode(t, "reference", []string{"/cockroach", "version"}); err != nil {
-		t.Skipf(`TODO(dt): No /cockroach binary in one-shot container, see #6086: %s`, err)
-	}
 	referenceTestScript := fmt.Sprintf(`
 set -xe
 mkdir /old
@@ -41,6 +53,7 @@ bin=/%s/cockroach
 # TODO(bdarnell): when --background is in referenceBinPath, use it here and below.
 $bin start &
 sleep 1
+
 echo "Use the reference binary to write a couple rows, then render its output to a file and shut down."
 $bin sql -e "CREATE DATABASE old"
 $bin sql -d old -e "CREATE TABLE testing_old (i int primary key, b bool, s string unique, d decimal, f float, t timestamp, v interval, index sb (s, b))"
@@ -56,6 +69,10 @@ $bin sql -d old -e "SELECT i, b, s, d, f, v, extract(epoch FROM t) FROM testing_
 # diff returns non-zero if different. With set -e above, that would exit here.
 diff new.everything old.everything
 
+# Scan all data (some of which may not have been touched by the SQL commands)
+# to wake up all Raft groups.
+$bin debug kv scan
+
 echo "Add a row with the new binary and render the updated data before shutting down."
 $bin sql -d old -e "INSERT INTO testing_old values (3, false, '!', decimal '2.14159', 2.14159, NOW(), interval '3h')"
 $bin sql -d old -e "SELECT i, b, s, d, f, v, extract(epoch FROM t) FROM testing_old" > new.everything
@@ -70,10 +87,7 @@ echo "Read the modified data using the reference binary again."
 bin=/%s/cockroach
 %s
 `, referenceBinPath, referenceBinPath, backwardReferenceTest)
-	err := testDockerSingleNode(t, "reference", []string{"/bin/bash", "-c", referenceTestScript})
-	if err != nil {
-		t.Errorf("expected success: %s", err)
-	}
+	runReferenceTestWithScript(t, referenceTestScript)
 }
 
 func TestDockerReadWriteBidirectionalReferenceVersion(t *testing.T) {
@@ -98,3 +112,23 @@ $bin quit && wait
 `
 	runReadWriteReferenceTest(t, `forward-reference-version`, backwardReferenceTest)
 }
+
+func TestDockerMigration_7429(t *testing.T) {
+	script := `
+set -eux
+bin=/cockroach
+
+touch out
+
+function finish {
+  cat out
+}
+
+trap finish EXIT
+
+$bin start --alsologtostderr=INFO --background --store=/cockroach-data-reference-7429 &> out
+$bin debug kv scan
+$bin quit
+`
+	runReferenceTestWithScript(t, script)
+}

acceptance/replication_test.go

@@ -38,6 +38,12 @@ func countRangeReplicas(db *client.DB) (int, error) {
 }
 
 func checkRangeReplication(t *testing.T, c cluster.Cluster, d time.Duration) {
+	if c.NumNodes() < 1 {
+		// Looks silly, but we actually start zero-node clusters in the
+		// reference tests.
+		t.Log("replication test is a no-op for empty cluster")
+		return
+	}
 	// Always talk to node 0.
 	client, dbStopper := c.NewClient(t, 0)
 	defer dbStopper.Stop()

acceptance/util_test.go

@@ -208,11 +208,17 @@ func StartCluster(t *testing.T, cfg cluster.TestConfig) (c cluster.Cluster) {
 	if !*flagRemote {
 		logDir := *flagLogDir
 		if logDir != "" {
-			_, _, fun := caller.Lookup(3)
-			if !testFuncRE.MatchString(fun) {
-				t.Fatalf("invalid caller %s; want TestX -> runTestOnConfigs -> func()", fun)
+			var i int
+			for ; i < 100; i++ {
+				_, _, fun := caller.Lookup(i)
+				if testFuncRE.MatchString(fun) {
+				}
+				logDir = filepath.Join(logDir, fun)
+				break
+			}
+			if i >= 100 {
+				panic("no caller matching Test(.*) in stack trace")
 			}
-			logDir = filepath.Join(logDir, fun)
 		}
 		l := cluster.CreateLocal(cfg, logDir, *flagPrivileged, stopper)
 		l.Start()
@@ -279,18 +285,18 @@ const (
 	postgresTestImage = "cockroachdb/postgres-test:" + postgresTestTag
 )
 
-func testDockerSingleNode(t *testing.T, name string, cmd []string) error {
+func testDocker(t *testing.T, num int, name string, cmd []string) error {
 	SkipUnlessLocal(t)
 	cfg := cluster.TestConfig{
 		Name:     name,
 		Duration: *flagDuration,
-		Nodes:    []cluster.NodeConfig{{Count: 1, Stores: []cluster.StoreConfig{{Count: 1}}}},
+		Nodes:    []cluster.NodeConfig{{Count: int32(num), Stores: []cluster.StoreConfig{{Count: 1}}}},
 	}
 	l := StartCluster(t, cfg).(*cluster.LocalCluster)
 	defer l.AssertAndStop(t)
 
 	containerConfig := container.Config{
-		Image: fmt.Sprintf(postgresTestImage),
+		Image: postgresTestImage,
 		Env: []string{
 			"PGHOST=roach0",
 			fmt.Sprintf("PGPORT=%s", base.DefaultPort),
@@ -302,3 +308,11 @@ func testDockerSingleNode(t *testing.T, name string, cmd []string) error {
 	hostConfig := container.HostConfig{NetworkMode: "host"}
 	return l.OneShot(postgresTestImage, types.ImagePullOptions{}, containerConfig, hostConfig, "docker-"+name)
 }
+
+func testDockerSingleNode(t *testing.T, name string, cmd []string) error {
+	return testDocker(t, 1, name, cmd)
+}
+
+func testDockerOneShot(t *testing.T, name string, cmd []string) error {
+	return testDocker(t, 0, name, cmd)
+}