Merge pull request #65829 from tbg/backport21.1-65386

release-21.1: acceptance: maybe deflake acceptance/multi-tenant

pkg/cli/mt_start_sql.go

@@ -137,5 +137,6 @@ func runStartSQL(cmd *cobra.Command, args []string) error {
 	ch := make(chan os.Signal, 1)
 	signal.Notify(ch, drainSignals...)
 	sig := <-ch
+	log.Flush()
 	return errors.Newf("received signal %v", sig)
 }

pkg/cmd/roachtest/multitenant.go

@@ -52,8 +52,7 @@ func runAcceptanceMultitenant(ctx context.Context, t *test, c *cluster) {
 	t.Status("stopping the server ahead of checking for the tenant server")
 
 	// Stop the server, which also ensures that log files get flushed.
-	cancel()
-	<-tenant.errCh
+	tenant.stop(ctx, t, c)
 
 	t.Status("checking log file contents")
 

pkg/cmd/roachtest/multitenant_upgrade.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
+	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/version"
 	"github.com/stretchr/testify/require"
 )
@@ -98,14 +99,32 @@ func (tn *tenantNode) start(ctx context.Context, t *test, c *cluster, binary str
 	require.NoError(t, err)
 	u.Host = c.ExternalIP(ctx, c.Node(tn.node))[0] + ":" + strconv.Itoa(tn.sqlPort)
 	tn.pgURL = u.String()
-	c.l.Printf("sql server for tenant %d should be running at %s", tn.tenantID, tn.pgURL)
 
-	const warmupPeriod = 3 * time.Second
-	select {
-	case err := <-tn.errCh:
+	// The tenant is usually responsive ~right away, but it
+	// has on occasions taken more than 3s for it to connect
+	// to the KV layer, and it won't open the SQL port until
+	// it has.
+	if err := retry.ForDuration(45*time.Second, func() error {
+		select {
+		case <-ctx.Done():
+			t.Fatal(ctx.Err())
+		case <-tn.errCh:
+			t.Fatal(err)
+		default:
+		}
+
+		db, err := gosql.Open("postgres", tn.pgURL)
+		if err != nil {
+			return err
+		}
+		defer db.Close()
+		_, err = db.ExecContext(ctx, `SELECT 1`)
+		return err
+	}); err != nil {
 		t.Fatal(err)
-	case <-time.After(warmupPeriod):
 	}
+
+	c.l.Printf("sql server for tenant %d running at %s", tn.tenantID, tn.pgURL)
 }
 
 // runMultiTenantUpgrade exercises upgrading tenants and their host cluster.