Merge #116080 #116091

116080: *: skip some tests under `race` r=rail,srosenberg a=rickystewart

All of these tests have OOM or timeout issues when running under `race`.

Epic: CRDB-8308
Release note: None

116091: sql: skip TestShowLastQueryStatistics under race r=mgartner a=mgartner

This commit skips TestShowLastQueryStatistics under the `race` test flag
because the added overhead of the race detector makes latency checks in
the test fail.

Fixes #115934

Release note: None


Co-authored-by: Ricky Stewart <[email protected]>
Co-authored-by: Marcus Gartner <[email protected]>

pkg/ccl/backupccl/backupdest/BUILD.bazel

@@ -56,6 +56,7 @@ go_test(
         "//pkg/server",
         "//pkg/sql",
         "//pkg/testutils/serverutils",
+        "//pkg/testutils/skip",
         "//pkg/testutils/testcluster",
         "//pkg/util/hlc",
         "//pkg/util/leaktest",

pkg/ccl/backupccl/backupdest/backup_destination_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/jobs/jobspb"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
 	"github.com/cockroachdb/cockroach/pkg/sql"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -38,6 +39,8 @@ func TestBackupRestoreResolveDestination(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	tc, _, _, cleanupFn := backuptestutils.StartBackupRestoreTestCluster(t, backuptestutils.MultiNode)
 	defer cleanupFn()
 

pkg/ccl/importerccl/BUILD.bazel

@@ -32,6 +32,7 @@ go_test(
         "//pkg/testutils",
         "//pkg/testutils/datapathutils",
         "//pkg/testutils/serverutils",
+        "//pkg/testutils/skip",
         "//pkg/testutils/sqlutils",
         "//pkg/testutils/testcluster",
         "//pkg/util/leaktest",

pkg/ccl/importerccl/ccl_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
@@ -57,6 +58,8 @@ func TestImportMultiRegion(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRaceWithIssue(t, 116049, "probable OOM")
+
 	baseDir := sharedTestdata(t)
 	tc, sqlDB, cleanup := multiregionccltestutils.TestingCreateMultiRegionCluster(
 		t, 3 /* numServers */, base.TestingKnobs{}, multiregionccltestutils.WithBaseDirectory(baseDir),

pkg/ccl/kvccl/kvfollowerreadsccl/followerreads_test.go

@@ -941,7 +941,7 @@ func TestSecondaryTenantFollowerReadsRouting(t *testing.T) {
 	defer log.Scope(t).Close(t)
 	defer utilccl.TestingEnableEnterprise()()
 
-	skip.UnderStressRace(t, "times out")
+	skip.UnderRace(t, "times out")
 
 	for _, testCase := range []struct {
 		name             string

pkg/ccl/serverccl/statusccl/tenant_grpc_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlstats"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -33,6 +34,8 @@ func TestTenantGRPCServices(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	ctx := context.Background()
 
 	testCluster := serverutils.StartCluster(t, 3, base.TestClusterArgs{

pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/BUILD.bazel

@@ -31,6 +31,7 @@ go_test(
         "//pkg/sql/catalog/tabledesc",
         "//pkg/testutils/datapathutils",
         "//pkg/testutils/serverutils",
+        "//pkg/testutils/skip",
         "//pkg/testutils/sqlutils",
         "//pkg/testutils/testcluster",
         "//pkg/util/leaktest",

pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/datadriven_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descs"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc"
 	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
@@ -80,6 +81,9 @@ import (
 //     Releases the protected timestamp record with id.
 func TestDataDriven(t *testing.T) {
 	t.Cleanup(leaktest.AfterTest(t))
+
+	skip.UnderRace(t, "likely to time out")
+
 	scope := log.Scope(t)
 	t.Cleanup(func() {
 		scope.Close(t)

pkg/ccl/streamingccl/streamingest/replication_stream_e2e_test.go

@@ -678,7 +678,7 @@ func TestStreamingAutoReplan(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
-	skip.UnderStressRace(t, "multi cluster/node config exhausts hardware")
+	skip.UnderRace(t, "multi cluster/node config exhausts hardware")
 
 	ctx := context.Background()
 	args := replicationtestutils.DefaultTenantStreamingClustersArgs
@@ -1202,7 +1202,7 @@ func TestLoadProducerAndIngestionProgress(t *testing.T) {
 func TestStreamingRegionalConstraint(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
-	skip.UnderStressRace(t, "takes too long under stress race")
+	skip.UnderRace(t, "takes too long under stress race")
 
 	ctx := context.Background()
 	regions := []string{"mars", "venus", "mercury"}
@@ -1273,7 +1273,7 @@ func TestStreamingRegionalConstraint(t *testing.T) {
 func TestStreamingMismatchedMRDatabase(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
-	skip.UnderStressRace(t, "takes too long under stress race")
+	skip.UnderRace(t, "takes too long under stress race")
 
 	ctx := context.Background()
 	regions := []string{"mars", "venus", "mercury"}
@@ -1354,7 +1354,7 @@ WHERE
 func TestStreamingZoneConfigsMismatchedRegions(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
-	skip.UnderStressRace(t, "takes too long under stress race")
+	skip.UnderRace(t, "takes too long under stress race")
 
 	ctx := context.Background()
 	regions := []string{"mars", "venus", "mercury"}

pkg/ccl/workloadccl/fixture_test.go

@@ -271,6 +271,8 @@ func TestImportFixtureNodeCount(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRaceWithIssue(t, 115977, "probable OOM")
+
 	ctx := context.Background()
 
 	const (

pkg/kv/kvserver/loqrecovery/server_integration_test.go

@@ -478,6 +478,8 @@ func TestRetrieveRangeStatus(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	ctx := context.Background()
 
 	tc, _, _ := prepTestCluster(t, 5)

pkg/kv/kvserver/reports/reporter_test.go

@@ -125,6 +125,9 @@ func TestConstraintConformanceReportIntegration(t *testing.T) {
 func TestCriticalLocalitiesReportIntegration(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
+
+	skip.UnderRace(t, "likely to timeout")
+
 	ctx := context.Background()
 	// 2 regions, 3 dcs per region.
 	tc := serverutils.StartCluster(t, 6, base.TestClusterArgs{

pkg/server/application_api/contention_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -41,6 +42,8 @@ func TestStatusAPIContentionEvents(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	ctx := context.Background()
 	testCluster := serverutils.StartCluster(t, 3, base.TestClusterArgs{})
 

pkg/server/application_api/sessions_test.go

@@ -234,6 +234,8 @@ func TestStatusCancelSessionGatewayMetadataPropagation(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	ctx := context.Background()
 	testCluster := serverutils.StartCluster(t, 3, base.TestClusterArgs{})
 	defer testCluster.Stopper().Stop(ctx)

pkg/server/application_api/sql_stats_test.go

@@ -332,6 +332,8 @@ func TestStatusAPITransactionStatementFingerprintIDsTruncation(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	testCluster := serverutils.StartCluster(t, 3, base.TestClusterArgs{})
 	defer testCluster.Stopper().Stop(context.Background())
 

pkg/server/storage_api/health_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/server/srvtestutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -108,6 +109,9 @@ func TestHealthAPI(t *testing.T) {
 func TestLivenessAPI(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
+
+	skip.UnderRace(t, "may OOM")
+
 	tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{})
 	defer tc.Stopper().Stop(context.Background())
 

pkg/sql/conn_executor_test.go

@@ -46,6 +46,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/pgtest"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
 	"github.com/cockroachdb/cockroach/pkg/util/duration"
@@ -1326,6 +1327,8 @@ func TestShowLastQueryStatistics(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "measure planning latency which is slower than usual under race")
+
 	ctx := context.Background()
 	params := base.TestServerArgs{}
 	s, sqlConn, _ := serverutils.StartServer(t, params)

pkg/sql/importer/exportcsv_test.go

@@ -40,6 +40,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/rowenc"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
@@ -102,6 +103,8 @@ func TestExportImportBank(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	db, _, cleanup := setupExportableBank(t, 3, 100)
 	defer cleanup()
 
@@ -190,6 +193,8 @@ func TestMultiNodeExportStmt(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	nodes := 5
 	exportRows := 100
 	db, _, cleanup := setupExportableBank(t, nodes, exportRows*2)

pkg/sql/importer/import_stmt_test.go

@@ -4537,7 +4537,7 @@ func TestImportDefaultNextVal(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 	defer setImportReaderParallelism(1)()
-	skip.UnderStressRace(t, "test hits a timeout before a successful run")
+	skip.UnderRace(t, "timeout and/or OOM")
 
 	const nodes = 3
 	numFiles := 1
@@ -4809,6 +4809,8 @@ func TestImportComputed(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	const nodes = 3
 
 	ctx := context.Background()
@@ -5419,6 +5421,8 @@ func TestImportMysql(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	const (
 		nodes = 3
 	)
@@ -5575,6 +5579,8 @@ func TestImportDelimited(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	const (
 		nodes = 3
 	)
@@ -5665,6 +5671,8 @@ func TestImportPgCopy(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	const (
 		nodes = 3
 	)
@@ -5751,6 +5759,8 @@ func TestImportPgDump(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	const (
 		nodes = 3
 	)
@@ -6510,6 +6520,8 @@ func TestImportCockroachDump(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	const (
 		nodes = 3
 	)
@@ -6609,6 +6621,8 @@ func TestImportAvro(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	const (
 		nodes = 3
 	)
@@ -6936,6 +6950,8 @@ func TestDetachedImport(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRace(t, "probable OOM")
+
 	const (
 		nodes = 3
 	)

pkg/sql/pgwire/pgwirecancel/cancel_test.go

@@ -76,6 +76,8 @@ func TestCancelQueryOtherNode(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRaceWithIssue(t, 116031, "likely OOM")
+
 	ctx, cancel := context.WithCancel(context.Background())
 	args := base.TestServerArgs{
 		Knobs: base.TestingKnobs{

pkg/sql/sqlstats/persistedsqlstats/controller_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
@@ -34,6 +35,8 @@ func TestPersistedSQLStatsReset(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	skip.UnderRaceWithIssue(t, 116037, "probable OOM")
+
 	ctx := context.Background()
 	cluster := serverutils.StartCluster(t, 3 /* numNodes */, base.TestClusterArgs{})
 	defer cluster.Stopper().Stop(ctx)

pkg/sql/sqlstats/persistedsqlstats/reader_test.go

@@ -38,6 +38,7 @@ func TestPersistedSQLStatsReadMemory(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 	skip.UnderStress(t, "the test is too slow to run under stress")
+	skip.UnderRaceWithIssue(t, 116036, "likely OOM")
 
 	testCluster, ctx := createCluster(t)
 	defer testCluster.Stopper().Stop(ctx)
@@ -51,6 +52,7 @@ func TestPersistedSQLStatsReadDisk(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 	skip.UnderStress(t, "the test is too slow to run under stress")
+	skip.UnderRaceWithIssue(t, 116035, "likely OOM")
 
 	testCluster, ctx := createCluster(t)
 	defer testCluster.Stopper().Stop(ctx)
@@ -65,6 +67,7 @@ func TestPersistedSQLStatsReadHybrid(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 	skip.UnderStress(t, "the test is too slow to run under stress")
+	skip.UnderRaceWithIssue(t, 116033, "likely OOM")
 
 	testCluster, ctx := createCluster(t)
 	defer testCluster.Stopper().Stop(ctx)