From 9d31864244de258329f70b47b5b29062c44bc3e1 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 28 Aug 2023 10:03:15 -0400 Subject: [PATCH] roachtest: add O-rsg to random syntax tests The `O-rsg` label is now applied to issues created when a test with randomized SQL syntax fails: sqlsmith, TLP, costfuzz, and unoptimized query oracle tests. Release note: None --- pkg/cmd/roachtest/github.go | 3 +++ pkg/cmd/roachtest/github_test.go | 27 +++++++++++++++---- pkg/cmd/roachtest/registry/test_spec.go | 4 +++ .../testdata/help_command_createpost_6.txt | 17 ++++++++++++ pkg/cmd/roachtest/tests/costfuzz.go | 1 + pkg/cmd/roachtest/tests/sqlsmith.go | 1 + pkg/cmd/roachtest/tests/tlp.go | 1 + .../tests/unoptimized_query_oracle.go | 1 + 8 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 pkg/cmd/roachtest/testdata/help_command_createpost_6.txt diff --git a/pkg/cmd/roachtest/github.go b/pkg/cmd/roachtest/github.go index a43138ee1c9a..dc3bacb14254 100644 --- a/pkg/cmd/roachtest/github.go +++ b/pkg/cmd/roachtest/github.go @@ -164,6 +164,9 @@ func (g *githubIssues) createPostRequest( } else if !spec.NonReleaseBlocker { labels = append(labels, "release-blocker") } + if len(spec.ExtraLabels) > 0 { + labels = append(labels, spec.ExtraLabels...) + } teams, err := g.teamLoader() if err != nil { diff --git a/pkg/cmd/roachtest/github_test.go b/pkg/cmd/roachtest/github_test.go index 93f177a0d1f7..a8743e486a0f 100644 --- a/pkg/cmd/roachtest/github_test.go +++ b/pkg/cmd/roachtest/github_test.go @@ -131,6 +131,7 @@ func TestCreatePostRequest(t *testing.T) { clusterCreationFailed bool loadTeamsFailed bool localSSD bool + extraLabels []string arch vm.CPUArch failure failure expectedPost bool @@ -138,7 +139,7 @@ func TestCreatePostRequest(t *testing.T) { expectedSkipTestFailure bool expectedParams map[string]string }{ - {true, false, false, false, "", createFailure(errors.New("other")), true, false, false, + {true, false, false, false, nil, "", createFailure(errors.New("other")), true, false, false, prefixAll(map[string]string{ "cloud": "gce", "encrypted": "false", @@ -149,7 +150,7 @@ func TestCreatePostRequest(t *testing.T) { "localSSD": "false", }), }, - {true, false, false, true, vm.ArchARM64, createFailure(errClusterProvisioningFailed), true, false, true, + {true, false, false, true, nil, vm.ArchARM64, createFailure(errClusterProvisioningFailed), true, false, true, prefixAll(map[string]string{ "cloud": "gce", "encrypted": "false", @@ -164,7 +165,7 @@ func TestCreatePostRequest(t *testing.T) { // !nonReleaseBlocker and issue is an SSH flake. Also ensure that // in the event of a failed cluster creation, nil `vmOptions` and // `clusterImpl` are not dereferenced - {false, true, false, false, "", createFailure(rperrors.ErrSSH255), true, false, true, + {false, true, false, false, nil, "", createFailure(rperrors.ErrSSH255), true, false, true, prefixAll(map[string]string{ "cloud": "gce", "ssd": "0", @@ -172,9 +173,21 @@ func TestCreatePostRequest(t *testing.T) { }), }, //Simulate failure loading TEAMS.yaml - {true, false, true, false, "", createFailure(errors.New("other")), false, false, false, nil}, + {true, false, true, false, nil, "", createFailure(errors.New("other")), false, false, false, nil}, //Error during post test assertions - {true, false, false, false, "", createFailure(errDuringPostAssertions), false, false, false, nil}, + {true, false, false, false, nil, "", createFailure(errDuringPostAssertions), false, false, false, nil}, + // Assert that extra labels in the test spec are added to the issue. + {true, false, false, false, []string{"foo-label"}, "", createFailure(errors.New("other")), true, false, false, + prefixAll(map[string]string{ + "cloud": "gce", + "encrypted": "false", + "fs": "ext4", + "ssd": "0", + "cpu": "4", + "arch": "amd64", + "localSSD": "false", + }), + }, } reg := makeTestRegistry(spec.GCE, "", "", false, false) @@ -187,6 +200,7 @@ func TestCreatePostRequest(t *testing.T) { Owner: OwnerUnitTest, Cluster: clusterSpec, NonReleaseBlocker: c.nonReleaseBlocker, + ExtraLabels: c.extraLabels, } ti := &testImpl{ @@ -238,6 +252,9 @@ func TestCreatePostRequest(t *testing.T) { } require.True(t, contains(req.ExtraLabels, nil, "O-roachtest")) + for _, l := range c.extraLabels { + require.True(t, contains(req.ExtraLabels, nil, l), "expected extra label %q", l) + } require.Equal(t, c.expectedReleaseBlocker, contains(req.ExtraLabels, nil, "release-blocker")) require.Equal(t, c.expectedSkipTestFailure, req.SkipLabelTestFailure) diff --git a/pkg/cmd/roachtest/registry/test_spec.go b/pkg/cmd/roachtest/registry/test_spec.go index 0b71394a2790..1b5cacb1a1db 100644 --- a/pkg/cmd/roachtest/registry/test_spec.go +++ b/pkg/cmd/roachtest/registry/test_spec.go @@ -119,6 +119,10 @@ type TestSpec struct { // fingerprint, i.e. the node count, the version, etc. that appear in the // infix. This will get rid of this awkward prefix naming restriction. SnapshotPrefix string + + // ExtraLabels are test-specific labels that will be added to the Github + // issue created when a failure occurs, in addition to default labels. + ExtraLabels []string } // PostValidation is a type of post-validation that runs after a test completes. diff --git a/pkg/cmd/roachtest/testdata/help_command_createpost_6.txt b/pkg/cmd/roachtest/testdata/help_command_createpost_6.txt new file mode 100644 index 000000000000..d41d8bebc8db --- /dev/null +++ b/pkg/cmd/roachtest/testdata/help_command_createpost_6.txt @@ -0,0 +1,17 @@ +echo +---- +---- + + +See: [roachtest README](https://github.com/cockroachdb/cockroach/blob/master/pkg/cmd/roachtest/README.md) + + + +See: [How To Investigate \(internal\)](https://cockroachlabs.atlassian.net/l/c/SSSBr8c7) + + + +See: [Grafana](https://go.crdb.dev/p/roachfana/foo/1689957243000/1689957733000) + +---- +---- diff --git a/pkg/cmd/roachtest/tests/costfuzz.go b/pkg/cmd/roachtest/tests/costfuzz.go index 9cc36ccda7fc..560cd6b33365 100644 --- a/pkg/cmd/roachtest/tests/costfuzz.go +++ b/pkg/cmd/roachtest/tests/costfuzz.go @@ -65,6 +65,7 @@ func registerCostFuzz(r registry.Registry) { name: "costfuzz", setupName: setupName, run: runCostFuzzQuery, }) }, + ExtraLabels: []string{"O-rsg"}, }) } } diff --git a/pkg/cmd/roachtest/tests/sqlsmith.go b/pkg/cmd/roachtest/tests/sqlsmith.go index 3e5511dae1cf..5f4886ed2b22 100644 --- a/pkg/cmd/roachtest/tests/sqlsmith.go +++ b/pkg/cmd/roachtest/tests/sqlsmith.go @@ -322,6 +322,7 @@ WITH into_db = 'defaultdb', unsafe_restore_incompatible_version; } runSQLSmith(ctx, t, c, setup, setting) }, + ExtraLabels: []string{"O-rsg"}, }) } diff --git a/pkg/cmd/roachtest/tests/tlp.go b/pkg/cmd/roachtest/tests/tlp.go index 45ee0a81019c..5b2f8353b0be 100644 --- a/pkg/cmd/roachtest/tests/tlp.go +++ b/pkg/cmd/roachtest/tests/tlp.go @@ -43,6 +43,7 @@ func registerTLP(r registry.Registry) { Leases: registry.MetamorphicLeases, NativeLibs: registry.LibGEOS, Run: runTLP, + ExtraLabels: []string{"O-rsg"}, }) } diff --git a/pkg/cmd/roachtest/tests/unoptimized_query_oracle.go b/pkg/cmd/roachtest/tests/unoptimized_query_oracle.go index 1c6dd4566fd3..fbc8b0751a02 100644 --- a/pkg/cmd/roachtest/tests/unoptimized_query_oracle.go +++ b/pkg/cmd/roachtest/tests/unoptimized_query_oracle.go @@ -69,6 +69,7 @@ func registerUnoptimizedQueryOracle(r registry.Registry) { }, }) }, + ExtraLabels: []string{"O-rsg"}, }) } }