From dd708c730ab8645604d31d25f22a866ad0845a7d Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Fri, 30 Apr 2021 13:00:48 -0700 Subject: [PATCH] issues: improve random syntax test crash reports Crashes during random syntax tests now report more useful information to the issue including the error message, the SQL that produced the error, and the database schema. The report will be formatted as: > Random syntax error: > > ``` > rsg_test.go:755: Crash detected: server panic: pq: internal error: something bad > ``` > Query: > > ``` > SELECT > foo > FROM > bar > LIMIT > 33:::INT8; > ``` > Schema: > > ``` > rsg_test.go:575: To reproduce, use schema: > rsg_test.go:577: > CREATE TABLE table1 (col1_0 BOOL); > ; > rsg_test.go:577: > > CREATE TYPE greeting AS ENUM ('hello', 'howdy', 'hi', 'good day', 'morning'); > ; > rsg_test.go:579: > rsg_test.go:580: -- test log scope end -- > test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestRandomSyntaxSQLSmith460792454 > --- FAIL: TestRandomSyntaxSQLSmith (300.69s) > ``` Release note: None --- pkg/cmd/internal/issues/condense.go | 62 ++++++++++++++-- pkg/cmd/internal/issues/condense_test.go | 30 ++++++++ pkg/cmd/internal/issues/formatter_unit.go | 11 ++- pkg/cmd/internal/issues/issues_test.go | 31 ++++++++ .../rsg-crash-matching-and-related-issue.txt | 68 +++++++++++++++++ .../post/rsg-crash-matching-issue.txt | 62 ++++++++++++++++ .../testdata/post/rsg-crash-no-issue.txt | 67 +++++++++++++++++ .../testdata/post/rsg-crash-related-issue.txt | 73 +++++++++++++++++++ pkg/sql/tests/rsg_test.go | 5 +- 9 files changed, 400 insertions(+), 9 deletions(-) create mode 100644 pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-and-related-issue.txt create mode 100644 pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-issue.txt create mode 100644 pkg/cmd/internal/issues/testdata/post/rsg-crash-no-issue.txt create mode 100644 pkg/cmd/internal/issues/testdata/post/rsg-crash-related-issue.txt diff --git a/pkg/cmd/internal/issues/condense.go b/pkg/cmd/internal/issues/condense.go index d206c3d9f322..ea4430a26d09 100644 --- a/pkg/cmd/internal/issues/condense.go +++ b/pkg/cmd/internal/issues/condense.go @@ -15,6 +15,18 @@ import ( "strings" ) +func firstNlines(input string, n int) string { + if input == "" { + return "" + } + pos := 0 + for pos < len(input) && n > 0 { + n-- + pos += strings.Index(input[pos:], "\n") + 1 + } + return input[:pos] +} + func lastNlines(input string, n int) string { if input == "" { return "" @@ -34,6 +46,14 @@ type FatalOrPanic struct { FirstStack string // the first stack, i.e. the goroutine relevant to error } +// RSGCrash contains information about a crash during random syntax tests +// obtained from a test log. +type RSGCrash struct { + Error, // the error message from the crash + Query, // the query that induced the crash + Schema string // the schema that the crash was induced with +} + // A CondensedMessage is a test log output garnished with useful helper methods // that extract concise information for seamless debugging. type CondensedMessage string @@ -41,15 +61,20 @@ type CondensedMessage string var panicRE = regexp.MustCompile(`(?ms)^(panic:.*?\n)(goroutine \d+.*?\n)\n`) var fatalRE = regexp.MustCompile(`(?ms)(^F\d{6}.*?\n)(goroutine \d+.*?\n)\n`) +// Note: These must be kept in-sync with the crash output of +// tests_test.testRandomSyntax. +var crasherRE = regexp.MustCompile(`(?s)( *rsg_test.go:\d{3}: Crash detected:.*?\n)(.*?;\n)`) +var reproRE = regexp.MustCompile(`(?s)( *rsg_test.go:\d{3}: To reproduce, use schema:)`) + // FatalOrPanic constructs a FatalOrPanic. If no fatal or panic occurred in the -// test, the zero value is returned. -func (s CondensedMessage) FatalOrPanic(numPrecedingLines int) FatalOrPanic { +// test, ok=false is returned. +func (s CondensedMessage) FatalOrPanic(numPrecedingLines int) (fop FatalOrPanic, ok bool) { ss := string(s) - var fop FatalOrPanic add := func(matches []int) { fop.LastLines = lastNlines(ss[:matches[2]], numPrecedingLines) fop.Error += ss[matches[2]:matches[3]] fop.FirstStack += ss[matches[4]:matches[5]] + ok = true } if sl := panicRE.FindStringSubmatchIndex(ss); sl != nil { add(sl) @@ -57,7 +82,25 @@ func (s CondensedMessage) FatalOrPanic(numPrecedingLines int) FatalOrPanic { if sl := fatalRE.FindStringSubmatchIndex(ss); sl != nil { add(sl) } - return fop + return fop, ok +} + +// RSGCrash constructs an RSGCrash. The query and reproduction SQL are limited +// to the first lineLimit lines. If no random syntax test crash occurred in the +// test, ok=false is returned. +func (s CondensedMessage) RSGCrash(lineLimit int) (c RSGCrash, ok bool) { + ss := string(s) + if cm := crasherRE.FindStringSubmatchIndex(ss); cm != nil { + c.Error = ss[cm[2]:cm[3]] + c.Query = firstNlines(ss[cm[4]:cm[5]], lineLimit) + if rm := reproRE.FindStringSubmatchIndex(ss); rm != nil { + // The "To reproduce" log is always near the end of the log file, so + // collect all lines after the first match position. + c.Schema = firstNlines(ss[rm[2]:], lineLimit) + } + return c, true + } + return RSGCrash{}, false } // String calls .Digest(30). @@ -66,12 +109,17 @@ func (s CondensedMessage) String() string { } // Digest returns the last n lines of the test log. If a panic or fatal error -// occurred, it instead returns the last n lines preceding that event, the -// event itself, and the first stack trace. +// occurred, it instead returns the last n lines preceding that event, the event +// itself, and the first stack trace. If a crash occurred during a random syntax +// test, it returns the error and up to n lines of both the query that caused +// the crash and the database schema. func (s CondensedMessage) Digest(n int) string { - if fop := s.FatalOrPanic(n); fop.Error != "" { + if fop, ok := s.FatalOrPanic(n); ok { return fop.LastLines + fop.Error + fop.FirstStack } + if c, ok := s.RSGCrash(n); ok { + return c.Error + c.Query + c.Schema + } // TODO(tbg): consider adding some smarts around the FAIL line here to handle // it similarly to FatalOrPanic (but without a stack trace). return lastNlines(string(s), n) diff --git a/pkg/cmd/internal/issues/condense_test.go b/pkg/cmd/internal/issues/condense_test.go index a30cf770c1db..4529fd01ce65 100644 --- a/pkg/cmd/internal/issues/condense_test.go +++ b/pkg/cmd/internal/issues/condense_test.go @@ -77,8 +77,28 @@ created by main.main exit status 2 ` +const rsgCrash = ` rsg_test.go:755: Crash detected: server panic: pq: internal error: something bad + SELECT + foo + FROM + bar + LIMIT + 33:::INT8; +` + +const rsgRepro = ` rsg_test.go:575: To reproduce, use schema: + rsg_test.go:577: + CREATE TABLE table1 (col1_0 BOOL); + ; + rsg_test.go:577: + CREATE TYPE greeting AS ENUM ('hello', 'hi'); + ; + rsg_test.go:579: +` + const panic5Lines = fiveLines + messagePanic + firstStack + restStack const fatal5Lines = fiveLines + messageFatal + firstStack + restStack +const crashAndRepro = fiveLines + rsgCrash + fiveLines + rsgRepro var errorCases = []condenseTestCase{ { @@ -109,6 +129,16 @@ var errorCases = []condenseTestCase{ messageFatal + firstStack, 0, }, + { + crashAndRepro, + rsgCrash + rsgRepro, + 100, + }, + { + crashAndRepro, + " rsg_test.go:755: Crash detected: server panic: pq: internal error: something bad\n", + 0, + }, } var lineCases = []condenseTestCase{ diff --git a/pkg/cmd/internal/issues/formatter_unit.go b/pkg/cmd/internal/issues/formatter_unit.go index baa379154309..e137d9a71603 100644 --- a/pkg/cmd/internal/issues/formatter_unit.go +++ b/pkg/cmd/internal/issues/formatter_unit.go @@ -41,7 +41,7 @@ var UnitTestFormatter = IssueFormatter{ r.Escaped(`: `) - if fop := data.CondensedMessage.FatalOrPanic(50); fop != (FatalOrPanic{}) { + if fop, ok := data.CondensedMessage.FatalOrPanic(50); ok { if fop.Error != "" { r.Escaped("Fatal error:") r.CodeBlock("", fop.Error) @@ -54,6 +54,15 @@ var UnitTestFormatter = IssueFormatter{ r.Collapsed("Log preceding fatal error", func() { r.CodeBlock("", fop.LastLines) }) + } else if rsgCrash, ok := data.CondensedMessage.RSGCrash(100); ok { + r.Escaped("Random syntax error:") + r.CodeBlock("", rsgCrash.Error) + r.Escaped("Query:") + r.CodeBlock("", rsgCrash.Query) + if rsgCrash.Schema != "" { + r.Escaped("Schema:") + r.CodeBlock("", rsgCrash.Schema) + } } else { r.CodeBlock("", data.CondensedMessage.Digest(50)) } diff --git a/pkg/cmd/internal/issues/issues_test.go b/pkg/cmd/internal/issues/issues_test.go index e1e6ca6471f5..348835302672 100644 --- a/pkg/cmd/internal/issues/issues_test.go +++ b/pkg/cmd/internal/issues/issues_test.go @@ -111,6 +111,37 @@ goroutine 13: author: "bran", reproCmd: "", }, + { + name: "rsg-crash", + packageName: "github.com/cockroachdb/cockroach/pkg/sql/tests", + testName: "TestRandomSyntaxSQLSmith", + message: `logging something + rsg_test.go:755: Crash detected: server panic: pq: internal error: something bad + SELECT + foo + FROM + bar + LIMIT + 33:::INT8; + + Stack trace: + rsg_test.go:764: 266003 executions, 235459 successful + rsg_test.go:575: To reproduce, use schema: + rsg_test.go:577: + CREATE TABLE table1 (col1_0 BOOL); + ; + rsg_test.go:577: + + CREATE TYPE greeting AS ENUM ('hello', 'howdy', 'hi', 'good day', 'morning'); + ; + rsg_test.go:579: + rsg_test.go:580: -- test log scope end -- +test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestRandomSyntaxSQLSmith460792454 +--- FAIL: TestRandomSyntaxSQLSmith (300.69s) +`, + author: "bran", + reproCmd: "make test TESTS=TestRandomSyntaxSQLSmith PKG=./pkg/sql/tests 2>&1", + }, } const ( diff --git a/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-and-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-and-related-issue.txt new file mode 100644 index 000000000000..1f18793d8d44 --- /dev/null +++ b/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-and-related-issue.txt @@ -0,0 +1,68 @@ +listCommits owner=cockroachdb repo=cockroach github.CommitsListOptions{SHA:"", Path:"", Author:"bran", Since:time.Time{wall:, ext:}, Until:time.Time{wall:, ext:}, ListOptions:github.ListOptions{Page:0, PerPage:1}} +searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "sql/tests: TestRandomSyntaxSQLSmith failed" label:branch-release-0.1: [github.Issue{Number:30, Title:"boom", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.1"}]}] +searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "sql/tests: TestRandomSyntaxSQLSmith failed" -label:branch-release-0.1: [github.Issue{Number:31, Title:"boom related", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}] +createComment owner=cockroachdb repo=cockroach issue=30: + +sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): + +Random syntax error: + +``` + rsg_test.go:755: Crash detected: server panic: pq: internal error: something bad +``` +Query: + +``` + SELECT + foo + FROM + bar + LIMIT + 33:::INT8; +``` +Schema: + +``` + rsg_test.go:575: To reproduce, use schema: + rsg_test.go:577: + CREATE TABLE table1 (col1_0 BOOL); + ; + rsg_test.go:577: + + CREATE TYPE greeting AS ENUM ('hello', 'howdy', 'hi', 'good day', 'morning'); + ; + rsg_test.go:579: + rsg_test.go:580: -- test log scope end -- +test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestRandomSyntaxSQLSmith460792454 +--- FAIL: TestRandomSyntaxSQLSmith (300.69s) +``` +
Reproduce +

+

To reproduce, try: + +```bash +make test TESTS=TestRandomSyntaxSQLSmith PKG=./pkg/sql/tests 2>&1 +``` +

+

Parameters in this failure: + +- TAGS=deadlock + +- GOFLAGS=race +

+

+
+
Same failure on other branches +

+ +- #31 boom related [C-test-failure O-robot release-0.2] +

+
+/cc @hodor + + +[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*TestRandomSyntaxSQLSmith.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/internal/issues) + + + +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EReproduce%3C%2Fsummary%3E%0A%3Cp%3E%0A%3Cp%3ETo+reproduce%2C+try%3A%0A%0A%60%60%60bash%0Amake+test+TESTS%3DTestRandomSyntaxSQLSmith+PKG%3D.%2Fpkg%2Fsql%2Ftests+2%3E%261%0A%60%60%60%0A%3C%2Fp%3E%0A%3Cp%3EParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40hodor%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E \ No newline at end of file diff --git a/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-issue.txt b/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-issue.txt new file mode 100644 index 000000000000..5522c68442bc --- /dev/null +++ b/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-issue.txt @@ -0,0 +1,62 @@ +listCommits owner=cockroachdb repo=cockroach github.CommitsListOptions{SHA:"", Path:"", Author:"bran", Since:time.Time{wall:, ext:}, Until:time.Time{wall:, ext:}, ListOptions:github.ListOptions{Page:0, PerPage:1}} +searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "sql/tests: TestRandomSyntaxSQLSmith failed" label:branch-release-0.1: [github.Issue{Number:30, Title:"boom", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.1"}]}] +searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "sql/tests: TestRandomSyntaxSQLSmith failed" -label:branch-release-0.1: [] +createComment owner=cockroachdb repo=cockroach issue=30: + +sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): + +Random syntax error: + +``` + rsg_test.go:755: Crash detected: server panic: pq: internal error: something bad +``` +Query: + +``` + SELECT + foo + FROM + bar + LIMIT + 33:::INT8; +``` +Schema: + +``` + rsg_test.go:575: To reproduce, use schema: + rsg_test.go:577: + CREATE TABLE table1 (col1_0 BOOL); + ; + rsg_test.go:577: + + CREATE TYPE greeting AS ENUM ('hello', 'howdy', 'hi', 'good day', 'morning'); + ; + rsg_test.go:579: + rsg_test.go:580: -- test log scope end -- +test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestRandomSyntaxSQLSmith460792454 +--- FAIL: TestRandomSyntaxSQLSmith (300.69s) +``` +
Reproduce +

+

To reproduce, try: + +```bash +make test TESTS=TestRandomSyntaxSQLSmith PKG=./pkg/sql/tests 2>&1 +``` +

+

Parameters in this failure: + +- TAGS=deadlock + +- GOFLAGS=race +

+

+
+/cc @hodor + + +[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*TestRandomSyntaxSQLSmith.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/internal/issues) + + + +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EReproduce%3C%2Fsummary%3E%0A%3Cp%3E%0A%3Cp%3ETo+reproduce%2C+try%3A%0A%0A%60%60%60bash%0Amake+test+TESTS%3DTestRandomSyntaxSQLSmith+PKG%3D.%2Fpkg%2Fsql%2Ftests+2%3E%261%0A%60%60%60%0A%3C%2Fp%3E%0A%3Cp%3EParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40hodor%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E \ No newline at end of file diff --git a/pkg/cmd/internal/issues/testdata/post/rsg-crash-no-issue.txt b/pkg/cmd/internal/issues/testdata/post/rsg-crash-no-issue.txt new file mode 100644 index 000000000000..f4b076214986 --- /dev/null +++ b/pkg/cmd/internal/issues/testdata/post/rsg-crash-no-issue.txt @@ -0,0 +1,67 @@ +listCommits owner=cockroachdb repo=cockroach github.CommitsListOptions{SHA:"", Path:"", Author:"bran", Since:time.Time{wall:, ext:}, Until:time.Time{wall:, ext:}, ListOptions:github.ListOptions{Page:0, PerPage:1}} +searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "sql/tests: TestRandomSyntaxSQLSmith failed" label:branch-release-0.1: [] +searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "sql/tests: TestRandomSyntaxSQLSmith failed" -label:branch-release-0.1: [] +getLatestTag: result v3.3.0 +listMilestones owner=cockroachdb repo=cockroach: result [github.Milestone{Number:2, Title:"3.3"} github.Milestone{Number:1, Title:"3.2"}] +createIssue owner=cockroachdb repo=cockroach: +github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "release-blocker"], Milestone:2} + +sql/tests: TestRandomSyntaxSQLSmith failed + +sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): + +Random syntax error: + +``` + rsg_test.go:755: Crash detected: server panic: pq: internal error: something bad +``` +Query: + +``` + SELECT + foo + FROM + bar + LIMIT + 33:::INT8; +``` +Schema: + +``` + rsg_test.go:575: To reproduce, use schema: + rsg_test.go:577: + CREATE TABLE table1 (col1_0 BOOL); + ; + rsg_test.go:577: + + CREATE TYPE greeting AS ENUM ('hello', 'howdy', 'hi', 'good day', 'morning'); + ; + rsg_test.go:579: + rsg_test.go:580: -- test log scope end -- +test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestRandomSyntaxSQLSmith460792454 +--- FAIL: TestRandomSyntaxSQLSmith (300.69s) +``` +
Reproduce +

+

To reproduce, try: + +```bash +make test TESTS=TestRandomSyntaxSQLSmith PKG=./pkg/sql/tests 2>&1 +``` +

+

Parameters in this failure: + +- TAGS=deadlock + +- GOFLAGS=race +

+

+
+/cc @hodor + + +[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*TestRandomSyntaxSQLSmith.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/internal/issues) + + + +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EReproduce%3C%2Fsummary%3E%0A%3Cp%3E%0A%3Cp%3ETo+reproduce%2C+try%3A%0A%0A%60%60%60bash%0Amake+test+TESTS%3DTestRandomSyntaxSQLSmith+PKG%3D.%2Fpkg%2Fsql%2Ftests+2%3E%261%0A%60%60%60%0A%3C%2Fp%3E%0A%3Cp%3EParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40hodor%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=sql%2Ftests%3A+TestRandomSyntaxSQLSmith+failed \ No newline at end of file diff --git a/pkg/cmd/internal/issues/testdata/post/rsg-crash-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/rsg-crash-related-issue.txt new file mode 100644 index 000000000000..0faa17055c20 --- /dev/null +++ b/pkg/cmd/internal/issues/testdata/post/rsg-crash-related-issue.txt @@ -0,0 +1,73 @@ +listCommits owner=cockroachdb repo=cockroach github.CommitsListOptions{SHA:"", Path:"", Author:"bran", Since:time.Time{wall:, ext:}, Until:time.Time{wall:, ext:}, ListOptions:github.ListOptions{Page:0, PerPage:1}} +searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "sql/tests: TestRandomSyntaxSQLSmith failed" label:branch-release-0.1: [] +searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "sql/tests: TestRandomSyntaxSQLSmith failed" -label:branch-release-0.1: [github.Issue{Number:31, Title:"boom related", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}] +getLatestTag: result v3.3.0 +listMilestones owner=cockroachdb repo=cockroach: result [github.Milestone{Number:2, Title:"3.3"} github.Milestone{Number:1, Title:"3.2"}] +createIssue owner=cockroachdb repo=cockroach: +github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "release-blocker"], Milestone:2} + +sql/tests: TestRandomSyntaxSQLSmith failed + +sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): + +Random syntax error: + +``` + rsg_test.go:755: Crash detected: server panic: pq: internal error: something bad +``` +Query: + +``` + SELECT + foo + FROM + bar + LIMIT + 33:::INT8; +``` +Schema: + +``` + rsg_test.go:575: To reproduce, use schema: + rsg_test.go:577: + CREATE TABLE table1 (col1_0 BOOL); + ; + rsg_test.go:577: + + CREATE TYPE greeting AS ENUM ('hello', 'howdy', 'hi', 'good day', 'morning'); + ; + rsg_test.go:579: + rsg_test.go:580: -- test log scope end -- +test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestRandomSyntaxSQLSmith460792454 +--- FAIL: TestRandomSyntaxSQLSmith (300.69s) +``` +
Reproduce +

+

To reproduce, try: + +```bash +make test TESTS=TestRandomSyntaxSQLSmith PKG=./pkg/sql/tests 2>&1 +``` +

+

Parameters in this failure: + +- TAGS=deadlock + +- GOFLAGS=race +

+

+
+
Same failure on other branches +

+ +- #31 boom related [C-test-failure O-robot release-0.2] +

+
+/cc @hodor + + +[This test on roachdash](https://roachdash.crdb.dev/?filter=status:open%20t:.*TestRandomSyntaxSQLSmith.*&sort=title+created&display=lastcommented+project) | [Improve this report!](https://github.com/cockroachdb/cockroach/tree/master/pkg/cmd/internal/issues) + + + +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EReproduce%3C%2Fsummary%3E%0A%3Cp%3E%0A%3Cp%3ETo+reproduce%2C+try%3A%0A%0A%60%60%60bash%0Amake+test+TESTS%3DTestRandomSyntaxSQLSmith+PKG%3D.%2Fpkg%2Fsql%2Ftests+2%3E%261%0A%60%60%60%0A%3C%2Fp%3E%0A%3Cp%3EParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40hodor%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=sql%2Ftests%3A+TestRandomSyntaxSQLSmith+failed \ No newline at end of file diff --git a/pkg/sql/tests/rsg_test.go b/pkg/sql/tests/rsg_test.go index bf06490baccf..ead1a8761813 100644 --- a/pkg/sql/tests/rsg_test.go +++ b/pkg/sql/tests/rsg_test.go @@ -752,7 +752,10 @@ func testRandomSyntax( countsMu.success++ } else { if c := (*crasher)(nil); errors.As(err, &c) { - t.Errorf("Crash detected: \n%s\n\nStack trace:\n%s", c.sql, c.detail) + // NOTE: Changes to this output format must be kept in-sync + // with logic in CondensedMessage.RSGCrash in order for + // crashes to be correctly reported to Github. + t.Errorf("Crash detected: %s\n%s;\n\nStack trace:\n%s", c.Error(), c.sql, c.detail) } } countsMu.Unlock()