diff --git a/cmd/junit2jira/main.go b/cmd/junit2jira/main.go index ba7001d..1ebfb2c 100644 --- a/cmd/junit2jira/main.go +++ b/cmd/junit2jira/main.go @@ -109,7 +109,7 @@ func run(p params) error { jiraClient: jiraClient, } - testSuites, err := junit.IngestDir(p.junitReportsDir) + testSuites, err := testcase.LoadTestSuites(p.junitReportsDir) if err != nil { log.Fatalf("could not read files: %s", err) } diff --git a/cmd/junit2jira/main_test.go b/cmd/junit2jira/main_test.go index 52da7ae..198b382 100644 --- a/cmd/junit2jira/main_test.go +++ b/cmd/junit2jira/main_test.go @@ -3,18 +3,18 @@ package main import ( "bytes" _ "embed" + "github.com/stackrox/junit2jira/pkg/testcase" "net/url" "testing" "github.com/andygrunwald/go-jira" - "github.com/joshdk/go-junit" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestParseJunitReport(t *testing.T) { t.Run("not existing", func(t *testing.T) { - tests, err := junit.IngestDir("not existing") + tests, err := testcase.LoadTestSuites("not existing") assert.Error(t, err) assert.Nil(t, tests) }) @@ -22,7 +22,7 @@ func TestParseJunitReport(t *testing.T) { j := junit2jira{ params: params{junitReportsDir: "testdata/jira/report.xml"}, } - testsSuites, err := junit.IngestDir(j.junitReportsDir) + testsSuites, err := testcase.LoadTestSuites(j.junitReportsDir) assert.NoError(t, err) tests, err := j.getMergedFailedTests(testsSuites) assert.NoError(t, err) @@ -47,7 +47,7 @@ func TestParseJunitReport(t *testing.T) { j := junit2jira{ params: params{junitReportsDir: "testdata/jira/report.xml", JobName: "job-name", threshold: 1}, } - testsSuites, err := junit.IngestDir(j.junitReportsDir) + testsSuites, err := testcase.LoadTestSuites(j.junitReportsDir) assert.NoError(t, err) tests, err := j.getMergedFailedTests(testsSuites) assert.NoError(t, err) @@ -65,7 +65,7 @@ github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssu j := junit2jira{ params: params{junitReportsDir: "testdata/jira", JobName: "job-name", BuildId: "1", threshold: 3}, } - testsSuites, err := junit.IngestDir(j.junitReportsDir) + testsSuites, err := testcase.LoadTestSuites(j.junitReportsDir) assert.NoError(t, err) tests, err := j.getMergedFailedTests(testsSuites) assert.NoError(t, err) @@ -75,6 +75,7 @@ github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssu []j2jTestCase{ { Message: `DefaultPoliciesTest / Verify policy Apache Struts CVE-2017-5638 is triggered FAILED +github.com/stackrox/rox/pkg/grpc / Test_APIServerSuite/Test_TwoTestsStartingAPIs FAILED central-basic / step 90-activate-scanner-v4 FAILED github.com/stackrox/rox/pkg/booleanpolicy/evaluator / TestDifferentBaseTypes FAILED github.com/stackrox/rox/sensor/kubernetes/localscanner / TestLocalScannerTLSIssuerIntegrationTests FAILED @@ -93,7 +94,7 @@ command-line-arguments / TestTimeout FAILED j := junit2jira{ params: params{junitReportsDir: "testdata/jira", BuildId: "1"}, } - testsSuites, err := junit.IngestDir(j.junitReportsDir) + testsSuites, err := testcase.LoadTestSuites(j.junitReportsDir) assert.NoError(t, err) tests, err := j.getMergedFailedTests(testsSuites) assert.NoError(t, err) @@ -135,6 +136,20 @@ command-line-arguments / TestTimeout FAILED "\n" + "\tat DefaultPoliciesTest.Verify policy #policyName is triggered(DefaultPoliciesTest.groovy:181)\n", }, + { + Name: "Test_APIServerSuite/Test_TwoTestsStartingAPIs", + Message: "No test result found", + Stdout: "", + Suite: "github.com/stackrox/rox/pkg/grpc", + BuildId: "1", + Error: ` testutils.go:94: Stopping [2] listeners + testutils.go:87: [http handler listener: stopped] + testutils.go:87: [gRPC server listener: not stopped in loop. Comparing with grpcServer pointer with listener.srv pointer (0xc002ab2e00 : 0xc002ab2e00)] + server_test.go:229: ----------------------------------------------- + server_test.go:230: STACK TRACE INFO + server_test.go:231: ----------------------------------------------- +`, + }, { Name: "TestDifferentBaseTypes", Suite: "github.com/stackrox/rox/pkg/booleanpolicy/evaluator", @@ -176,9 +191,9 @@ command-line-arguments / TestTimeout FAILED }) t.Run("gradle", func(t *testing.T) { j := junit2jira{ - params: params{junitReportsDir: "testdata/jira/TEST-DefaultPoliciesTest.xml", BuildId: "1"}, + params: params{junitReportsDir: "testdata/jira/csv/TEST-DefaultPoliciesTest.xml", BuildId: "1"}, } - testsSuites, err := junit.IngestDir(j.junitReportsDir) + testsSuites, err := testcase.LoadTestSuites(j.junitReportsDir) assert.NoError(t, err) tests, err := j.getMergedFailedTests(testsSuites) assert.NoError(t, err) @@ -304,19 +319,17 @@ waitForViolation(deploymentName, policyName, 60) func TestCsvOutput(t *testing.T) { p := params{ - BuildId: "1", - JobName: "comma ,", - Orchestrator: "test", - BuildTag: "0.0.0", - BaseLink: `quote "`, - BuildLink: "buildLink", - timestamp: "time", + BuildId: "1", + JobName: "comma ,", + Orchestrator: "test", + BuildTag: "0.0.0", + BaseLink: `quote "`, + BuildLink: "buildLink", + timestamp: "time", + junitReportsDir: "testdata/jira/csv", } buf := bytes.NewBufferString("") - testSuites, err := junit.IngestFiles([]string{ - "testdata/jira/TEST-DefaultPoliciesTest.xml", - "testdata/jira/kuttl-report.xml", - }) + testSuites, err := testcase.LoadTestSuites(p.junitReportsDir) assert.NoError(t, err) err = junit2csv(testSuites, p, buf) assert.NoError(t, err) @@ -337,6 +350,10 @@ func TestCsvOutput(t *testing.T) { 1,time,DefaultPoliciesTest,Verify that built-in services don't trigger unexpected alerts,0,skipped,"comma ,",0.0.0 1,time,DefaultPoliciesTest,Verify that alert counts API is consistent with alerts,0,skipped,"comma ,",0.0.0 1,time,DefaultPoliciesTest,Verify that alert groups API is consistent with alerts,0,skipped,"comma ,",0.0.0 +1,time,github.com/stackrox/rox/pkg/grpc,Test_APIServerSuite/Test_TwoTestsStartingAPIs,0,error,"comma ,",0.0.0 +1,time,github.com/stackrox/rox/pkg/grpc/authz/user,TestLogInternalErrorInterceptor,0,passed,"comma ,",0.0.0 +1,time,fallback-classname,TestNoClassname,0,passed,"comma ,",0.0.0 +1,time,TestNoName,fallback-name,0,passed,"comma ,",0.0.0 1,time,central-basic,setup,0,passed,"comma ,",0.0.0 1,time,central-basic,step 0-image-pull-secrets,0,passed,"comma ,",0.0.0 1,time,central-basic,step 10-central-cr,0,passed,"comma ,",0.0.0 diff --git a/cmd/junit2jira/testdata/jira/TEST-DefaultPoliciesTest.xml b/cmd/junit2jira/testdata/jira/csv/TEST-DefaultPoliciesTest.xml similarity index 100% rename from cmd/junit2jira/testdata/jira/TEST-DefaultPoliciesTest.xml rename to cmd/junit2jira/testdata/jira/csv/TEST-DefaultPoliciesTest.xml diff --git a/cmd/junit2jira/testdata/jira/csv/golang-junit-report1.xml b/cmd/junit2jira/testdata/jira/csv/golang-junit-report1.xml new file mode 100644 index 0000000..5171955 --- /dev/null +++ b/cmd/junit2jira/testdata/jira/csv/golang-junit-report1.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + diff --git a/cmd/junit2jira/testdata/jira/csv/golang-junit-report2-bad.xml b/cmd/junit2jira/testdata/jira/csv/golang-junit-report2-bad.xml new file mode 100644 index 0000000..80bbf9b --- /dev/null +++ b/cmd/junit2jira/testdata/jira/csv/golang-junit-report2-bad.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + diff --git a/cmd/junit2jira/testdata/jira/kuttl-report.xml b/cmd/junit2jira/testdata/jira/csv/kuttl-report.xml similarity index 100% rename from cmd/junit2jira/testdata/jira/kuttl-report.xml rename to cmd/junit2jira/testdata/jira/csv/kuttl-report.xml diff --git a/pkg/testcase/testcase.go b/pkg/testcase/testcase.go index 007bb33..528f9c1 100644 --- a/pkg/testcase/testcase.go +++ b/pkg/testcase/testcase.go @@ -3,10 +3,13 @@ package testcase import ( "fmt" "github.com/joshdk/go-junit" + "slices" "strings" ) const subTestFormat = "\nSub test %s: %s" +const fallbackName = "fallback-name" +const fallbackClassname = "fallback-classname" type TestCase struct { Name string @@ -19,6 +22,66 @@ type TestCase struct { IsSubtest bool } +type ignoreTestCase struct { + Name string + Classname string +} + +var ignoreList = []ignoreTestCase{ + {Name: "", Classname: "runtime.MemStats"}, +} + +// LoadTestSuites loads all reports in provided directory. +func LoadTestSuites(reportDir string) ([]junit.Suite, error) { + testSuites, err := junit.IngestDir(reportDir) + if err != nil { + return nil, err + } + + return getClearedSuites(testSuites), nil +} + +func deleteHelperTest(testElem junit.Test) bool { + for _, ignore := range ignoreList { + if testElem.Name == ignore.Name && testElem.Classname == ignore.Classname { + return true + } + } + + return false +} + +func addFallbacks(tests []junit.Test) []junit.Test { + for i, _ := range tests { + if tests[i].Classname == "" { + tests[i].Classname = fallbackClassname + } + + if tests[i].Name == "" { + tests[i].Name = fallbackName + } + } + + return tests +} + +// getClearedSuites recursively removes ignored tests. +func getClearedSuites(suites []junit.Suite) []junit.Suite { + resSuites := make([]junit.Suite, 0, len(suites)) + for _, suite := range suites { + suite.Suites = getClearedSuites(suite.Suites) + suite.Tests = addFallbacks(slices.DeleteFunc(suite.Tests, deleteHelperTest)) + + // Filter out empty suites. + if len(suite.Suites) == 0 && len(suite.Tests) == 0 { + continue + } + resSuites = append(resSuites, suite) + } + + return resSuites +} + func (tc *TestCase) addSubTest(subTest junit.Test) { if subTest.Message != "" { tc.Message += fmt.Sprintf(subTestFormat, subTest.Name, subTest.Message) diff --git a/pkg/testcase/testcase_test.go b/pkg/testcase/testcase_test.go new file mode 100644 index 0000000..48a50e5 --- /dev/null +++ b/pkg/testcase/testcase_test.go @@ -0,0 +1,410 @@ +package testcase + +import ( + "github.com/joshdk/go-junit" + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_getClearedSuites(t *testing.T) { + tests := map[string]struct { + ignoreList []ignoreTestCase + suites []junit.Suite + expectedSuites []junit.Suite + }{ + "empty suites": {}, + "simple no match": { + ignoreList: []ignoreTestCase{ + {Name: "match", Classname: "me"}, + }, + suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "do not match", + Classname: "me", + }, + }, + }, + }, + expectedSuites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "do not match", + Classname: "me", + }, + }, + }, + }, + }, + "simple match": { + ignoreList: []ignoreTestCase{ + {Name: "match", Classname: "me"}, + }, + suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "match", + Classname: "me", + }, + }, + }, + }, + expectedSuites: []junit.Suite{}, + }, + "nested suites only": { + ignoreList: []ignoreTestCase{ + {Name: "match", Classname: "me"}, + }, + suites: []junit.Suite{ + { + Suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "skip", + Classname: "me", + }, + { + Name: "match", + Classname: "me", + }, + { + Name: "match", + Classname: "other", + }, + }, + }, + }, + Tests: []junit.Test{}, + }, + }, + expectedSuites: []junit.Suite{ + { + Suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "skip", + Classname: "me", + }, + { + Name: "match", + Classname: "other", + }, + }, + }, + }, + Tests: []junit.Test{}, + }, + }, + }, + "nested suites and tests": { + ignoreList: []ignoreTestCase{ + {Name: "match", Classname: "me"}, + }, + suites: []junit.Suite{ + { + Suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "skip", + Classname: "me", + }, + { + Name: "match", + Classname: "me", + }, + { + Name: "match", + Classname: "other", + }, + }, + }, + }, + Tests: []junit.Test{ + { + Name: "skip", + Classname: "me", + }, + { + Name: "match", + Classname: "me", + }, + { + Name: "match", + Classname: "other", + }, + }, + }, + }, + expectedSuites: []junit.Suite{ + { + Suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "skip", + Classname: "me", + }, + { + Name: "match", + Classname: "other", + }, + }, + }, + }, + Tests: []junit.Test{ + { + Name: "skip", + Classname: "me", + }, + { + Name: "match", + Classname: "other", + }, + }, + }, + }, + }, + "match all nested suites only": { + ignoreList: []ignoreTestCase{ + {Name: "match", Classname: "me"}, + {Name: "me", Classname: "including"}, + }, + suites: []junit.Suite{ + { + Suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "match", + Classname: "me", + }, + { + Name: "me", + Classname: "including", + }, + }, + }, + }, + Tests: []junit.Test{}, + }, + }, + expectedSuites: []junit.Suite{}, + }, + "match all test only": { + ignoreList: []ignoreTestCase{ + {Name: "match", Classname: "me"}, + }, + suites: []junit.Suite{ + { + Suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "skip", + Classname: "me", + }, + }, + }, + }, + Tests: []junit.Test{ + { + Name: "match", + Classname: "me", + }, + }, + }, + }, + expectedSuites: []junit.Suite{ + { + Suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "skip", + Classname: "me", + }, + }, + }, + }, + Tests: []junit.Test{}, + }, + }, + }, + "match all suites and tests": { + ignoreList: []ignoreTestCase{ + {Name: "remove", Classname: "it"}, + {Name: "why", Classname: "me"}, + }, + suites: []junit.Suite{ + { + Suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "remove", + Classname: "it", + }, + }, + }, + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "why", + Classname: "me", + }, + }, + }, + }, + Tests: []junit.Test{ + { + Name: "remove", + Classname: "it", + }, + { + Name: "why", + Classname: "me", + }, + }, + }, + }, + expectedSuites: []junit.Suite{}, + }, + "match empty name and classname": { + ignoreList: []ignoreTestCase{ + {Name: "only-name", Classname: ""}, + {Name: "", Classname: "only-class"}, + }, + suites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "only-name", + Classname: "", + }, + { + Name: "only-name", + Classname: "only-class", + }, + { + Name: "", + Classname: "only-class", + }, + }, + }, + }, + expectedSuites: []junit.Suite{ + { + Suites: []junit.Suite{}, + Tests: []junit.Test{ + { + Name: "only-name", + Classname: "only-class", + }, + }, + }, + }, + }, + } + + for testName, tt := range tests { + t.Run(testName, func(t *testing.T) { + ignoreList = tt.ignoreList + actual := getClearedSuites(tt.suites) + assert.ElementsMatch(t, tt.expectedSuites, actual) + }) + } +} + +func Test_addFallbacks(t *testing.T) { + tests := map[string]struct { + tests []junit.Test + expectedTest []junit.Test + }{ + "empty": { + tests: []junit.Test{}, + expectedTest: []junit.Test{}, + }, + "mysterious test case": { + tests: []junit.Test{ + { + Name: "", + Classname: "", + }, + }, + expectedTest: []junit.Test{ + { + Name: fallbackName, + Classname: fallbackClassname, + }, + }, + }, + "add name": { + tests: []junit.Test{ + { + Name: "", + Classname: "no name", + }, + }, + expectedTest: []junit.Test{ + { + Name: fallbackName, + Classname: "no name", + }, + }, + }, + "add classname": { + tests: []junit.Test{ + { + Name: "no class", + Classname: "", + }, + }, + expectedTest: []junit.Test{ + { + Name: "no class", + Classname: fallbackClassname, + }, + }, + }, + "all good": { + tests: []junit.Test{ + { + Name: "with name", + Classname: "and class", + }, + }, + expectedTest: []junit.Test{ + { + Name: "with name", + Classname: "and class", + }, + }, + }, + } + + for testName, tt := range tests { + t.Run(testName, func(t *testing.T) { + actual := addFallbacks(tt.tests) + assert.ElementsMatch(t, tt.expectedTest, actual) + }) + } +}