From 5d541d01b5d9d173a6927faf038a781f845b1003 Mon Sep 17 00:00:00 2001 From: Ryan King Date: Tue, 12 Apr 2022 21:11:05 -0400 Subject: [PATCH] Fix scorecard XUnit output (#5652) Currently, scorecard's XUnit output does not conform to any XUnit schema. This makes the scorecard XUnit output conform with the XUnit schema defined by the Jenkins XUnit plugin. This is sample output from running scorecard against the sample Go v3 Memcached operator: ```xml time="2022-04-12T19:21:52Z" level=debug msg="Found manifests directory" name=bundle-test time="2022-04-12T19:21:52Z" level=debug msg="Found metadata directory" name=bundle-test time="2022-04-12T19:21:52Z" level=debug msg="Getting mediaType info from manifests directory" name=bundle-test time="2022-04-12T19:21:52Z" level=debug msg="Found annotations file" name=bundle-test time="2022-04-12T19:21:52Z" level=debug msg="Could not find optional dependencies file" name=bundle-test ``` The full output can be recreated by installing the `operator-sdk` tool built from this commit and running scorecard against the sample Go v3 memcached operator: ``` cd /path/to/operator-sdk make install operator-sdk scorecard testdata/go/v3/memcached-operator ``` The XUnit XSD schema can be found here: https://github.com/jenkinsci/xunit-plugin/blob/master/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd The tool used to validate the output against the schema can be found here: https://www.freeformatter.com/xml-validator-xsd.html Signed-off-by: Ryan King --- internal/cmd/operator-sdk/scorecard/cmd.go | 58 ++++---- .../cmd/operator-sdk/scorecard/xunit/xunit.go | 132 +++++++++++------- 2 files changed, 117 insertions(+), 73 deletions(-) diff --git a/internal/cmd/operator-sdk/scorecard/cmd.go b/internal/cmd/operator-sdk/scorecard/cmd.go index e4b27246549..df0253b30ec 100644 --- a/internal/cmd/operator-sdk/scorecard/cmd.go +++ b/internal/cmd/operator-sdk/scorecard/cmd.go @@ -130,37 +130,43 @@ func (c *scorecardCmd) printOutput(output v1alpha3.TestList) error { } func (c *scorecardCmd) convertXunit(output v1alpha3.TestList) xunit.TestSuites { - var resultSuite xunit.TestSuites - resultSuite.Name = "scorecard" + const ( + imagePropName = "spec.image" + entrypointPropName = "spec.entrypoint" + testPropName = "labels.test" + clusterPhasePropName = "labels.cluster-phase" + ) - jsonTestItems := output.Items - for _, item := range jsonTestItems { - tempResults := item.Status.Results - for _, res := range tempResults { - var tCase xunit.TestCase - var tSuite xunit.TestSuite - tSuite.Name = res.Name - tCase.Name = res.Name - tCase.Classname = "scorecard" - if res.State == v1alpha3.ErrorState { - tCase.Errors = append(tCase.Errors, xunit.XUnitComplexError{Type: "Error", Message: strings.Join(res.Errors, ",")}) - tSuite.Errors = strings.Join(res.Errors, ",") - } else if res.State == v1alpha3.FailState { - tCase.Failures = append(tCase.Failures, xunit.XUnitComplexFailure{Type: "Failure", Message: res.Log}) - tSuite.Failures = res.Log - } - tSuite.TestCases = append(tSuite.TestCases, tCase) - tSuite.URL = item.Spec.Image - if item.Spec.UniqueID != "" { - tSuite.ID = item.Spec.UniqueID - } else { - tSuite.ID = res.Name + suites := make([]xunit.TestSuite, 0, len(output.Items)) + for i, item := range output.Items { + suiteName, ok := item.Spec.Labels["test"] + if !ok { + suiteName = fmt.Sprintf("testsuite-%03d", i+1) + } + + ts := xunit.NewSuite(suiteName) + ts.AddProperty(imagePropName, item.Spec.Image) + ts.AddProperty(entrypointPropName, strings.Join(item.Spec.Entrypoint, " ")) + ts.AddProperty(testPropName, suiteName) + if phase, ok := item.Spec.Labels["cluster-phase"]; ok { + ts.AddProperty(clusterPhasePropName, phase) + } + + for _, tc := range item.Status.Results { + switch tc.State { + case v1alpha3.PassState: + ts.AddSuccess(tc.Name, tc.CreationTimestamp.Time, tc.Log) + case v1alpha3.FailState: + ts.AddFailure(tc.Name, tc.CreationTimestamp.Time, tc.Log, strings.Join(tc.Errors, "\n")) + case v1alpha3.ErrorState: + ts.AddError(tc.Name, tc.CreationTimestamp.Time, tc.Log, strings.Join(tc.Errors, "\n")) } - resultSuite.TestSuite = append(resultSuite.TestSuite, tSuite) } + + suites = append(suites, ts) } - return resultSuite + return xunit.NewTestSuites("scorecard", suites) } func (c *scorecardCmd) run() (err error) { diff --git a/internal/cmd/operator-sdk/scorecard/xunit/xunit.go b/internal/cmd/operator-sdk/scorecard/xunit/xunit.go index 5ce5e877d8d..54cfa3ea54f 100644 --- a/internal/cmd/operator-sdk/scorecard/xunit/xunit.go +++ b/internal/cmd/operator-sdk/scorecard/xunit/xunit.go @@ -14,63 +14,101 @@ package xunitapi -// TestCase contain the core information from a test run, including its name and status -type TestCase struct { - // Name is the name of the test - Name string `xml:"name,attr,omitempty"` - Time string `xml:"time,attr,omitempty"` - Classname string `xml:"classname,attr,omitempty"` - Group string `xml:"group,attr,omitempty"` - Failures []XUnitComplexFailure `xml:"failure,omitempty"` - Errors []XUnitComplexError `xml:"error,omitempty"` - Skipped []XUnitComplexSkipped `xml:"skipped,omitempty"` +import ( + "encoding/xml" + "time" +) + +// NewTestSuites returns a new XUnit result from the given test suites. +func NewTestSuites(name string, testSuites []TestSuite) TestSuites { + return TestSuites{ + Name: name, + TestSuites: testSuites, + } +} + +// TestSuites is the top level object for amassing Xunit test results +type TestSuites struct { + XMLName xml.Name `xml:"testsuites"` // Component name: + Name string `xml:"name,attr"` + TestSuites []TestSuite `xml:"testsuite"` +} + +// Preperty is a named property that will be formatted as an XML tag. +type Property struct { + Name string `xml:"name,attr"` + Value interface{} `xml:"value,attr"` } // TestSuite contains for details about a test beyond the final status type TestSuite struct { - // Name is the name of the test - Name string `xml:"name,attr,omitempty"` - Tests string `xml:"tests,attr,omitempty"` - Failures string `xml:"failures,attr,omitempty"` - Errors string `xml:"errors,attr,omitempty"` - Group string `xml:"group,attr,omitempty"` - Skipped string `xml:"skipped,attr,omitempty"` - Timestamp string `xml:"timestamp,attr,omitempty"` - Hostname string `xml:"hostnames,attr,omitempty"` - ID string `xml:"id,attr,omitempty"` - Package string `xml:"package,attr,omitempty"` - File string `xml:"file,attr,omitempty"` - Log string `xml:"log,attr,omitempty"` - URL string `xml:"url,attr,omitempty"` - Version string `xml:"version,attr,omitempty"` - TestSuites []TestSuite `xml:"testsuite,omitempty"` - TestCases []TestCase `xml:"testcase,omitempty"` + Name string `xml:"name,attr"` + Properties struct { + Properties []Property `xml:"property"` + } `xml:"properties,omitempty"` + TestCases []TestCase `xml:"testcase,omitempty"` + Tests int `xml:"tests,attr"` + Skipped int `xml:"skipped,attr"` + Failures int `xml:"failures,attr"` + Errors int `xml:"errors,attr"` } -// TestSuites is the top level object for amassing Xunit test results -type TestSuites struct { - // Name is the name of the test - Name string `xml:"name,attr,omitempty"` - Tests string `xml:"tests,attr,omitempty"` - Failures string `xml:"failures,attr,omitempty"` - Errors string `xml:"errors,attr,omitempty"` - TestSuite []TestSuite `xml:"testsuite,omitempty"` +// NewSuite creates a new test suite with the given name. +func NewSuite(name string) TestSuite { + return TestSuite{Name: name} } -// XUnitComplexError contains a type header along with the error messages -type XUnitComplexError struct { - Type string `xml:"type,attr,omitempty"` - Message string `xml:"message,attr,omitempty"` +// AddProperty adds the property key/value to the test suite. +func (ts *TestSuite) AddProperty(name, value string) { + ts.Properties.Properties = append(ts.Properties.Properties, Property{Name: name, Value: value}) } -// XUnitComplexFailure contains a type header along with the failure logs -type XUnitComplexFailure struct { - Type string `xml:"type,attr,omitempty"` - Message string `xml:"message,attr,omitempty"` +// AddSuccess adds a passing test case to the suite. +func (ts *TestSuite) AddSuccess(name string, time time.Time, logs string) { + ts.addTest(name, time, logs, nil) +} + +// AddFailure adds a failed test case to the suite. +func (ts *TestSuite) AddFailure(name string, time time.Time, logs, msg string) { + ts.Failures++ + ts.addTest(name, time, logs, &Result{ + Name: xml.Name{Local: "failure"}, + Type: "failure", + Message: msg, + }) +} + +// AddError adds an errored test case to the suite. +func (ts *TestSuite) AddError(name string, time time.Time, logs, msg string) { + ts.Errors++ + ts.addTest(name, time, logs, &Result{ + Name: xml.Name{Local: "error"}, + Type: "error", + Message: msg, + }) +} + +func (ts *TestSuite) addTest(name string, time time.Time, logs string, result *Result) { + ts.Tests++ + ts.TestCases = append(ts.TestCases, TestCase{ + Name: name, + Time: time, + SystemOut: logs, + Result: result, + }) +} + +// TestCase contains information about an individual test case. +type TestCase struct { + Name string `xml:"name,attr"` + Time time.Time `xml:"time,attr"` + SystemOut string `xml:"system-out"` + Result *Result `xml:",omitempty"` } -// XUnitComplexSkipped contains a type header along with associated run logs -type XUnitComplexSkipped struct { - Type string `xml:"type,attr,omitempty"` - Message string `xml:"message,attr,omitempty"` +// Result represents the final state of the test case. +type Result struct { + Name xml.Name + Type string + Message string }