Skip to content

Commit

Permalink
Fix scorecard XUnit output (#5652)
Browse files Browse the repository at this point in the history 
Currently, scorecard's XUnit output does not conform to any XUnit
schema. This makes the scorecard XUnit output conform with the XUnit
schema defined by the Jenkins XUnit plugin.

This is sample output from running scorecard against the sample Go v3
Memcached operator:

```xml
<testsuites name="scorecard">
  <testsuite name="olm-bundle-validation-test" tests="1" skipped="0" failures="0" errors="0">
    <properties>
      <property name="spec.image" value="quay.io/operator-framework/scorecard-test:v1.19.0"></property>
      <property name="spec.entrypoint" value="scorecard-test olm-bundle-validation"></property>
      <property name="labels.test" value="olm-bundle-validation-test"></property>
    </properties>
    <testcase name="olm-bundle-validation" time="0001-01-01T00:00:00Z">
      <system-out>time=&#34;2022-04-12T19:21:52Z&#34; level=debug msg=&#34;Found manifests directory&#34; name=bundle-test&#xA;time=&#34;2022-04-12T19:21:52Z&#34; level=debug msg=&#34;Found metadata directory&#34; name=bundle-test&#xA;time=&#34;2022-04-12T19:21:52Z&#34; level=debug msg=&#34;Getting mediaType info from manifests directory&#34; name=bundle-test&#xA;time=&#34;2022-04-12T19:21:52Z&#34; level=debug msg=&#34;Found annotations file&#34; name=bundle-test&#xA;time=&#34;2022-04-12T19:21:52Z&#34; level=debug msg=&#34;Could not find optional dependencies file&#34; name=bundle-test&#xA;</system-out>
    </testcase>
  </testsuite>
  <!-- Some suites omitted for readability -->
</testsuites>
```

The full output can be recreated by installing the `operator-sdk` tool
built from this commit and running scorecard against the sample Go v3
memcached operator:

```
cd /path/to/operator-sdk
make install
operator-sdk scorecard testdata/go/v3/memcached-operator
```

The XUnit XSD schema can be found here:
https://github.com/jenkinsci/xunit-plugin/blob/master/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd

The tool used to validate the output against the schema can be found
here: https://www.freeformatter.com/xml-validator-xsd.html

Signed-off-by: Ryan King <[email protected]>
  • Loading branch information
@ryantking
ryantking authored Apr 13, 2022
1 parent e3276b8 commit 5d541d0
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 73 deletions.
58 changes: 32 additions & 26 deletions internal/cmd/operator-sdk/scorecard/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,37 +130,43 @@ func (c *scorecardCmd) printOutput(output v1alpha3.TestList) error {
}

func (c *scorecardCmd) convertXunit(output v1alpha3.TestList) xunit.TestSuites {
var resultSuite xunit.TestSuites
resultSuite.Name = "scorecard"
const (
imagePropName = "spec.image"
entrypointPropName = "spec.entrypoint"
testPropName = "labels.test"
clusterPhasePropName = "labels.cluster-phase"
)

jsonTestItems := output.Items
for _, item := range jsonTestItems {
tempResults := item.Status.Results
for _, res := range tempResults {
var tCase xunit.TestCase
var tSuite xunit.TestSuite
tSuite.Name = res.Name
tCase.Name = res.Name
tCase.Classname = "scorecard"
if res.State == v1alpha3.ErrorState {
tCase.Errors = append(tCase.Errors, xunit.XUnitComplexError{Type: "Error", Message: strings.Join(res.Errors, ",")})
tSuite.Errors = strings.Join(res.Errors, ",")
} else if res.State == v1alpha3.FailState {
tCase.Failures = append(tCase.Failures, xunit.XUnitComplexFailure{Type: "Failure", Message: res.Log})
tSuite.Failures = res.Log
}
tSuite.TestCases = append(tSuite.TestCases, tCase)
tSuite.URL = item.Spec.Image
if item.Spec.UniqueID != "" {
tSuite.ID = item.Spec.UniqueID
} else {
tSuite.ID = res.Name
suites := make([]xunit.TestSuite, 0, len(output.Items))
for i, item := range output.Items {
suiteName, ok := item.Spec.Labels["test"]
if !ok {
suiteName = fmt.Sprintf("testsuite-%03d", i+1)
}

ts := xunit.NewSuite(suiteName)
ts.AddProperty(imagePropName, item.Spec.Image)
ts.AddProperty(entrypointPropName, strings.Join(item.Spec.Entrypoint, " "))
ts.AddProperty(testPropName, suiteName)
if phase, ok := item.Spec.Labels["cluster-phase"]; ok {
ts.AddProperty(clusterPhasePropName, phase)
}

for _, tc := range item.Status.Results {
switch tc.State {
case v1alpha3.PassState:
ts.AddSuccess(tc.Name, tc.CreationTimestamp.Time, tc.Log)
case v1alpha3.FailState:
ts.AddFailure(tc.Name, tc.CreationTimestamp.Time, tc.Log, strings.Join(tc.Errors, "\n"))
case v1alpha3.ErrorState:
ts.AddError(tc.Name, tc.CreationTimestamp.Time, tc.Log, strings.Join(tc.Errors, "\n"))
}
resultSuite.TestSuite = append(resultSuite.TestSuite, tSuite)
}

suites = append(suites, ts)
}

return resultSuite
return xunit.NewTestSuites("scorecard", suites)
}

func (c *scorecardCmd) run() (err error) {
Expand Down
132 changes: 85 additions & 47 deletions internal/cmd/operator-sdk/scorecard/xunit/xunit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,63 +14,101 @@

package xunitapi

// TestCase contain the core information from a test run, including its name and status
type TestCase struct {
// Name is the name of the test
Name string `xml:"name,attr,omitempty"`
Time string `xml:"time,attr,omitempty"`
Classname string `xml:"classname,attr,omitempty"`
Group string `xml:"group,attr,omitempty"`
Failures []XUnitComplexFailure `xml:"failure,omitempty"`
Errors []XUnitComplexError `xml:"error,omitempty"`
Skipped []XUnitComplexSkipped `xml:"skipped,omitempty"`
import (
"encoding/xml"
"time"
)

// NewTestSuites returns a new XUnit result from the given test suites.
func NewTestSuites(name string, testSuites []TestSuite) TestSuites {
return TestSuites{
Name: name,
TestSuites: testSuites,
}
}

// TestSuites is the top level object for amassing Xunit test results
type TestSuites struct {
XMLName xml.Name `xml:"testsuites"` // Component name: <testsuites>
Name string `xml:"name,attr"`
TestSuites []TestSuite `xml:"testsuite"`
}

// Preperty is a named property that will be formatted as an XML tag.
type Property struct {
Name string `xml:"name,attr"`
Value interface{} `xml:"value,attr"`
}

// TestSuite contains for details about a test beyond the final status
type TestSuite struct {
// Name is the name of the test
Name string `xml:"name,attr,omitempty"`
Tests string `xml:"tests,attr,omitempty"`
Failures string `xml:"failures,attr,omitempty"`
Errors string `xml:"errors,attr,omitempty"`
Group string `xml:"group,attr,omitempty"`
Skipped string `xml:"skipped,attr,omitempty"`
Timestamp string `xml:"timestamp,attr,omitempty"`
Hostname string `xml:"hostnames,attr,omitempty"`
ID string `xml:"id,attr,omitempty"`
Package string `xml:"package,attr,omitempty"`
File string `xml:"file,attr,omitempty"`
Log string `xml:"log,attr,omitempty"`
URL string `xml:"url,attr,omitempty"`
Version string `xml:"version,attr,omitempty"`
TestSuites []TestSuite `xml:"testsuite,omitempty"`
TestCases []TestCase `xml:"testcase,omitempty"`
Name string `xml:"name,attr"`
Properties struct {
Properties []Property `xml:"property"`
} `xml:"properties,omitempty"`
TestCases []TestCase `xml:"testcase,omitempty"`
Tests int `xml:"tests,attr"`
Skipped int `xml:"skipped,attr"`
Failures int `xml:"failures,attr"`
Errors int `xml:"errors,attr"`
}

// TestSuites is the top level object for amassing Xunit test results
type TestSuites struct {
// Name is the name of the test
Name string `xml:"name,attr,omitempty"`
Tests string `xml:"tests,attr,omitempty"`
Failures string `xml:"failures,attr,omitempty"`
Errors string `xml:"errors,attr,omitempty"`
TestSuite []TestSuite `xml:"testsuite,omitempty"`
// NewSuite creates a new test suite with the given name.
func NewSuite(name string) TestSuite {
return TestSuite{Name: name}
}

// XUnitComplexError contains a type header along with the error messages
type XUnitComplexError struct {
Type string `xml:"type,attr,omitempty"`
Message string `xml:"message,attr,omitempty"`
// AddProperty adds the property key/value to the test suite.
func (ts *TestSuite) AddProperty(name, value string) {
ts.Properties.Properties = append(ts.Properties.Properties, Property{Name: name, Value: value})
}

// XUnitComplexFailure contains a type header along with the failure logs
type XUnitComplexFailure struct {
Type string `xml:"type,attr,omitempty"`
Message string `xml:"message,attr,omitempty"`
// AddSuccess adds a passing test case to the suite.
func (ts *TestSuite) AddSuccess(name string, time time.Time, logs string) {
ts.addTest(name, time, logs, nil)
}

// AddFailure adds a failed test case to the suite.
func (ts *TestSuite) AddFailure(name string, time time.Time, logs, msg string) {
ts.Failures++
ts.addTest(name, time, logs, &Result{
Name: xml.Name{Local: "failure"},
Type: "failure",
Message: msg,
})
}

// AddError adds an errored test case to the suite.
func (ts *TestSuite) AddError(name string, time time.Time, logs, msg string) {
ts.Errors++
ts.addTest(name, time, logs, &Result{
Name: xml.Name{Local: "error"},
Type: "error",
Message: msg,
})
}

func (ts *TestSuite) addTest(name string, time time.Time, logs string, result *Result) {
ts.Tests++
ts.TestCases = append(ts.TestCases, TestCase{
Name: name,
Time: time,
SystemOut: logs,
Result: result,
})
}

// TestCase contains information about an individual test case.
type TestCase struct {
Name string `xml:"name,attr"`
Time time.Time `xml:"time,attr"`
SystemOut string `xml:"system-out"`
Result *Result `xml:",omitempty"`
}

// XUnitComplexSkipped contains a type header along with associated run logs
type XUnitComplexSkipped struct {
Type string `xml:"type,attr,omitempty"`
Message string `xml:"message,attr,omitempty"`
// Result represents the final state of the test case.
type Result struct {
Name xml.Name
Type string
Message string
}

0 comments on commit 5d541d0

Please sign in to comment.