Skip to content

Commit

Permalink
Add Details map to post-processed Items (#838)
Browse files Browse the repository at this point in the history 
Currently just used for junit and specifically meant to address
a gap in necessary capabilities for processing e2e results.

When tests fail, you want to see what tests fail but also why they
failed. Without this functionality you still have to resort to
untarring the entire tarball just to see basic logs like "timeout"

Fixes #827

Signed-off-by: John Schnake <[email protected]>
  • Loading branch information
@johnSchnake
johnSchnake authored Aug 15, 2019
1 parent 0fb9865 commit 5238cfa
Show file tree
Hide file tree
Showing 9 changed files with 37 additions and 9 deletions.
11 changes: 9 additions & 2 deletions cmd/sonobuoy/app/results.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,16 @@ import (
)

const (
resultModeReport = "report"
// resultModeReport prints a human-readable summary of the results to stdout.
resultModeReport = "report"

// resultModeDetailed will dump each leaf node (e.g. test) as a json object. If the results
// are just references to files (like systemd-logs) then it will print the file for each
// leaf node, prefixed with the path.
resultModeDetailed = "detailed"
resultModeDump = "dump"

// resultModeDump will just copy the post-processed yaml file to stdout.
resultModeDump = "dump"
)

type resultsInput struct {
Expand Down
22 changes: 21 additions & 1 deletion pkg/client/results/junit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,14 @@ import (
"github.com/sirupsen/logrus"
)

const (
// JunitStdoutKey is the key in the Items.Details map for the system-out output.
JunitStdoutKey = "system-out"

// JunitFailureKey is the key in the Items.Details map for the failure output.
JunitFailureKey = "failure"
)

// Filter keeps only the tests that match the predicate function.
func Filter(predicate func(testCase reporters.JUnitTestCase) bool, testSuite reporters.JUnitTestSuite) []reporters.JUnitTestCase {
out := make([]reporters.JUnitTestCase, 0)
Expand Down Expand Up @@ -100,8 +108,20 @@ func processJunitFile(pluginDir, currentFile string) (Item, error) {
case Skipped(t):
status = StatusSkipped
}
testItem := Item{Name: t.Name, Status: status}

hasFailureMsg := (t.FailureMessage != nil && t.FailureMessage.Message != "")
if hasFailureMsg || t.SystemOut != "" {
testItem.Details = map[string]string{}
}
if hasFailureMsg {
testItem.Details[JunitFailureKey] = t.FailureMessage.Message
}
if t.SystemOut != "" {
testItem.Details[JunitStdoutKey] = t.SystemOut
}

resultObj.Items = append(resultObj.Items, Item{Name: t.Name, Status: status})
resultObj.Items = append(resultObj.Items, testItem)
}

return resultObj, nil
Expand Down
1 change: 1 addition & 0 deletions pkg/client/results/processing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ type Item struct {
Name string `json:"name" yaml:"name"`
Status string `json:"status" yaml:"status"`
Metadata map[string]string `json:"meta,omitempty" yaml:"meta,omitempty"`
Details map[string]string `json:"details,omitempty" yaml:"details,omitempty"`
Items []Item `json:"items,omitempty" yaml:"items,omitempty"`
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/client/results/testdata/mockResults/plugins/ds-junit-02/ds-junit-02.golden.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"name":"ds-junit-02","status":"failed","items":[{"name":"global","status":"failed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-node] ConfigMap should fail to create ConfigMap with empty key [Conformance]","status":"passed"},{"name":"[sig-storage] Downward API volume should set DefaultMode on files [LinuxOnly] [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]},{"name":"output2.xml","status":"failed","meta":{"file":"results/global/output2.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-apps] Daemon set [Serial] should rollback without unnecessary restarts [Conformance]","status":"failed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]}]}]}
{"name":"ds-junit-02","status":"failed","items":[{"name":"global","status":"failed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-node] ConfigMap should fail to create ConfigMap with empty key [Conformance]","status":"passed"},{"name":"[sig-storage] Downward API volume should set DefaultMode on files [LinuxOnly] [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]},{"name":"output2.xml","status":"failed","meta":{"file":"results/global/output2.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-apps] Daemon set [Serial] should rollback without unnecessary restarts [Conformance]","status":"failed","details":{"failure":"/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:696\nConformance test suite needs a cluster with at least 2 nodes.\nExpected\n \u003cint\u003e: 1\nto be \u003e\n \u003cint\u003e: 1\n/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/daemon_set.go:385","system-out":"[BeforeEach] ..."}},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]}]}]}
2 changes: 1 addition & 1 deletion pkg/client/results/testdata/mockResults/plugins/ds-junit-03/ds-junit-03.golden.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"name":"ds-junit-03","status":"failed","items":[{"name":"global","status":"failed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-node] ConfigMap should fail to create ConfigMap with empty key [Conformance]","status":"passed"},{"name":"[sig-storage] Downward API volume should set DefaultMode on files [LinuxOnly] [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]},{"name":"output2.xml","status":"failed","meta":{"file":"results/global/output2.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-apps] Daemon set [Serial] should rollback without unnecessary restarts [Conformance]","status":"failed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]}]}]}
{"name":"ds-junit-03","status":"failed","items":[{"name":"global","status":"failed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-node] ConfigMap should fail to create ConfigMap with empty key [Conformance]","status":"passed"},{"name":"[sig-storage] Downward API volume should set DefaultMode on files [LinuxOnly] [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]},{"name":"output2.xml","status":"failed","meta":{"file":"results/global/output2.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-apps] Daemon set [Serial] should rollback without unnecessary restarts [Conformance]","status":"failed","details":{"failure":"/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:696\nConformance test suite needs a cluster with at least 2 nodes.\nExpected\n \u003cint\u003e: 1\nto be \u003e\n \u003cint\u003e: 1\n/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/daemon_set.go:385","system-out":"[BeforeEach] ..."}},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]}]}]}
2 changes: 1 addition & 1 deletion pkg/client/results/testdata/mockResults/plugins/ds-raw-03/ds-raw-03.golden.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"name":"ds-raw-03","status":"passed","items":[{"name":"global","status":"passed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"}},{"name":"output2.xml","status":"passed","meta":{"file":"results/global/output2.xml"}}]}]}
{"name":"ds-raw-03","status":"passed","items":[{"name":"global","status":"passed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"}},{"name":"output2.xml","status":"passed","meta":{"file":"results/global/output2.xml"}}]}]}
2 changes: 1 addition & 1 deletion pkg/client/results/testdata/mockResults/plugins/job-junit-02/job-junit-02.golden.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"name":"job-junit-02","status":"failed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-node] ConfigMap should fail to create ConfigMap with empty key [Conformance]","status":"passed"},{"name":"[sig-storage] Downward API volume should set DefaultMode on files [LinuxOnly] [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]},{"name":"output2.xml","status":"failed","meta":{"file":"results/global/output2.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-apps] Daemon set [Serial] should rollback without unnecessary restarts [Conformance]","status":"failed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]}]}
{"name":"job-junit-02","status":"failed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-node] ConfigMap should fail to create ConfigMap with empty key [Conformance]","status":"passed"},{"name":"[sig-storage] Downward API volume should set DefaultMode on files [LinuxOnly] [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]},{"name":"output2.xml","status":"failed","meta":{"file":"results/global/output2.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-apps] Daemon set [Serial] should rollback without unnecessary restarts [Conformance]","status":"failed","details":{"failure":"/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:696\nConformance test suite needs a cluster with at least 2 nodes.\nExpected\n \u003cint\u003e: 1\nto be \u003e\n \u003cint\u003e: 1\n/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/daemon_set.go:385","system-out":"[BeforeEach] ..."}},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]}]}
2 changes: 1 addition & 1 deletion pkg/client/results/testdata/mockResults/plugins/job-junit-03/job-junit-03.golden.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"name":"job-junit-03","status":"failed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-node] ConfigMap should fail to create ConfigMap with empty key [Conformance]","status":"passed"},{"name":"[sig-storage] Downward API volume should set DefaultMode on files [LinuxOnly] [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]},{"name":"output2.xml","status":"failed","meta":{"file":"results/global/output2.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-apps] Daemon set [Serial] should rollback without unnecessary restarts [Conformance]","status":"failed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]}]}
{"name":"job-junit-03","status":"failed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-node] ConfigMap should fail to create ConfigMap with empty key [Conformance]","status":"passed"},{"name":"[sig-storage] Downward API volume should set DefaultMode on files [LinuxOnly] [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]},{"name":"output2.xml","status":"failed","meta":{"file":"results/global/output2.xml"},"items":[{"name":"[k8s.io] Pods should be submitted and removed [NodeConformance] [Conformance]","status":"passed"},{"name":"[sig-apps] Daemon set [Serial] should rollback without unnecessary restarts [Conformance]","status":"failed","details":{"failure":"/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:696\nConformance test suite needs a cluster with at least 2 nodes.\nExpected\n \u003cint\u003e: 1\nto be \u003e\n \u003cint\u003e: 1\n/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/daemon_set.go:385","system-out":"[BeforeEach] ..."}},{"name":"[sig-storage] In-tree Volumes [Driver: local][LocalVolumeType: dir-link-bindmounted] [Testpattern: Dynamic PV (default fs)] subPath should support existing directories when readOnly specified in the volumeSource","status":"skipped"},{"name":"[sig-storage] In-tree Volumes [Driver: rbd][Feature:Volumes] [Testpattern: Pre-provisioned PV (default fs)] subPath should support restarting containers using file as subpath [Slow]","status":"skipped"}]}]}
2 changes: 1 addition & 1 deletion pkg/client/results/testdata/mockResults/plugins/job-raw-03/job-raw-03.golden.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"name":"job-raw-03","status":"passed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"}},{"name":"output2.xml","status":"passed","meta":{"file":"results/global/output2.xml"}}]}
{"name":"job-raw-03","status":"passed","items":[{"name":"output.xml","status":"passed","meta":{"file":"results/global/output.xml"}},{"name":"output2.xml","status":"passed","meta":{"file":"results/global/output2.xml"}}]}

0 comments on commit 5238cfa

Please sign in to comment.