Skip to content

Commit

Permalink
Run unit and integration tests with gotestsum (elastic#22541)
Browse files Browse the repository at this point in the history 
The PR introduces gotestsum to drive the go based unit and integration tests. This is meant as replacement for github.com/jstemmer/go-junit-report, which seems to be unmaintained for a while.

Only the GoTest mage helper is modified to use gotestsum instead of go-junit-reporter. In addition to the change we add unit tests for GoTest that do run actual tests. The output of these helper-tests is captured and validated against known regular expressions.

(cherry picked from commit 2eb3c29)
  • Loading branch information
Steffen Siering committed Jan 14, 2021
1 parent bf0a64a commit 37dc89a
Show file tree
Hide file tree
Showing 12 changed files with 994 additions and 260 deletions.
2 changes: 1 addition & 1 deletion .ci/scripts/install-tools.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,4 @@ gcc --version
where gcc

REM Reset the USERPROFILE
SET USERPROFILE=%PREVIOUS_USERPROFILE%
SET USERPROFILE=%PREVIOUS_USERPROFILE%
523 changes: 485 additions & 38 deletions NOTICE.txt
View file

Large diffs are not rendered by default.

232 changes: 62 additions & 170 deletions dev-tools/mage/gotest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package mage

import (
"bytes"
"context"
"fmt"
"io"
Expand All @@ -28,16 +27,14 @@ import (
"os/exec"
"path"
"path/filepath"
"runtime"
"sort"
"strings"
"time"

"github.com/jstemmer/go-junit-report/formatter"
"github.com/jstemmer/go-junit-report/parser"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/pkg/errors"

"github.com/elastic/beats/v7/dev-tools/mage/gotool"
)

// GoTestArgs are the arguments used for the "go*Test" targets and they define
Expand All @@ -52,6 +49,7 @@ type GoTestArgs struct {
OutputFile string // File to write verbose test output to.
JUnitReportFile string // File to write a JUnit XML test report to.
CoverageProfileFile string // Test coverage profile file (enables -cover).
Output io.Writer // Write stderr and stdout to Output if set
}

// TestBinaryArgs are the arguments used when building binary for testing.
Expand Down Expand Up @@ -183,40 +181,80 @@ func GoTestIntegrationForModule(ctx context.Context) error {
return nil
}

// InstallGoTestTools installs additional tools that are required to run unit and integration tests.
func InstallGoTestTools() {
gotool.Install(
gotool.Install.Package("gotest.tools/gotestsum"),
)
}

// GoTest invokes "go test" and reports the results to stdout. It returns an
// error if there was any failure executing the tests or if there were any
// test failures.
func GoTest(ctx context.Context, params GoTestArgs) error {
mg.Deps(InstallGoTestTools)

fmt.Println(">> go test:", params.TestName, "Testing")

// Build args list to Go.
args := []string{"test"}
args = append(args, "-v")
// We use gotestsum to drive the tests and produce a junit report.
// The tool runs `go test -json` in order to produce a structured log which makes it easier
// to parse the actual test output.
// Of OutputFile is given the original JSON file will be written as well.
//
// The runner needs to set CLI flags for gotestsum and for "go test". We track the different
// CLI flags in the gotestsumArgs and testArgs variables, such that we can finally produce command like:
// $ gotestsum <gotestsum args> -- <go test args>
//
// The additional arguments given via GoTestArgs are applied to `go test` only. Callers can not
// modify any of the gotestsum arguments.

gotestsumArgs := []string{"--no-color"}
if mg.Verbose() {
gotestsumArgs = append(gotestsumArgs, "-f", "standard-verbose")
} else {
gotestsumArgs = append(gotestsumArgs, "-f", "standard-quiet")
}
if params.JUnitReportFile != "" {
CreateDir(params.JUnitReportFile)
gotestsumArgs = append(gotestsumArgs, "--junitfile", params.JUnitReportFile)
}
if params.OutputFile != "" {
CreateDir(params.OutputFile)
gotestsumArgs = append(gotestsumArgs, "--jsonfile", params.OutputFile+".json")
}

var testArgs []string

// -race is only supported on */amd64
if os.Getenv("DEV_ARCH") == "amd64" {
if params.Race {
args = append(args, "-race")
testArgs = append(testArgs, "-race")
}
}
if len(params.Tags) > 0 {
args = append(args, "-tags", strings.Join(params.Tags, " "))
params := strings.Join(params.Tags, " ")
if params != "" {
testArgs = append(testArgs, "-tags", params)
}
}
if params.CoverageProfileFile != "" {
params.CoverageProfileFile = createDir(filepath.Clean(params.CoverageProfileFile))
args = append(args,
testArgs = append(testArgs,
"-covermode=atomic",
"-coverprofile="+params.CoverageProfileFile,
)
}
args = append(args, params.ExtraFlags...)
args = append(args, params.Packages...)
testArgs = append(testArgs, params.ExtraFlags...)
testArgs = append(testArgs, params.Packages...)

goTest := makeCommand(ctx, params.Env, "go", args...)
args := append(gotestsumArgs, append([]string{"--"}, testArgs...)...)

goTest := makeCommand(ctx, params.Env, "gotestsum", args...)
// Wire up the outputs.
bufferOutput := new(bytes.Buffer)
outputs := []io.Writer{bufferOutput}
var outputs []io.Writer
if params.Output != nil {
outputs = append(outputs, params.Output)
}

if params.OutputFile != "" {
fileOutput, err := os.Create(createDir(params.OutputFile))
Expand All @@ -227,18 +265,16 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
outputs = append(outputs, fileOutput)
}
output := io.MultiWriter(outputs...)
goTest.Stdout = output
goTest.Stderr = output

if mg.Verbose() {
if params.Output == nil {
goTest.Stdout = io.MultiWriter(output, os.Stdout)
goTest.Stderr = io.MultiWriter(output, os.Stderr)
} else {
goTest.Stdout = output
goTest.Stderr = output
}

// Execute 'go test' and measure duration.
start := time.Now()
err := goTest.Run()
duration := time.Since(start)

var goTestErr *exec.ExitError
if err != nil {
// Command ran.
Expand All @@ -251,30 +287,11 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
goTestErr = exitErr
}

// Parse the verbose test output.
report, err := parser.Parse(bytes.NewBuffer(bufferOutput.Bytes()), BeatName)
if err != nil {
return errors.Wrap(err, "failed to parse go test output")
}
if goTestErr != nil && len(report.Packages) == 0 {
if goTestErr != nil {
// No packages were tested. Probably the code didn't compile.
fmt.Println(bytes.NewBuffer(bufferOutput.Bytes()).String())
return errors.Wrap(goTestErr, "go test returned a non-zero value")
}

// Generate a JUnit XML report.
if params.JUnitReportFile != "" {
junitReport, err := os.Create(createDir(params.JUnitReportFile))
if err != nil {
return errors.Wrap(err, "failed to create junit report")
}
defer junitReport.Close()

if err = formatter.JUnitReportXML(report, false, runtime.Version(), junitReport); err != nil {
return errors.Wrap(err, "failed to write junit report")
}
}

// Generate a HTML code coverage report.
var htmlCoverReport string
if params.CoverageProfileFile != "" {
Expand All @@ -288,27 +305,9 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
}
}

// Summarize the results and log to stdout.
summary, err := NewGoTestSummary(duration, report, map[string]string{
"Output File": params.OutputFile,
"JUnit Report": params.JUnitReportFile,
"Coverage Report": htmlCoverReport,
})
if err != nil {
return err
}
if !mg.Verbose() && summary.Fail > 0 {
fmt.Println(summary.Failures())
}
fmt.Println(summary.String())

// Return an error indicating that testing failed.
if summary.Fail > 0 || goTestErr != nil {
if goTestErr != nil {
fmt.Println(">> go test:", params.TestName, "Test Failed")
if summary.Fail > 0 {
return errors.Errorf("go test failed: %d test failures", summary.Fail)
}

return errors.Wrap(goTestErr, "go test returned a non-zero value")
}

Expand All @@ -329,117 +328,10 @@ func makeCommand(ctx context.Context, env map[string]string, cmd string, args ..
c.Stderr = os.Stderr
c.Stdin = os.Stdin
log.Println("exec:", cmd, strings.Join(args, " "))
fmt.Println("exec:", cmd, strings.Join(args, " "))
return c
}

// GoTestSummary is a summary of test results.
type GoTestSummary struct {
*parser.Report // Report generated by parsing test output.
Pass int // Number of passing tests.
Fail int // Number of failed tests.
Skip int // Number of skipped tests.
Packages int // Number of packages tested.
Duration time.Duration // Total go test running duration.
Files map[string]string
}

// NewGoTestSummary builds a new GoTestSummary. It returns an error if it cannot
// resolve the absolute paths to the given files.
func NewGoTestSummary(d time.Duration, r *parser.Report, outputFiles map[string]string) (*GoTestSummary, error) {
files := map[string]string{}
for name, file := range outputFiles {
if file == "" {
continue
}
absFile, err := filepath.Abs(file)
if err != nil {
return nil, errors.Wrapf(err, "failed resolving absolute path for %v", file)
}
files[name+":"] = absFile
}

summary := &GoTestSummary{
Report: r,
Duration: d,
Packages: len(r.Packages),
Files: files,
}

for _, pkg := range r.Packages {
for _, t := range pkg.Tests {
switch t.Result {
case parser.PASS:
summary.Pass++
case parser.FAIL:
summary.Fail++
case parser.SKIP:
summary.Skip++
default:
return nil, errors.Errorf("Unknown test result value: %v", t.Result)
}
}
}

return summary, nil
}

// Failures returns a string containing the list of failed test cases and their
// output.
func (s *GoTestSummary) Failures() string {
b := new(strings.Builder)

if s.Fail > 0 {
fmt.Fprintln(b, "FAILURES:")
for _, pkg := range s.Report.Packages {
for _, t := range pkg.Tests {
if t.Result != parser.FAIL {
continue
}
fmt.Fprintln(b, "Package:", pkg.Name)
fmt.Fprintln(b, "Test: ", t.Name)
for _, line := range t.Output {
if strings.TrimSpace(line) != "" {
fmt.Fprintln(b, line)
}
}
fmt.Fprintln(b, "----")
}
}
}

return strings.TrimRight(b.String(), "\n")
}

// String returns a summary of the testing results (number of fail/pass/skip,
// test duration, number packages, output files).
func (s *GoTestSummary) String() string {
b := new(strings.Builder)

fmt.Fprintln(b, "SUMMARY:")
fmt.Fprintln(b, " Fail: ", s.Fail)
fmt.Fprintln(b, " Skip: ", s.Skip)
fmt.Fprintln(b, " Pass: ", s.Pass)
fmt.Fprintln(b, " Packages:", len(s.Report.Packages))
fmt.Fprintln(b, " Duration:", s.Duration)

// Sort the list of files and compute the column width.
var names []string
var nameWidth int
for name := range s.Files {
if len(name) > nameWidth {
nameWidth = len(name)
}
names = append(names, name)
}
sort.Strings(names)

for _, name := range names {
fmt.Fprintf(b, " %-*s %s\n", nameWidth, name, s.Files[name])
}

return strings.TrimRight(b.String(), "\n")
}

// BuildSystemTestBinary runs BuildSystemTestGoBinary with default values.
func BuildSystemTestBinary() error {
return BuildSystemTestGoBinary(DefaultTestBinaryArgs())
Expand Down
Loading

0 comments on commit 37dc89a

Please sign in to comment.