Skip to content

Commit

Permalink
Cherry-pick elastic#22541 to 7.x: Run unit and integration tests with…
Browse files Browse the repository at this point in the history 
… gotestsum (elastic#23512)
  • Loading branch information
Steffen Siering authored Jan 14, 2021
1 parent cbc3b75 commit b90b70e
Show file tree
Hide file tree
Showing 12 changed files with 994 additions and 260 deletions.
2 changes: 1 addition & 1 deletion .ci/scripts/install-tools.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,4 @@ gcc --version
where gcc

REM Reset the USERPROFILE
SET USERPROFILE=%PREVIOUS_USERPROFILE%
SET USERPROFILE=%PREVIOUS_USERPROFILE%
523 changes: 485 additions & 38 deletions NOTICE.txt
View file

Large diffs are not rendered by default.

232 changes: 62 additions & 170 deletions dev-tools/mage/gotest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package mage

import (
"bytes"
"context"
"fmt"
"io"
Expand All @@ -28,16 +27,14 @@ import (
"os/exec"
"path"
"path/filepath"
"runtime"
"sort"
"strings"
"time"

"github.com/jstemmer/go-junit-report/formatter"
"github.com/jstemmer/go-junit-report/parser"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/pkg/errors"

"github.com/elastic/beats/v7/dev-tools/mage/gotool"
)

// GoTestArgs are the arguments used for the "go*Test" targets and they define
Expand All @@ -52,6 +49,7 @@ type GoTestArgs struct {
OutputFile string // File to write verbose test output to.
JUnitReportFile string // File to write a JUnit XML test report to.
CoverageProfileFile string // Test coverage profile file (enables -cover).
Output io.Writer // Write stderr and stdout to Output if set
}

// TestBinaryArgs are the arguments used when building binary for testing.
Expand Down Expand Up @@ -183,40 +181,80 @@ func GoTestIntegrationForModule(ctx context.Context) error {
return nil
}

// InstallGoTestTools installs additional tools that are required to run unit and integration tests.
func InstallGoTestTools() {
gotool.Install(
gotool.Install.Package("gotest.tools/gotestsum"),
)
}

// GoTest invokes "go test" and reports the results to stdout. It returns an
// error if there was any failure executing the tests or if there were any
// test failures.
func GoTest(ctx context.Context, params GoTestArgs) error {
mg.Deps(InstallGoTestTools)

fmt.Println(">> go test:", params.TestName, "Testing")

// Build args list to Go.
args := []string{"test"}
args = append(args, "-v")
// We use gotestsum to drive the tests and produce a junit report.
// The tool runs `go test -json` in order to produce a structured log which makes it easier
// to parse the actual test output.
// Of OutputFile is given the original JSON file will be written as well.
//
// The runner needs to set CLI flags for gotestsum and for "go test". We track the different
// CLI flags in the gotestsumArgs and testArgs variables, such that we can finally produce command like:
// $ gotestsum <gotestsum args> -- <go test args>
//
// The additional arguments given via GoTestArgs are applied to `go test` only. Callers can not
// modify any of the gotestsum arguments.

gotestsumArgs := []string{"--no-color"}
if mg.Verbose() {
gotestsumArgs = append(gotestsumArgs, "-f", "standard-verbose")
} else {
gotestsumArgs = append(gotestsumArgs, "-f", "standard-quiet")
}
if params.JUnitReportFile != "" {
CreateDir(params.JUnitReportFile)
gotestsumArgs = append(gotestsumArgs, "--junitfile", params.JUnitReportFile)
}
if params.OutputFile != "" {
CreateDir(params.OutputFile)
gotestsumArgs = append(gotestsumArgs, "--jsonfile", params.OutputFile+".json")
}

var testArgs []string

// -race is only supported on */amd64
if os.Getenv("DEV_ARCH") == "amd64" {
if params.Race {
args = append(args, "-race")
testArgs = append(testArgs, "-race")
}
}
if len(params.Tags) > 0 {
args = append(args, "-tags", strings.Join(params.Tags, " "))
params := strings.Join(params.Tags, " ")
if params != "" {
testArgs = append(testArgs, "-tags", params)
}
}
if params.CoverageProfileFile != "" {
params.CoverageProfileFile = createDir(filepath.Clean(params.CoverageProfileFile))
args = append(args,
testArgs = append(testArgs,
"-covermode=atomic",
"-coverprofile="+params.CoverageProfileFile,
)
}
args = append(args, params.ExtraFlags...)
args = append(args, params.Packages...)
testArgs = append(testArgs, params.ExtraFlags...)
testArgs = append(testArgs, params.Packages...)

goTest := makeCommand(ctx, params.Env, "go", args...)
args := append(gotestsumArgs, append([]string{"--"}, testArgs...)...)

goTest := makeCommand(ctx, params.Env, "gotestsum", args...)
// Wire up the outputs.
bufferOutput := new(bytes.Buffer)
outputs := []io.Writer{bufferOutput}
var outputs []io.Writer
if params.Output != nil {
outputs = append(outputs, params.Output)
}

if params.OutputFile != "" {
fileOutput, err := os.Create(createDir(params.OutputFile))
Expand All @@ -227,18 +265,16 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
outputs = append(outputs, fileOutput)
}
output := io.MultiWriter(outputs...)
goTest.Stdout = output
goTest.Stderr = output

if mg.Verbose() {
if params.Output == nil {
goTest.Stdout = io.MultiWriter(output, os.Stdout)
goTest.Stderr = io.MultiWriter(output, os.Stderr)
} else {
goTest.Stdout = output
goTest.Stderr = output
}

// Execute 'go test' and measure duration.
start := time.Now()
err := goTest.Run()
duration := time.Since(start)

var goTestErr *exec.ExitError
if err != nil {
// Command ran.
Expand All @@ -251,30 +287,11 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
goTestErr = exitErr
}

// Parse the verbose test output.
report, err := parser.Parse(bytes.NewBuffer(bufferOutput.Bytes()), BeatName)
if err != nil {
return errors.Wrap(err, "failed to parse go test output")
}
if goTestErr != nil && len(report.Packages) == 0 {
if goTestErr != nil {
// No packages were tested. Probably the code didn't compile.
fmt.Println(bytes.NewBuffer(bufferOutput.Bytes()).String())
return errors.Wrap(goTestErr, "go test returned a non-zero value")
}

// Generate a JUnit XML report.
if params.JUnitReportFile != "" {
junitReport, err := os.Create(createDir(params.JUnitReportFile))
if err != nil {
return errors.Wrap(err, "failed to create junit report")
}
defer junitReport.Close()

if err = formatter.JUnitReportXML(report, false, runtime.Version(), junitReport); err != nil {
return errors.Wrap(err, "failed to write junit report")
}
}

// Generate a HTML code coverage report.
var htmlCoverReport string
if params.CoverageProfileFile != "" {
Expand All @@ -288,27 +305,9 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
}
}

// Summarize the results and log to stdout.
summary, err := NewGoTestSummary(duration, report, map[string]string{
"Output File": params.OutputFile,
"JUnit Report": params.JUnitReportFile,
"Coverage Report": htmlCoverReport,
})
if err != nil {
return err
}
if !mg.Verbose() && summary.Fail > 0 {
fmt.Println(summary.Failures())
}
fmt.Println(summary.String())

// Return an error indicating that testing failed.
if summary.Fail > 0 || goTestErr != nil {
if goTestErr != nil {
fmt.Println(">> go test:", params.TestName, "Test Failed")
if summary.Fail > 0 {
return errors.Errorf("go test failed: %d test failures", summary.Fail)
}

return errors.Wrap(goTestErr, "go test returned a non-zero value")
}

Expand All @@ -329,117 +328,10 @@ func makeCommand(ctx context.Context, env map[string]string, cmd string, args ..
c.Stderr = os.Stderr
c.Stdin = os.Stdin
log.Println("exec:", cmd, strings.Join(args, " "))
fmt.Println("exec:", cmd, strings.Join(args, " "))
return c
}

// GoTestSummary is a summary of test results.
type GoTestSummary struct {
*parser.Report // Report generated by parsing test output.
Pass int // Number of passing tests.
Fail int // Number of failed tests.
Skip int // Number of skipped tests.
Packages int // Number of packages tested.
Duration time.Duration // Total go test running duration.
Files map[string]string
}

// NewGoTestSummary builds a new GoTestSummary. It returns an error if it cannot
// resolve the absolute paths to the given files.
func NewGoTestSummary(d time.Duration, r *parser.Report, outputFiles map[string]string) (*GoTestSummary, error) {
files := map[string]string{}
for name, file := range outputFiles {
if file == "" {
continue
}
absFile, err := filepath.Abs(file)
if err != nil {
return nil, errors.Wrapf(err, "failed resolving absolute path for %v", file)
}
files[name+":"] = absFile
}

summary := &GoTestSummary{
Report: r,
Duration: d,
Packages: len(r.Packages),
Files: files,
}

for _, pkg := range r.Packages {
for _, t := range pkg.Tests {
switch t.Result {
case parser.PASS:
summary.Pass++
case parser.FAIL:
summary.Fail++
case parser.SKIP:
summary.Skip++
default:
return nil, errors.Errorf("Unknown test result value: %v", t.Result)
}
}
}

return summary, nil
}

// Failures returns a string containing the list of failed test cases and their
// output.
func (s *GoTestSummary) Failures() string {
b := new(strings.Builder)

if s.Fail > 0 {
fmt.Fprintln(b, "FAILURES:")
for _, pkg := range s.Report.Packages {
for _, t := range pkg.Tests {
if t.Result != parser.FAIL {
continue
}
fmt.Fprintln(b, "Package:", pkg.Name)
fmt.Fprintln(b, "Test: ", t.Name)
for _, line := range t.Output {
if strings.TrimSpace(line) != "" {
fmt.Fprintln(b, line)
}
}
fmt.Fprintln(b, "----")
}
}
}

return strings.TrimRight(b.String(), "\n")
}

// String returns a summary of the testing results (number of fail/pass/skip,
// test duration, number packages, output files).
func (s *GoTestSummary) String() string {
b := new(strings.Builder)

fmt.Fprintln(b, "SUMMARY:")
fmt.Fprintln(b, " Fail: ", s.Fail)
fmt.Fprintln(b, " Skip: ", s.Skip)
fmt.Fprintln(b, " Pass: ", s.Pass)
fmt.Fprintln(b, " Packages:", len(s.Report.Packages))
fmt.Fprintln(b, " Duration:", s.Duration)

// Sort the list of files and compute the column width.
var names []string
var nameWidth int
for name := range s.Files {
if len(name) > nameWidth {
nameWidth = len(name)
}
names = append(names, name)
}
sort.Strings(names)

for _, name := range names {
fmt.Fprintf(b, " %-*s %s\n", nameWidth, name, s.Files[name])
}

return strings.TrimRight(b.String(), "\n")
}

// BuildSystemTestBinary runs BuildSystemTestGoBinary with default values.
func BuildSystemTestBinary() error {
return BuildSystemTestGoBinary(DefaultTestBinaryArgs())
Expand Down
Loading

0 comments on commit b90b70e

Please sign in to comment.