Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry-pick #22541 to 7.x: Run unit and integration tests with gotestsum #23512

Merged
merged 1 commit into from
Jan 14, 2021
Merged

Cherry-pick #22541 to 7.x: Run unit and integration tests with gotestsum #23512

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci/scripts/install-tools.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,4 @@ gcc --version
where gcc

REM Reset the USERPROFILE
SET USERPROFILE=%PREVIOUS_USERPROFILE%
SET USERPROFILE=%PREVIOUS_USERPROFILE%
523 changes: 485 additions & 38 deletions NOTICE.txt
View file
Open in desktop

Large diffs are not rendered by default.

232 changes: 62 additions & 170 deletions dev-tools/mage/gotest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package mage

import (
"bytes"
"context"
"fmt"
"io"
Expand All @@ -28,16 +27,14 @@ import (
"os/exec"
"path"
"path/filepath"
"runtime"
"sort"
"strings"
"time"

"github.com/jstemmer/go-junit-report/formatter"
"github.com/jstemmer/go-junit-report/parser"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/pkg/errors"

"github.com/elastic/beats/v7/dev-tools/mage/gotool"
)

// GoTestArgs are the arguments used for the "go*Test" targets and they define
Expand All @@ -52,6 +49,7 @@ type GoTestArgs struct {
OutputFile string // File to write verbose test output to.
JUnitReportFile string // File to write a JUnit XML test report to.
CoverageProfileFile string // Test coverage profile file (enables -cover).
Output io.Writer // Write stderr and stdout to Output if set
}

// TestBinaryArgs are the arguments used when building binary for testing.
Expand Down Expand Up @@ -183,40 +181,80 @@ func GoTestIntegrationForModule(ctx context.Context) error {
return nil
}

// InstallGoTestTools installs additional tools that are required to run unit and integration tests.
func InstallGoTestTools() {
gotool.Install(
gotool.Install.Package("gotest.tools/gotestsum"),
)
}

// GoTest invokes "go test" and reports the results to stdout. It returns an
// error if there was any failure executing the tests or if there were any
// test failures.
func GoTest(ctx context.Context, params GoTestArgs) error {
mg.Deps(InstallGoTestTools)

fmt.Println(">> go test:", params.TestName, "Testing")

// Build args list to Go.
args := []string{"test"}
args = append(args, "-v")
// We use gotestsum to drive the tests and produce a junit report.
// The tool runs `go test -json` in order to produce a structured log which makes it easier
// to parse the actual test output.
// Of OutputFile is given the original JSON file will be written as well.
//
// The runner needs to set CLI flags for gotestsum and for "go test". We track the different
// CLI flags in the gotestsumArgs and testArgs variables, such that we can finally produce command like:
// $ gotestsum <gotestsum args> -- <go test args>
//
// The additional arguments given via GoTestArgs are applied to `go test` only. Callers can not
// modify any of the gotestsum arguments.

gotestsumArgs := []string{"--no-color"}
if mg.Verbose() {
gotestsumArgs = append(gotestsumArgs, "-f", "standard-verbose")
} else {
gotestsumArgs = append(gotestsumArgs, "-f", "standard-quiet")
}
if params.JUnitReportFile != "" {
CreateDir(params.JUnitReportFile)
gotestsumArgs = append(gotestsumArgs, "--junitfile", params.JUnitReportFile)
}
if params.OutputFile != "" {
CreateDir(params.OutputFile)
gotestsumArgs = append(gotestsumArgs, "--jsonfile", params.OutputFile+".json")
}

var testArgs []string

// -race is only supported on */amd64
if os.Getenv("DEV_ARCH") == "amd64" {
if params.Race {
args = append(args, "-race")
testArgs = append(testArgs, "-race")
}
}
if len(params.Tags) > 0 {
args = append(args, "-tags", strings.Join(params.Tags, " "))
params := strings.Join(params.Tags, " ")
if params != "" {
testArgs = append(testArgs, "-tags", params)
}
}
if params.CoverageProfileFile != "" {
params.CoverageProfileFile = createDir(filepath.Clean(params.CoverageProfileFile))
args = append(args,
testArgs = append(testArgs,
"-covermode=atomic",
"-coverprofile="+params.CoverageProfileFile,
)
}
args = append(args, params.ExtraFlags...)
args = append(args, params.Packages...)
testArgs = append(testArgs, params.ExtraFlags...)
testArgs = append(testArgs, params.Packages...)

goTest := makeCommand(ctx, params.Env, "go", args...)
args := append(gotestsumArgs, append([]string{"--"}, testArgs...)...)

goTest := makeCommand(ctx, params.Env, "gotestsum", args...)
// Wire up the outputs.
bufferOutput := new(bytes.Buffer)
outputs := []io.Writer{bufferOutput}
var outputs []io.Writer
if params.Output != nil {
outputs = append(outputs, params.Output)
}

if params.OutputFile != "" {
fileOutput, err := os.Create(createDir(params.OutputFile))
Expand All @@ -227,18 +265,16 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
outputs = append(outputs, fileOutput)
}
output := io.MultiWriter(outputs...)
goTest.Stdout = output
goTest.Stderr = output

if mg.Verbose() {
if params.Output == nil {
goTest.Stdout = io.MultiWriter(output, os.Stdout)
goTest.Stderr = io.MultiWriter(output, os.Stderr)
} else {
goTest.Stdout = output
goTest.Stderr = output
}

// Execute 'go test' and measure duration.
start := time.Now()
err := goTest.Run()
duration := time.Since(start)

var goTestErr *exec.ExitError
if err != nil {
// Command ran.
Expand All @@ -251,30 +287,11 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
goTestErr = exitErr
}

// Parse the verbose test output.
report, err := parser.Parse(bytes.NewBuffer(bufferOutput.Bytes()), BeatName)
if err != nil {
return errors.Wrap(err, "failed to parse go test output")
}
if goTestErr != nil && len(report.Packages) == 0 {
if goTestErr != nil {
// No packages were tested. Probably the code didn't compile.
fmt.Println(bytes.NewBuffer(bufferOutput.Bytes()).String())
return errors.Wrap(goTestErr, "go test returned a non-zero value")
}

// Generate a JUnit XML report.
if params.JUnitReportFile != "" {
junitReport, err := os.Create(createDir(params.JUnitReportFile))
if err != nil {
return errors.Wrap(err, "failed to create junit report")
}
defer junitReport.Close()

if err = formatter.JUnitReportXML(report, false, runtime.Version(), junitReport); err != nil {
return errors.Wrap(err, "failed to write junit report")
}
}

// Generate a HTML code coverage report.
var htmlCoverReport string
if params.CoverageProfileFile != "" {
Expand All @@ -288,27 +305,9 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
}
}

// Summarize the results and log to stdout.
summary, err := NewGoTestSummary(duration, report, map[string]string{
"Output File": params.OutputFile,
"JUnit Report": params.JUnitReportFile,
"Coverage Report": htmlCoverReport,
})
if err != nil {
return err
}
if !mg.Verbose() && summary.Fail > 0 {
fmt.Println(summary.Failures())
}
fmt.Println(summary.String())

// Return an error indicating that testing failed.
if summary.Fail > 0 || goTestErr != nil {
if goTestErr != nil {
fmt.Println(">> go test:", params.TestName, "Test Failed")
if summary.Fail > 0 {
return errors.Errorf("go test failed: %d test failures", summary.Fail)
}

return errors.Wrap(goTestErr, "go test returned a non-zero value")
}

Expand All @@ -329,117 +328,10 @@ func makeCommand(ctx context.Context, env map[string]string, cmd string, args ..
c.Stderr = os.Stderr
c.Stdin = os.Stdin
log.Println("exec:", cmd, strings.Join(args, " "))
fmt.Println("exec:", cmd, strings.Join(args, " "))
return c
}

// GoTestSummary is a summary of test results.
type GoTestSummary struct {
*parser.Report // Report generated by parsing test output.
Pass int // Number of passing tests.
Fail int // Number of failed tests.
Skip int // Number of skipped tests.
Packages int // Number of packages tested.
Duration time.Duration // Total go test running duration.
Files map[string]string
}

// NewGoTestSummary builds a new GoTestSummary. It returns an error if it cannot
// resolve the absolute paths to the given files.
func NewGoTestSummary(d time.Duration, r *parser.Report, outputFiles map[string]string) (*GoTestSummary, error) {
files := map[string]string{}
for name, file := range outputFiles {
if file == "" {
continue
}
absFile, err := filepath.Abs(file)
if err != nil {
return nil, errors.Wrapf(err, "failed resolving absolute path for %v", file)
}
files[name+":"] = absFile
}

summary := &GoTestSummary{
Report: r,
Duration: d,
Packages: len(r.Packages),
Files: files,
}

for _, pkg := range r.Packages {
for _, t := range pkg.Tests {
switch t.Result {
case parser.PASS:
summary.Pass++
case parser.FAIL:
summary.Fail++
case parser.SKIP:
summary.Skip++
default:
return nil, errors.Errorf("Unknown test result value: %v", t.Result)
}
}
}

return summary, nil
}

// Failures returns a string containing the list of failed test cases and their
// output.
func (s *GoTestSummary) Failures() string {
b := new(strings.Builder)

if s.Fail > 0 {
fmt.Fprintln(b, "FAILURES:")
for _, pkg := range s.Report.Packages {
for _, t := range pkg.Tests {
if t.Result != parser.FAIL {
continue
}
fmt.Fprintln(b, "Package:", pkg.Name)
fmt.Fprintln(b, "Test: ", t.Name)
for _, line := range t.Output {
if strings.TrimSpace(line) != "" {
fmt.Fprintln(b, line)
}
}
fmt.Fprintln(b, "----")
}
}
}

return strings.TrimRight(b.String(), "\n")
}

// String returns a summary of the testing results (number of fail/pass/skip,
// test duration, number packages, output files).
func (s *GoTestSummary) String() string {
b := new(strings.Builder)

fmt.Fprintln(b, "SUMMARY:")
fmt.Fprintln(b, " Fail: ", s.Fail)
fmt.Fprintln(b, " Skip: ", s.Skip)
fmt.Fprintln(b, " Pass: ", s.Pass)
fmt.Fprintln(b, " Packages:", len(s.Report.Packages))
fmt.Fprintln(b, " Duration:", s.Duration)

// Sort the list of files and compute the column width.
var names []string
var nameWidth int
for name := range s.Files {
if len(name) > nameWidth {
nameWidth = len(name)
}
names = append(names, name)
}
sort.Strings(names)

for _, name := range names {
fmt.Fprintf(b, " %-*s %s\n", nameWidth, name, s.Files[name])
}

return strings.TrimRight(b.String(), "\n")
}

// BuildSystemTestBinary runs BuildSystemTestGoBinary with default values.
func BuildSystemTestBinary() error {
return BuildSystemTestGoBinary(DefaultTestBinaryArgs())
Expand Down
Loading