Run unit and integration tests with gotestsum

.ci/scripts/install-tools.bat

@@ -79,4 +79,4 @@ gcc --version
 where gcc
 
 REM Reset the USERPROFILE
-SET USERPROFILE=%PREVIOUS_USERPROFILE%
+SET USERPROFILE=%PREVIOUS_USERPROFILE%

dev-tools/mage/gotest.go

@@ -18,7 +18,6 @@
 package mage
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -28,16 +27,14 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
-	"runtime"
-	"sort"
 	"strings"
 	"time"
 
-	"github.com/jstemmer/go-junit-report/formatter"
-	"github.com/jstemmer/go-junit-report/parser"
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
 	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
 
 // GoTestArgs are the arguments used for the "go*Test" targets and they define
@@ -52,6 +49,7 @@ type GoTestArgs struct {
 	OutputFile          string            // File to write verbose test output to.
 	JUnitReportFile     string            // File to write a JUnit XML test report to.
 	CoverageProfileFile string            // Test coverage profile file (enables -cover).
+	Output              io.Writer         // Write stderr and stdout to Output if set
 }
 
 // TestBinaryArgs are the arguments used when building binary for testing.
@@ -183,40 +181,80 @@ func GoTestIntegrationForModule(ctx context.Context) error {
 	return nil
 }
 
+// InstallGoTestTools installs additional tools that are required to run unit and integration tests.
+func InstallGoTestTools() {
+	gotool.Install(
+		gotool.Install.Package("gotest.tools/gotestsum"),
+	)
+}
+
 // GoTest invokes "go test" and reports the results to stdout. It returns an
 // error if there was any failure executing the tests or if there were any
 // test failures.
 func GoTest(ctx context.Context, params GoTestArgs) error {
+	mg.Deps(InstallGoTestTools)
+
 	fmt.Println(">> go test:", params.TestName, "Testing")
 
-	// Build args list to Go.
-	args := []string{"test"}
-	args = append(args, "-v")
+	// We use gotestsum to drive the tests and produce a junit report.
+	// The tool runs `go test -json` in order to produce a structured log which makes it easier
+	// to parse the actual test output.
+	// Of OutputFile is given the original JSON file will be written as well.
+	//
+	// The runner needs to set CLI flags for gotestsum and for "go test". We track the different
+	// CLI flags in the gotestsumArgs and testArgs variables, such that we can finally produce command like:
+	//   $ gotestsum <gotestsum args> -- <go test args>
+	//
+	// The additional arguments given via GoTestArgs are applied to `go test` only. Callers can not
+	// modify any of the gotestsum arguments.
+
+	gotestsumArgs := []string{"--no-color"}
+	if mg.Verbose() {
+		gotestsumArgs = append(gotestsumArgs, "-f", "standard-verbose")
+	} else {
+		gotestsumArgs = append(gotestsumArgs, "-f", "standard-quiet")
+	}
+	if params.JUnitReportFile != "" {
+		CreateDir(params.JUnitReportFile)
+		gotestsumArgs = append(gotestsumArgs, "--junitfile", params.JUnitReportFile)
+	}
+	if params.OutputFile != "" {
+		CreateDir(params.OutputFile)
+		gotestsumArgs = append(gotestsumArgs, "--jsonfile", params.OutputFile+".json")
+	}
+
+	var testArgs []string
 
 	// -race is only supported on */amd64
 	if os.Getenv("DEV_ARCH") == "amd64" {
 		if params.Race {
-			args = append(args, "-race")
+			testArgs = append(testArgs, "-race")
 		}
 	}
 	if len(params.Tags) > 0 {
-		args = append(args, "-tags", strings.Join(params.Tags, " "))
+		params := strings.Join(params.Tags, " ")
+		if params != "" {
+			testArgs = append(testArgs, "-tags", params)
+		}
 	}
 	if params.CoverageProfileFile != "" {
 		params.CoverageProfileFile = createDir(filepath.Clean(params.CoverageProfileFile))
-		args = append(args,
+		testArgs = append(testArgs,
 			"-covermode=atomic",
 			"-coverprofile="+params.CoverageProfileFile,
 		)
 	}
-	args = append(args, params.ExtraFlags...)
-	args = append(args, params.Packages...)
+	testArgs = append(testArgs, params.ExtraFlags...)
+	testArgs = append(testArgs, params.Packages...)
 
-	goTest := makeCommand(ctx, params.Env, "go", args...)
+	args := append(gotestsumArgs, append([]string{"--"}, testArgs...)...)
 
+	goTest := makeCommand(ctx, params.Env, "gotestsum", args...)
 	// Wire up the outputs.
-	bufferOutput := new(bytes.Buffer)
-	outputs := []io.Writer{bufferOutput}
+	var outputs []io.Writer
+	if params.Output != nil {
+		outputs = append(outputs, params.Output)
+	}
 
 	if params.OutputFile != "" {
 		fileOutput, err := os.Create(createDir(params.OutputFile))
@@ -227,18 +265,16 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 		outputs = append(outputs, fileOutput)
 	}
 	output := io.MultiWriter(outputs...)
-	goTest.Stdout = output
-	goTest.Stderr = output
-
-	if mg.Verbose() {
+	if params.Output == nil {
 		goTest.Stdout = io.MultiWriter(output, os.Stdout)
 		goTest.Stderr = io.MultiWriter(output, os.Stderr)
+	} else {
+		goTest.Stdout = output
+		goTest.Stderr = output
 	}
 
-	// Execute 'go test' and measure duration.
-	start := time.Now()
 	err := goTest.Run()
-	duration := time.Since(start)
+
 	var goTestErr *exec.ExitError
 	if err != nil {
 		// Command ran.
@@ -251,30 +287,11 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 		goTestErr = exitErr
 	}
 
-	// Parse the verbose test output.
-	report, err := parser.Parse(bytes.NewBuffer(bufferOutput.Bytes()), BeatName)
-	if err != nil {
-		return errors.Wrap(err, "failed to parse go test output")
-	}
-	if goTestErr != nil && len(report.Packages) == 0 {
+	if goTestErr != nil {
 		// No packages were tested. Probably the code didn't compile.
-		fmt.Println(bytes.NewBuffer(bufferOutput.Bytes()).String())
 		return errors.Wrap(goTestErr, "go test returned a non-zero value")
 	}
 
-	// Generate a JUnit XML report.
-	if params.JUnitReportFile != "" {
-		junitReport, err := os.Create(createDir(params.JUnitReportFile))
-		if err != nil {
-			return errors.Wrap(err, "failed to create junit report")
-		}
-		defer junitReport.Close()
-
-		if err = formatter.JUnitReportXML(report, false, runtime.Version(), junitReport); err != nil {
-			return errors.Wrap(err, "failed to write junit report")
-		}
-	}
-
 	// Generate a HTML code coverage report.
 	var htmlCoverReport string
 	if params.CoverageProfileFile != "" {
@@ -288,27 +305,9 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 		}
 	}
 
-	// Summarize the results and log to stdout.
-	summary, err := NewGoTestSummary(duration, report, map[string]string{
-		"Output File":     params.OutputFile,
-		"JUnit Report":    params.JUnitReportFile,
-		"Coverage Report": htmlCoverReport,
-	})
-	if err != nil {
-		return err
-	}
-	if !mg.Verbose() && summary.Fail > 0 {
-		fmt.Println(summary.Failures())
-	}
-	fmt.Println(summary.String())
-
 	// Return an error indicating that testing failed.
-	if summary.Fail > 0 || goTestErr != nil {
+	if goTestErr != nil {
 		fmt.Println(">> go test:", params.TestName, "Test Failed")
-		if summary.Fail > 0 {
-			return errors.Errorf("go test failed: %d test failures", summary.Fail)
-		}
-
 		return errors.Wrap(goTestErr, "go test returned a non-zero value")
 	}
 
@@ -329,117 +328,10 @@ func makeCommand(ctx context.Context, env map[string]string, cmd string, args ..
 	c.Stderr = os.Stderr
 	c.Stdin = os.Stdin
 	log.Println("exec:", cmd, strings.Join(args, " "))
+	fmt.Println("exec:", cmd, strings.Join(args, " "))
 	return c
 }
 
-// GoTestSummary is a summary of test results.
-type GoTestSummary struct {
-	*parser.Report               // Report generated by parsing test output.
-	Pass           int           // Number of passing tests.
-	Fail           int           // Number of failed tests.
-	Skip           int           // Number of skipped tests.
-	Packages       int           // Number of packages tested.
-	Duration       time.Duration // Total go test running duration.
-	Files          map[string]string
-}
-
-// NewGoTestSummary builds a new GoTestSummary. It returns an error if it cannot
-// resolve the absolute paths to the given files.
-func NewGoTestSummary(d time.Duration, r *parser.Report, outputFiles map[string]string) (*GoTestSummary, error) {
-	files := map[string]string{}
-	for name, file := range outputFiles {
-		if file == "" {
-			continue
-		}
-		absFile, err := filepath.Abs(file)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed resolving absolute path for %v", file)
-		}
-		files[name+":"] = absFile
-	}
-
-	summary := &GoTestSummary{
-		Report:   r,
-		Duration: d,
-		Packages: len(r.Packages),
-		Files:    files,
-	}
-
-	for _, pkg := range r.Packages {
-		for _, t := range pkg.Tests {
-			switch t.Result {
-			case parser.PASS:
-				summary.Pass++
-			case parser.FAIL:
-				summary.Fail++
-			case parser.SKIP:
-				summary.Skip++
-			default:
-				return nil, errors.Errorf("Unknown test result value: %v", t.Result)
-			}
-		}
-	}
-
-	return summary, nil
-}
-
-// Failures returns a string containing the list of failed test cases and their
-// output.
-func (s *GoTestSummary) Failures() string {
-	b := new(strings.Builder)
-
-	if s.Fail > 0 {
-		fmt.Fprintln(b, "FAILURES:")
-		for _, pkg := range s.Report.Packages {
-			for _, t := range pkg.Tests {
-				if t.Result != parser.FAIL {
-					continue
-				}
-				fmt.Fprintln(b, "Package:", pkg.Name)
-				fmt.Fprintln(b, "Test:   ", t.Name)
-				for _, line := range t.Output {
-					if strings.TrimSpace(line) != "" {
-						fmt.Fprintln(b, line)
-					}
-				}
-				fmt.Fprintln(b, "----")
-			}
-		}
-	}
-
-	return strings.TrimRight(b.String(), "\n")
-}
-
-// String returns a summary of the testing results (number of fail/pass/skip,
-// test duration, number packages, output files).
-func (s *GoTestSummary) String() string {
-	b := new(strings.Builder)
-
-	fmt.Fprintln(b, "SUMMARY:")
-	fmt.Fprintln(b, "  Fail:    ", s.Fail)
-	fmt.Fprintln(b, "  Skip:    ", s.Skip)
-	fmt.Fprintln(b, "  Pass:    ", s.Pass)
-	fmt.Fprintln(b, "  Packages:", len(s.Report.Packages))
-	fmt.Fprintln(b, "  Duration:", s.Duration)
-
-	// Sort the list of files and compute the column width.
-	var names []string
-	var nameWidth int
-	for name := range s.Files {
-		if len(name) > nameWidth {
-			nameWidth = len(name)
-		}
-		names = append(names, name)
-	}
-	sort.Strings(names)
-
-	for _, name := range names {
-		fmt.Fprintf(b, "  %-*s %s\n", nameWidth, name, s.Files[name])
-	}
-
-	return strings.TrimRight(b.String(), "\n")
-}
-
 // BuildSystemTestBinary runs BuildSystemTestGoBinary with default values.
 func BuildSystemTestBinary() error {
 	return BuildSystemTestGoBinary(DefaultTestBinaryArgs())