mage: optionally mount module cache for crossbuild

Introduce another mage variable, CrossBuildMountModcache,
which defaults to false. When set to true, the host's
module cache ($GOPATH/pkg/mod) will be mounted into the
crossbuild Docker containers, read-only. To ensure the
cache is up-to-date, we run "go mod download" on the host
before starting the Docker containers.

Also, fix buildGoDaemon to stop assuming that tsg/go-daemon
is vendored. Instead, use "go list -m github.com/tsg/go-daemon"
to find the directory in either the vendor directory or
the module cache.

dev-tools/mage/crossbuild.go

@@ -19,6 +19,7 @@ package mage
 
 import (
 	"fmt"
+	"go/build"
 	"log"
 	"os"
 	"path/filepath"
@@ -31,6 +32,7 @@ import (
 	"github.com/magefile/mage/sh"
 	"github.com/pkg/errors"
 
+	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 	"github.com/elastic/beats/v7/libbeat/common/file"
 )
 
@@ -127,6 +129,12 @@ func CrossBuild(options ...CrossBuildOption) error {
 		return nil
 	}
 
+	if CrossBuildMountModcache {
+		// Make sure the module dependencies are downloaded on the host,
+		// as they will be mounted into the container read-only.
+		mg.Deps(func() error { return gotool.Mod.Download() })
+	}
+
 	// Build the magefile for Linux so we can run it inside the container.
 	mg.Deps(buildMage)
 
@@ -250,6 +258,11 @@ func (b GolangCrossBuilder) Build() error {
 	if UseVendor {
 		args = append(args, "--env", "GOFLAGS=-mod=vendor")
 	}
+	if CrossBuildMountModcache {
+		// Mount $GOPATH/pkg/mod into the container, read-only.
+		hostDir := filepath.Join(build.Default.GOPATH, "pkg", "mod")
+		args = append(args, "-v", hostDir+":/go/pkg/mod:ro")
+	}
 
 	args = append(args,
 		"--rm",

dev-tools/mage/godaemon.go

@@ -21,6 +21,9 @@ import (
 	"errors"
 	"log"
 	"os"
+	"path/filepath"
+
+	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
 
 var (
@@ -41,9 +44,19 @@ func BuildGoDaemon() error {
 			"only be executed within the golang-crossbuild docker environment.")
 	}
 
+	// Locate tsg/go-daemon.
+	var args []string
+	if UseVendor {
+		args = append(args, "-mod=vendor")
+	}
+	godaemonDir, err := gotool.ListModulePath("github.com/tsg/go-daemon", args...)
+	if err != nil {
+		return err
+	}
+
 	// Test if binaries are up-to-date.
 	output := MustExpand("build/golang-crossbuild/god-{{.Platform.GOOS}}-{{.Platform.Arch}}")
-	input := MustExpand("{{ elastic_beats_dir }}/vendor/github.com/tsg/go-daemon/src/god.c")
+	input := filepath.Join(godaemonDir, "src", "god.c")
 	if IsUpToDate(output, input) {
 		log.Println(">>> buildGoDaemon is up-to-date for", Platform.Name)
 		return nil

dev-tools/mage/gotool/go.go

@@ -93,14 +93,19 @@ func ListTestFiles(pkg string) ([]string, error) {
 	return getLines(callGo(nil, "list", "-f", tmpl, pkg))
 }
 
-// ListModulePath returns the path to the module in the cache.
-func ListModulePath(pkg string) (string, error) {
-	const tmpl = `{{.Dir}}`
+// ListModulePath returns the path to the specified module.
+//
+// Additional parameters like "-mod=vendor" may be passed
+// to control behaviour.
+func ListModulePath(pkg string, args ...string) (string, error) {
 	env := map[string]string{
-		// make sure to look in the module cache
+		// make sure to look in the module cache,
+		// unless otherwise configured in args.
 		"GOFLAGS": "",
 	}
-	lines, err := getLines(callGo(env, "list", "-m", "-f", tmpl, pkg))
+	args = append([]string{"-m", "-f", "{{.Dir}}"}, args...)
+	args = append(args, pkg)
+	lines, err := getLines(callGo(env, "list", args...))
 	if err != nil {
 		return "", err
 	}

dev-tools/mage/gotool/modules.go

@@ -19,10 +19,11 @@ package gotool
 
 // Mod is the command go mod.
 var Mod = goMod{
-	Init:   modCommand{"init"}.run,
-	Tidy:   modCommand{"tidy"}.run,
-	Verify: modCommand{"verify"}.run,
-	Vendor: modCommand{"vendor"}.run,
+	Download: modCommand{"download"}.run,
+	Init:     modCommand{"init"}.run,
+	Tidy:     modCommand{"tidy"}.run,
+	Verify:   modCommand{"verify"}.run,
+	Vendor:   modCommand{"vendor"}.run,
 }
 
 type modCommand struct {
@@ -40,12 +41,16 @@ func (cmd modCommand) run(opts ...ArgOpt) error {
 }
 
 type goMod struct {
-	Init   modInit
-	Tidy   modTidy
-	Verify modVerify
-	Vendor modVendor
+	Download modDownload
+	Init     modInit
+	Tidy     modTidy
+	Verify   modVerify
+	Vendor   modVendor
 }
 
+// modDownload cleans the go.mod file
+type modDownload func(opts ...ArgOpt) error
+
 // modInit initializes a new go module in folder.
 type modInit func(opts ...ArgOpt) error
 

dev-tools/mage/settings.go

@@ -60,6 +60,10 @@ var (
 	TestCoverage = false
 	UseVendor    = true
 
+	// CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into
+	// the crossbuild images at /go/pkg/mod, read-only.
+	CrossBuildMountModcache = false
+
 	BeatName        = EnvOr("BEAT_NAME", filepath.Base(CWD()))
 	BeatServiceName = EnvOr("BEAT_SERVICE_NAME", BeatName)
 	BeatIndexPrefix = EnvOr("BEAT_INDEX_PREFIX", BeatName)