From bbf9d6697f4a308b44484aca94359b347ebd1413 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Fri, 6 Mar 2020 20:41:18 +0800 Subject: [PATCH] mage: optionally mount module cache for crossbuild (#16837) * mage: optionally mount module cache for crossbuild Introduce another mage variable, CrossBuildMountModcache, which defaults to false. When set to true, the host's module cache ($GOPATH/pkg/mod) will be mounted into the crossbuild Docker containers, read-only. To ensure the cache is up-to-date, we run "go mod download" on the host before starting the Docker containers. Also, fix buildGoDaemon to stop assuming that tsg/go-daemon is vendored. Instead, use "go list -m github.com/tsg/go-daemon" to find the directory in either the vendor directory or the module cache. Rather than having arguments for ListModulePath, create separate functions for listing in the module cache and vendor directory. Create an unexported function in dev-tools/mage which calls the appropriate one depending on the value of UseVendor. We now use this in "findElasticBeatsDir". Inline defaultConfigFileParams, otherwise the OSSBeatDir and LibbeatDir calls end up calling ElasticBeatsDir at package init time, before apm-server uas an opportunity to set UseVendor=false. --- dev-tools/mage/common.go | 12 ++++++++++ dev-tools/mage/config.go | 30 ++++++++++++------------- dev-tools/mage/crossbuild.go | 13 +++++++++++ dev-tools/mage/godaemon.go | 7 +++++- dev-tools/mage/gomod.go | 2 +- dev-tools/mage/gotool/go.go | 27 +++++++++++++++++----- dev-tools/mage/gotool/modules.go | 21 ++++++++++------- dev-tools/mage/settings.go | 7 ++++-- generator/common/beatgen/beatgen.go | 6 +++++ generator/common/beatgen/setup/setup.go | 2 +- 10 files changed, 93 insertions(+), 34 deletions(-) diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index f17054bbd1f..e109d752506 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -49,6 +49,8 @@ import ( "github.com/magefile/mage/sh" "github.com/magefile/mage/target" "github.com/pkg/errors" + + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) // Expand expands the given Go text/template string. @@ -791,3 +793,13 @@ func ParseVersion(version string) (major, minor, patch int, err error) { patch, _ = strconv.Atoi(data["patch"]) return } + +// listModuleDir calls gotool.ListModuleVendorDir or +// gotool.ListModuleCacheDir, depending on the value of +// UseVendor. +func listModuleDir(modpath string) (string, error) { + if UseVendor { + return gotool.ListModuleVendorDir(modpath) + } + return gotool.ListModuleCacheDir(modpath) +} diff --git a/dev-tools/mage/config.go b/dev-tools/mage/config.go index 724970d87ae..668c71e1074 100644 --- a/dev-tools/mage/config.go +++ b/dev-tools/mage/config.go @@ -38,21 +38,6 @@ var ( shortTemplate = filepath.Join("build", BeatName+".yml.tmpl") referenceTemplate = filepath.Join("build", BeatName+".reference.yml.tmpl") dockerTemplate = filepath.Join("build", BeatName+".docker.yml.tmpl") - - defaultConfigFileParams = ConfigFileParams{ - ShortParts: []string{ - OSSBeatDir("_meta/beat.yml"), - LibbeatDir("_meta/config.yml.tmpl"), - }, - ReferenceParts: []string{ - OSSBeatDir("_meta/beat.reference.yml"), - LibbeatDir("_meta/config.reference.yml.tmpl"), - }, - DockerParts: []string{ - OSSBeatDir("_meta/beat.docker.yml"), - LibbeatDir("_meta/config.docker.yml"), - }, - } ) // ConfigFileType is a bitset that indicates what types of config files to @@ -97,7 +82,20 @@ func (c ConfigFileParams) Empty() bool { // host for the generated configs. Defaults to linux/amd64. func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error { if args.Empty() { - args = defaultConfigFileParams + args = ConfigFileParams{ + ShortParts: []string{ + OSSBeatDir("_meta/beat.yml"), + LibbeatDir("_meta/config.yml.tmpl"), + }, + ReferenceParts: []string{ + OSSBeatDir("_meta/beat.reference.yml"), + LibbeatDir("_meta/config.reference.yml.tmpl"), + }, + DockerParts: []string{ + OSSBeatDir("_meta/beat.docker.yml"), + LibbeatDir("_meta/config.docker.yml"), + }, + } } if err := makeConfigTemplates(types, args); err != nil { diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index b4ca802be1b..1d8fb72a337 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -19,6 +19,7 @@ package mage import ( "fmt" + "go/build" "log" "os" "path/filepath" @@ -31,6 +32,7 @@ import ( "github.com/magefile/mage/sh" "github.com/pkg/errors" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" "github.com/elastic/beats/v7/libbeat/common/file" ) @@ -127,6 +129,12 @@ func CrossBuild(options ...CrossBuildOption) error { return nil } + if CrossBuildMountModcache { + // Make sure the module dependencies are downloaded on the host, + // as they will be mounted into the container read-only. + mg.Deps(func() error { return gotool.Mod.Download() }) + } + // Build the magefile for Linux so we can run it inside the container. mg.Deps(buildMage) @@ -250,6 +258,11 @@ func (b GolangCrossBuilder) Build() error { if UseVendor { args = append(args, "--env", "GOFLAGS=-mod=vendor") } + if CrossBuildMountModcache { + // Mount $GOPATH/pkg/mod into the container, read-only. + hostDir := filepath.Join(build.Default.GOPATH, "pkg", "mod") + args = append(args, "-v", hostDir+":/go/pkg/mod:ro") + } args = append(args, "--rm", diff --git a/dev-tools/mage/godaemon.go b/dev-tools/mage/godaemon.go index 7511cae8044..7397dac74d9 100644 --- a/dev-tools/mage/godaemon.go +++ b/dev-tools/mage/godaemon.go @@ -21,6 +21,7 @@ import ( "errors" "log" "os" + "path/filepath" ) var ( @@ -42,8 +43,12 @@ func BuildGoDaemon() error { } // Test if binaries are up-to-date. + godaemonDir, err := listModuleDir("github.com/tsg/go-daemon") + if err != nil { + return err + } + input := filepath.Join(godaemonDir, "src", "god.c") output := MustExpand("build/golang-crossbuild/god-{{.Platform.GOOS}}-{{.Platform.Arch}}") - input := MustExpand("{{ elastic_beats_dir }}/vendor/github.com/tsg/go-daemon/src/god.c") if IsUpToDate(output, input) { log.Println(">>> buildGoDaemon is up-to-date for", Platform.Name) return nil diff --git a/dev-tools/mage/gomod.go b/dev-tools/mage/gomod.go index 7cc6b7d8c68..9d785b47971 100644 --- a/dev-tools/mage/gomod.go +++ b/dev-tools/mage/gomod.go @@ -75,7 +75,7 @@ func Vendor() error { // copy packages which require the whole tree for _, p := range copyAll { - path, err := gotool.ListModulePath(p.name) + path, err := gotool.ListModuleVendorDir(p.name) if err != nil { return err } diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go index 9e6cc08a471..9022e0c0137 100644 --- a/dev-tools/mage/gotool/go.go +++ b/dev-tools/mage/gotool/go.go @@ -93,14 +93,31 @@ func ListTestFiles(pkg string) ([]string, error) { return getLines(callGo(nil, "list", "-f", tmpl, pkg)) } -// ListModulePath returns the path to the module in the cache. -func ListModulePath(pkg string) (string, error) { - const tmpl = `{{.Dir}}` +// ListModuleCacheDir returns the module cache directory containing +// the specified module. If the module does not exist in the cache, +// an error will be returned. +func ListModuleCacheDir(pkg string) (string, error) { + return listModuleDir(pkg, false) +} + +// ListModuleVendorDir returns the vendor directory containing the +// specified module. If the module has not been vendored, an error +// will be returned. +func ListModuleVendorDir(pkg string) (string, error) { + return listModuleDir(pkg, true) +} + +func listModuleDir(pkg string, vendor bool) (string, error) { env := map[string]string{ - // make sure to look in the module cache + // Make sure GOFLAGS does not influence behaviour. "GOFLAGS": "", } - lines, err := getLines(callGo(env, "list", "-m", "-f", tmpl, pkg)) + args := []string{"-m", "-f", "{{.Dir}}"} + if vendor { + args = append(args, "-mod=vendor") + } + args = append(args, pkg) + lines, err := getLines(callGo(env, "list", args...)) if err != nil { return "", err } diff --git a/dev-tools/mage/gotool/modules.go b/dev-tools/mage/gotool/modules.go index ebaf2869ec5..d7a880756b7 100644 --- a/dev-tools/mage/gotool/modules.go +++ b/dev-tools/mage/gotool/modules.go @@ -19,10 +19,11 @@ package gotool // Mod is the command go mod. var Mod = goMod{ - Init: modCommand{"init"}.run, - Tidy: modCommand{"tidy"}.run, - Verify: modCommand{"verify"}.run, - Vendor: modCommand{"vendor"}.run, + Download: modCommand{"download"}.run, + Init: modCommand{"init"}.run, + Tidy: modCommand{"tidy"}.run, + Verify: modCommand{"verify"}.run, + Vendor: modCommand{"vendor"}.run, } type modCommand struct { @@ -40,12 +41,16 @@ func (cmd modCommand) run(opts ...ArgOpt) error { } type goMod struct { - Init modInit - Tidy modTidy - Verify modVerify - Vendor modVendor + Download modDownload + Init modInit + Tidy modTidy + Verify modVerify + Vendor modVendor } +// modDownload cleans the go.mod file +type modDownload func(opts ...ArgOpt) error + // modInit initializes a new go module in folder. type modInit func(opts ...ArgOpt) error diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 5b96e58e9f0..66b867057ca 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -60,6 +60,10 @@ var ( TestCoverage = false UseVendor = true + // CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into + // the crossbuild images at /go/pkg/mod, read-only. + CrossBuildMountModcache = false + BeatName = EnvOr("BEAT_NAME", filepath.Base(CWD())) BeatServiceName = EnvOr("BEAT_SERVICE_NAME", BeatName) BeatIndexPrefix = EnvOr("BEAT_INDEX_PREFIX", BeatName) @@ -280,8 +284,7 @@ func findElasticBeatsDir() (string, error) { if repo.IsElasticBeats() { return repo.RootDir, nil } - - return gotool.ListModulePath("github.com/elastic/beats/v7") + return listModuleDir("github.com/elastic/beats/v7") } var ( diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go index a4a1f436a13..e92bbc51276 100644 --- a/generator/common/beatgen/beatgen.go +++ b/generator/common/beatgen/beatgen.go @@ -101,6 +101,12 @@ func Generate() error { return err } + // Make sure the ElasticBeatsDir value is cached + // before changing directory below. + if _, err := devtools.ElasticBeatsDir(); err != nil { + return err + } + err = setup.GenNewBeat(cfg) if err != nil { return errors.Wrap(err, "error generating new beat") diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go index c4ef95c7654..b372660971d 100644 --- a/generator/common/beatgen/setup/setup.go +++ b/generator/common/beatgen/setup/setup.go @@ -112,7 +112,7 @@ func CopyVendor() error { return err } - path, err := gotool.ListModulePath("github.com/elastic/beats/v7") + path, err := gotool.ListModuleCacheDir("github.com/elastic/beats/v7") if err != nil { return err }