diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index f17054bbd1f..e109d752506 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -49,6 +49,8 @@ import ( "github.com/magefile/mage/sh" "github.com/magefile/mage/target" "github.com/pkg/errors" + + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) // Expand expands the given Go text/template string. @@ -791,3 +793,13 @@ func ParseVersion(version string) (major, minor, patch int, err error) { patch, _ = strconv.Atoi(data["patch"]) return } + +// listModuleDir calls gotool.ListModuleVendorDir or +// gotool.ListModuleCacheDir, depending on the value of +// UseVendor. +func listModuleDir(modpath string) (string, error) { + if UseVendor { + return gotool.ListModuleVendorDir(modpath) + } + return gotool.ListModuleCacheDir(modpath) +} diff --git a/dev-tools/mage/config.go b/dev-tools/mage/config.go index 724970d87ae..668c71e1074 100644 --- a/dev-tools/mage/config.go +++ b/dev-tools/mage/config.go @@ -38,21 +38,6 @@ var ( shortTemplate = filepath.Join("build", BeatName+".yml.tmpl") referenceTemplate = filepath.Join("build", BeatName+".reference.yml.tmpl") dockerTemplate = filepath.Join("build", BeatName+".docker.yml.tmpl") - - defaultConfigFileParams = ConfigFileParams{ - ShortParts: []string{ - OSSBeatDir("_meta/beat.yml"), - LibbeatDir("_meta/config.yml.tmpl"), - }, - ReferenceParts: []string{ - OSSBeatDir("_meta/beat.reference.yml"), - LibbeatDir("_meta/config.reference.yml.tmpl"), - }, - DockerParts: []string{ - OSSBeatDir("_meta/beat.docker.yml"), - LibbeatDir("_meta/config.docker.yml"), - }, - } ) // ConfigFileType is a bitset that indicates what types of config files to @@ -97,7 +82,20 @@ func (c ConfigFileParams) Empty() bool { // host for the generated configs. Defaults to linux/amd64. func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error { if args.Empty() { - args = defaultConfigFileParams + args = ConfigFileParams{ + ShortParts: []string{ + OSSBeatDir("_meta/beat.yml"), + LibbeatDir("_meta/config.yml.tmpl"), + }, + ReferenceParts: []string{ + OSSBeatDir("_meta/beat.reference.yml"), + LibbeatDir("_meta/config.reference.yml.tmpl"), + }, + DockerParts: []string{ + OSSBeatDir("_meta/beat.docker.yml"), + LibbeatDir("_meta/config.docker.yml"), + }, + } } if err := makeConfigTemplates(types, args); err != nil { diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index b4ca802be1b..1d8fb72a337 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -19,6 +19,7 @@ package mage import ( "fmt" + "go/build" "log" "os" "path/filepath" @@ -31,6 +32,7 @@ import ( "github.com/magefile/mage/sh" "github.com/pkg/errors" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" "github.com/elastic/beats/v7/libbeat/common/file" ) @@ -127,6 +129,12 @@ func CrossBuild(options ...CrossBuildOption) error { return nil } + if CrossBuildMountModcache { + // Make sure the module dependencies are downloaded on the host, + // as they will be mounted into the container read-only. + mg.Deps(func() error { return gotool.Mod.Download() }) + } + // Build the magefile for Linux so we can run it inside the container. mg.Deps(buildMage) @@ -250,6 +258,11 @@ func (b GolangCrossBuilder) Build() error { if UseVendor { args = append(args, "--env", "GOFLAGS=-mod=vendor") } + if CrossBuildMountModcache { + // Mount $GOPATH/pkg/mod into the container, read-only. + hostDir := filepath.Join(build.Default.GOPATH, "pkg", "mod") + args = append(args, "-v", hostDir+":/go/pkg/mod:ro") + } args = append(args, "--rm", diff --git a/dev-tools/mage/godaemon.go b/dev-tools/mage/godaemon.go index 7511cae8044..7397dac74d9 100644 --- a/dev-tools/mage/godaemon.go +++ b/dev-tools/mage/godaemon.go @@ -21,6 +21,7 @@ import ( "errors" "log" "os" + "path/filepath" ) var ( @@ -42,8 +43,12 @@ func BuildGoDaemon() error { } // Test if binaries are up-to-date. + godaemonDir, err := listModuleDir("github.com/tsg/go-daemon") + if err != nil { + return err + } + input := filepath.Join(godaemonDir, "src", "god.c") output := MustExpand("build/golang-crossbuild/god-{{.Platform.GOOS}}-{{.Platform.Arch}}") - input := MustExpand("{{ elastic_beats_dir }}/vendor/github.com/tsg/go-daemon/src/god.c") if IsUpToDate(output, input) { log.Println(">>> buildGoDaemon is up-to-date for", Platform.Name) return nil diff --git a/dev-tools/mage/gomod.go b/dev-tools/mage/gomod.go index 7cc6b7d8c68..9d785b47971 100644 --- a/dev-tools/mage/gomod.go +++ b/dev-tools/mage/gomod.go @@ -75,7 +75,7 @@ func Vendor() error { // copy packages which require the whole tree for _, p := range copyAll { - path, err := gotool.ListModulePath(p.name) + path, err := gotool.ListModuleVendorDir(p.name) if err != nil { return err } diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go index 9e6cc08a471..9022e0c0137 100644 --- a/dev-tools/mage/gotool/go.go +++ b/dev-tools/mage/gotool/go.go @@ -93,14 +93,31 @@ func ListTestFiles(pkg string) ([]string, error) { return getLines(callGo(nil, "list", "-f", tmpl, pkg)) } -// ListModulePath returns the path to the module in the cache. -func ListModulePath(pkg string) (string, error) { - const tmpl = `{{.Dir}}` +// ListModuleCacheDir returns the module cache directory containing +// the specified module. If the module does not exist in the cache, +// an error will be returned. +func ListModuleCacheDir(pkg string) (string, error) { + return listModuleDir(pkg, false) +} + +// ListModuleVendorDir returns the vendor directory containing the +// specified module. If the module has not been vendored, an error +// will be returned. +func ListModuleVendorDir(pkg string) (string, error) { + return listModuleDir(pkg, true) +} + +func listModuleDir(pkg string, vendor bool) (string, error) { env := map[string]string{ - // make sure to look in the module cache + // Make sure GOFLAGS does not influence behaviour. "GOFLAGS": "", } - lines, err := getLines(callGo(env, "list", "-m", "-f", tmpl, pkg)) + args := []string{"-m", "-f", "{{.Dir}}"} + if vendor { + args = append(args, "-mod=vendor") + } + args = append(args, pkg) + lines, err := getLines(callGo(env, "list", args...)) if err != nil { return "", err } diff --git a/dev-tools/mage/gotool/modules.go b/dev-tools/mage/gotool/modules.go index ebaf2869ec5..d7a880756b7 100644 --- a/dev-tools/mage/gotool/modules.go +++ b/dev-tools/mage/gotool/modules.go @@ -19,10 +19,11 @@ package gotool // Mod is the command go mod. var Mod = goMod{ - Init: modCommand{"init"}.run, - Tidy: modCommand{"tidy"}.run, - Verify: modCommand{"verify"}.run, - Vendor: modCommand{"vendor"}.run, + Download: modCommand{"download"}.run, + Init: modCommand{"init"}.run, + Tidy: modCommand{"tidy"}.run, + Verify: modCommand{"verify"}.run, + Vendor: modCommand{"vendor"}.run, } type modCommand struct { @@ -40,12 +41,16 @@ func (cmd modCommand) run(opts ...ArgOpt) error { } type goMod struct { - Init modInit - Tidy modTidy - Verify modVerify - Vendor modVendor + Download modDownload + Init modInit + Tidy modTidy + Verify modVerify + Vendor modVendor } +// modDownload cleans the go.mod file +type modDownload func(opts ...ArgOpt) error + // modInit initializes a new go module in folder. type modInit func(opts ...ArgOpt) error diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 5b96e58e9f0..66b867057ca 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -60,6 +60,10 @@ var ( TestCoverage = false UseVendor = true + // CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into + // the crossbuild images at /go/pkg/mod, read-only. + CrossBuildMountModcache = false + BeatName = EnvOr("BEAT_NAME", filepath.Base(CWD())) BeatServiceName = EnvOr("BEAT_SERVICE_NAME", BeatName) BeatIndexPrefix = EnvOr("BEAT_INDEX_PREFIX", BeatName) @@ -280,8 +284,7 @@ func findElasticBeatsDir() (string, error) { if repo.IsElasticBeats() { return repo.RootDir, nil } - - return gotool.ListModulePath("github.com/elastic/beats/v7") + return listModuleDir("github.com/elastic/beats/v7") } var ( diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go index a4a1f436a13..e92bbc51276 100644 --- a/generator/common/beatgen/beatgen.go +++ b/generator/common/beatgen/beatgen.go @@ -101,6 +101,12 @@ func Generate() error { return err } + // Make sure the ElasticBeatsDir value is cached + // before changing directory below. + if _, err := devtools.ElasticBeatsDir(); err != nil { + return err + } + err = setup.GenNewBeat(cfg) if err != nil { return errors.Wrap(err, "error generating new beat") diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go index c4ef95c7654..b372660971d 100644 --- a/generator/common/beatgen/setup/setup.go +++ b/generator/common/beatgen/setup/setup.go @@ -112,7 +112,7 @@ func CopyVendor() error { return err } - path, err := gotool.ListModulePath("github.com/elastic/beats/v7") + path, err := gotool.ListModuleCacheDir("github.com/elastic/beats/v7") if err != nil { return err }