Skip to content

Commit

Permalink
Update go version to 1.14.4 (elastic#19753)
Browse files Browse the repository at this point in the history
This PR adds support for Golang 1.14.4 in Beats. The dependency `golang.org/x/tools` is pinned to `release-branch.go1.14`.

The generated Beats no longer contain a vendor folder.

Go 1.14 has been out for quite some time. It is high time we update.

Closes elastic#18829
Closes elastic#16566

(cherry picked from commit 7580c72)
  • Loading branch information
kvch committed Jul 13, 2020
1 parent 83a678a commit 5972891
Show file tree
Hide file tree
Showing 36 changed files with 48 additions and 125 deletions.
2 changes: 1 addition & 1 deletion .go-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.13.10
1.14.4
1 change: 1 addition & 0 deletions CHANGELOG-developer.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -88,3 +88,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
- Add fields validation for histogram subfields. {pull}17759[17759]
- Events intended for the Elasticsearch output can now take an `op_type` metadata field of type events.OpType or string to indicate the `op_type` to use for bulk indexing. {pull}12606[12606]
- Remove vendor folder from repository. {pull}18655[18655]
- Update Go version to 1.14.4. {pull}19753[19753]
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ check: python-env
@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
@$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
@$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false)
# check if vendor folder does not exists
[ ! -d vendor ]
@# Validate that all updates were committed
@$(MAKE) update
@$(MAKE) check-headers
Expand Down
4 changes: 2 additions & 2 deletions NOTICE.txt
Original file line number Diff line number Diff line change
Expand Up @@ -15460,11 +15460,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

--------------------------------------------------------------------------------
Dependency : golang.org/x/tools
Version: v0.0.0-20200615191743-991d59a616de
Version: v0.0.0-20200602230032-c00d67ef29d0
Licence type (autodetected): BSD-3-Clause
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20200615191743-991d59a616de/LICENSE:
Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]20200602230032-c00d67ef29d0/LICENSE:

Copyright (c) 2009 The Go Authors. All rights reserved.

Expand Down
2 changes: 1 addition & 1 deletion auditbeat/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.13.10
FROM golang:1.14.4

RUN \
apt-get update \
Expand Down
10 changes: 0 additions & 10 deletions dev-tools/mage/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,16 +118,6 @@ func Build(params BuildArgs) error {
}
env["CGO_ENABLED"] = cgoEnabled

if UseVendor {
var goFlags string
goFlags, ok := env["GOFLAGS"]
if !ok {
env["GOFLAGS"] = "-mod=vendor"
} else {
env["GOFLAGS"] = strings.Join([]string{goFlags, "-mod=vendor"}, " ")
}
}

// Spec
args := []string{
"build",
Expand Down
12 changes: 0 additions & 12 deletions dev-tools/mage/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,6 @@ import (
"github.com/magefile/mage/sh"
"github.com/magefile/mage/target"
"github.com/pkg/errors"

"github.com/elastic/beats/v7/dev-tools/mage/gotool"
)

// Expand expands the given Go text/template string.
Expand Down Expand Up @@ -784,16 +782,6 @@ func binaryExtension(goos string) string {
return ""
}

// listModuleDir calls gotool.ListModuleVendorDir or
// gotool.ListModuleCacheDir, depending on the value of
// UseVendor.
func listModuleDir(modpath string) (string, error) {
if UseVendor {
return gotool.ListModuleVendorDir(modpath)
}
return gotool.ListModuleCacheDir(modpath)
}

var parseVersionRegex = regexp.MustCompile(`(?m)^[^\d]*(?P<major>\d)+\.(?P<minor>\d)+(?:\.(?P<patch>\d)+.*)?$`)

// ParseVersion extracts the major, minor, and optional patch number from a
Expand Down
3 changes: 0 additions & 3 deletions dev-tools/mage/crossbuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -255,9 +255,6 @@ func (b GolangCrossBuilder) Build() error {
if versionQualified {
args = append(args, "--env", "VERSION_QUALIFIER="+versionQualifier)
}
if UseVendor {
args = append(args, "--env", "GOFLAGS=-mod=vendor")
}
if CrossBuildMountModcache {
// Mount $GOPATH/pkg/mod into the container, read-only.
hostDir := filepath.Join(build.Default.GOPATH, "pkg", "mod")
Expand Down
32 changes: 8 additions & 24 deletions dev-tools/mage/fields.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,16 +101,12 @@ func generateFieldsYAML(baseDir, output string, moduleDirs ...string) error {
return err
}

cmd := []string{"run"}
if UseVendor {
cmd = append(cmd, "-mod", "vendor")
}
cmd = append(cmd,
cmd := []string{"run",
filepath.Join(beatsDir, globalFieldsCmdPath),
"-es_beats_path", beatsDir,
"-beat_path", baseDir,
"-out", CreateDir(output),
)
}
globalFieldsCmd := sh.RunCmd("go", cmd...)

return globalFieldsCmd(moduleDirs...)
Expand All @@ -130,18 +126,14 @@ func GenerateFieldsGo(fieldsYML, out string) error {
return err
}

cmd := []string{"run"}
if UseVendor {
cmd = append(cmd, "-mod", "vendor")
}
cmd = append(cmd,
cmd := []string{"run",
filepath.Join(beatsDir, assetCmdPath),
"-pkg", "include",
"-in", fieldsYML,
"-out", CreateDir(out),
"-license", toLibbeatLicenseName(BeatLicense),
BeatName,
)
}
assetCmd := sh.RunCmd("go", cmd...)

return assetCmd()
Expand All @@ -162,16 +154,12 @@ func GenerateModuleFieldsGo(moduleDir string) error {
moduleDir = CWD(moduleDir)
}

cmd := []string{"run"}
if UseVendor {
cmd = append(cmd, "-mod", "vendor")
}
cmd = append(cmd,
cmd := []string{"run",
filepath.Join(beatsDir, moduleFieldsCmdPath),
"-beat", BeatName,
"-license", toLibbeatLicenseName(BeatLicense),
moduleDir,
)
}
moduleFieldsCmd := sh.RunCmd("go", cmd...)

return moduleFieldsCmd()
Expand All @@ -194,16 +182,12 @@ func GenerateIncludeListGo(options IncludeListOptions) error {
return err
}

cmd := []string{"run"}
if UseVendor {
cmd = append(cmd, "-mod", "vendor")
}
cmd = append(cmd,
cmd := []string{"run",
filepath.Join(beatsDir, moduleIncludeListCmdPath),
"-license", toLibbeatLicenseName(BeatLicense),
"-out", options.Outfile, "-buildTags", options.BuildTags,
"-pkg", options.Pkg,
)
}

includeListCmd := sh.RunCmd("go", cmd...)

Expand Down
17 changes: 4 additions & 13 deletions dev-tools/mage/fmt.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,19 +66,10 @@ func GoImports() error {
}

fmt.Println(">> fmt - goimports: Formatting Go code")
if UseVendor {
if err := gotool.Install(
gotool.Install.Vendored(),
gotool.Install.Package(filepath.Join(GoImportsImportPath)),
); err != nil {
return err
}
} else {
if err := gotool.Install(
gotool.Install.Package(filepath.Join(GoImportsImportPath)),
); err != nil {
return err
}
if err := gotool.Install(
gotool.Install.Package(filepath.Join(GoImportsImportPath)),
); err != nil {
return err
}

args := append(
Expand Down
4 changes: 3 additions & 1 deletion dev-tools/mage/godaemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
"log"
"os"
"path/filepath"

"github.com/elastic/beats/v7/dev-tools/mage/gotool"
)

var (
Expand All @@ -43,7 +45,7 @@ func BuildGoDaemon() error {
}

// Test if binaries are up-to-date.
godaemonDir, err := listModuleDir("github.com/tsg/go-daemon")
godaemonDir, err := gotool.ListModuleCacheDir("github.com/tsg/go-daemon")
if err != nil {
return err
}
Expand Down
3 changes: 0 additions & 3 deletions dev-tools/mage/install.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,6 @@ func InstallVendored(importPath string) error {

// InstallGoLicenser target installs go-licenser
func InstallGoLicenser() error {
if UseVendor {
return InstallVendored(GoLicenserImportPath)
}
return gotool.Get(
gotool.Get.Package(GoLicenserImportPath),
)
Expand Down
3 changes: 0 additions & 3 deletions dev-tools/mage/integtest.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,9 +245,6 @@ func initRunner(tester IntegrationTester, dir string, passInEnv map[string]strin
if mg.Verbose() {
env["MAGEFILE_VERBOSE"] = "1"
}
if UseVendor {
env["GOFLAGS"] = "-mod=vendor"
}

runner := &IntegrationRunner{
steps: runnerSteps,
Expand Down
7 changes: 1 addition & 6 deletions dev-tools/mage/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@ var (
XPackDir = "../x-pack"
RaceDetector = false
TestCoverage = false
UseVendor = false

// CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into
// the crossbuild images at /go/pkg/mod, read-only.
Expand Down Expand Up @@ -110,10 +109,6 @@ func init() {
if err != nil {
panic(errors.Wrap(err, "failed to parse TEST_COVERAGE env value"))
}
UseVendor, err = strconv.ParseBool(EnvOr("USE_VENDOR", "false"))
if err != nil {
panic(errors.Wrap(err, "failed to parse USE_VENDOR env value"))
}

Snapshot, err = strconv.ParseBool(EnvOr("SNAPSHOT", "false"))
if err != nil {
Expand Down Expand Up @@ -286,7 +281,7 @@ func findElasticBeatsDir() (string, error) {
if repo.IsElasticBeats() {
return repo.RootDir, nil
}
return listModuleDir(elasticBeatsModulePath)
return gotool.ListModuleCacheDir(elasticBeatsModulePath)
}

var (
Expand Down
4 changes: 2 additions & 2 deletions dev-tools/packaging/packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@ shared:
- &macos_agent_pkg_spec
<<: *common
extra_vars:
# OS X 10.11 Mountain Lion is the oldest supported by Go 1.13.
# https://golang.org/doc/go1.13#ports
# OS X 10.11 El Capitan is the oldest supported by Go 1.14.
# https://golang.org/doc/go1.14#ports
min_supported_osx_version: 10.11
identifier: 'co.{{.BeatVendor | tolower}}.beats.{{.BeatName}}'
install_path: /Library/Application Support
Expand Down
4 changes: 0 additions & 4 deletions docs/devguide/newbeat.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -515,9 +515,6 @@ To depend on the latest `master` of `github.com/elastic/beats` run the following
go get github.com/elastic/beats@master
----------------------------------------------------------------------

To move the dependencies to vendor, you need to manually fetch the new
`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`.

We suggest you read the following section to learn about maintaining dependencies
using go modules:
* https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies]
Expand All @@ -526,6 +523,5 @@ Please note that it is your choice whether you put dependencies in the folder ve

If you choose to abandon the vendor folder, you have to adjust the following things in your Beat:

* add `devtools.UseVendor = false` to the `magefile.go` of your Beat
* add `devtools.CrossBuildMountModcache = true` to the `magefile.go` of your Beat
* modify the path to `ES_BEATS` in the `Makefile` so it points to the folder under the module cache (`go list -m -f '{{.Dir}}' $ES_BEATS_IMPORT_PATH`)
2 changes: 1 addition & 1 deletion filebeat/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.13.10
FROM golang:1.14.4

RUN \
apt-get update \
Expand Down
2 changes: 1 addition & 1 deletion generator/_templates/beat/{beat}/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
SYSTEM_TESTS=false
TEST_ENVIRONMENT=false
ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
ES_BEATS?=$(shell go list -m -f '{{.Dir}}' ${ES_BEATS_IMPORT_PATH})
LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile
GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"
Expand Down
7 changes: 1 addition & 6 deletions generator/_templates/beat/{beat}/magefile.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"github.com/elastic/beats/v7/dev-tools/mage/target/common"
"github.com/elastic/beats/v7/dev-tools/mage/target/pkg"
"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
"github.com/elastic/beats/v7/generator/common/beatgen"
)

func init() {
Expand All @@ -23,11 +22,7 @@ func init() {
devtools.BeatDescription = "One sentence description of the Beat."
devtools.BeatVendor = "{full_name}"
devtools.BeatProjectType = devtools.CommunityProject
}

// VendorUpdate updates the vendor dir
func VendorUpdate() error {
return beatgen.VendorUpdate()
devtools.CrossBuildMountModcache = true
}

// Package packages the Beat for distribution.
Expand Down
6 changes: 1 addition & 5 deletions generator/_templates/metricbeat/{beat}/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,11 @@ BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
SYSTEM_TESTS=false
TEST_ENVIRONMENT=false
ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
ES_BEATS?=$(shell go list -m -f '{{.Dir}}' ${ES_BEATS_IMPORT_PATH})
GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"
MAGE_IMPORT_PATH=github.com/magefile/mage
CHECK_HEADERS_DISABLED=true

# Path to the libbeat Makefile
-include $(ES_BEATS)/metricbeat/Makefile

.PHONY: copy-vendor
copy-vendor:
mage vendorUpdate
7 changes: 1 addition & 6 deletions generator/_templates/metricbeat/{beat}/magefile.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"github.com/elastic/beats/v7/dev-tools/mage/target/common"
"github.com/elastic/beats/v7/dev-tools/mage/target/pkg"
"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
"github.com/elastic/beats/v7/generator/common/beatgen"
metricbeat "github.com/elastic/beats/v7/metricbeat/scripts/mage"

// mage:import
Expand All @@ -26,11 +25,7 @@ func init() {

devtools.BeatDescription = "One sentence description of the Beat."
devtools.BeatVendor = "{full_name}"
}

// VendorUpdate updates elastic/beats/v7 in the vendor dir
func VendorUpdate() error {
return beatgen.VendorUpdate()
devtools.CrossBuildMountModcache = true
}

// CollectAll generates the docs and the fields.
Expand Down
1 change: 0 additions & 1 deletion generator/common/beatgen/beatgen.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,6 @@ func Generate() error {
return errors.Wrap(err, "error while getting required beats version")
}

mg.Deps(setup.CopyVendor)
mg.Deps(setup.GitInit)

if cfg["type"] == "metricbeat" {
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/elastic/beats/v7

go 1.13
go 1.14

require (
4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b
Expand Down Expand Up @@ -190,5 +190,5 @@ replace (
github.com/google/gopacket => github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41
github.com/insomniacslk/dhcp => github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 // indirect
github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c
golang.org/x/tools => golang.org/x/tools v0.0.0-20200615191743-991d59a616de // release 1.13
golang.org/x/tools => golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0 // release 1.14
)
5 changes: 3 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -844,8 +844,9 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20200615191743-991d59a616de h1:kFKSx8iHlOzmtGWtfJW+b2UzcJ+rMWHHyUBpjrZq8To=
golang.org/x/tools v0.0.0-20200615191743-991d59a616de/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0 h1:6txNFSnY+tteYoO+hf01EpdYcYZiurdC9MDIrcUzEu4=
golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
Loading

0 comments on commit 5972891

Please sign in to comment.