diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy index 376ceb4190f..d66072fc435 100644 --- a/.ci/apm-beats-update.groovy +++ b/.ci/apm-beats-update.groovy @@ -35,7 +35,14 @@ pipeline { when { beforeAgent true expression { - return isCommentTrigger() || isUserTrigger() + def ret = isCommentTrigger() || isUserTrigger() + if(!ret){ + currentBuild.result = 'NOT_BUILT' + currentBuild.description = "The build has been skipped" + currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)" + echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, and GitHub comment") + } + return ret } } /** diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 91b94221c99..280abe2f9e7 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -38,7 +38,14 @@ pipeline { when { beforeAgent true expression { - return isCommentTrigger() || isUserTrigger() || isUpstreamTrigger() + def ret = isCommentTrigger() || isUserTrigger() || isUpstreamTrigger() + if(!ret){ + currentBuild.result = 'NOT_BUILT' + currentBuild.description = "The build has been skipped" + currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)" + echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job") + } + return ret } } stages { diff --git a/.go-version b/.go-version index f3352b3fef9..4e00d0ac079 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.13.10 +1.14.4 diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 20b7c8f8a58..75da545123d 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -95,3 +95,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Add IP* fields to `fields.yml` generator script in Filebeat. {issue}17998[17998] {pull}18256[18256] - Events intended for the Elasticsearch output can now take an `op_type` metadata field of type events.OpType or string to indicate the `op_type` to use for bulk indexing. {pull}12606[12606] - Remove vendor folder from repository. {pull}18655[18655] +- Update Go version to 1.14.4. {pull}19753[19753] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index be2001edb9b..76ebb7f0002 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -148,7 +148,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - system/package: Fix parsing of Installed-Size field of DEB packages. {issue}16661[16661] {pull}17188[17188] - system module: Fix panic during initialisation when /proc/stat can't be read. {pull}17569[17569] - system/package: Fix an error that can occur while trying to persist package metadata. {issue}18536[18536] {pull}18887[18887] -- system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033] +- system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033] {pull}19764[19764] - system/socket: Fixed tracking of long-running connections. {pull}19033[19033] - system/package: Fix librpm loading on Fedora 31/32. {pull}NNNN[NNNN] - file_integrity: Create fsnotify watcher only when starting file_integrity module {pull}19505[19505] @@ -560,6 +560,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add registry and code signature information and ECS categorization fields for sysmon module {pull}18058[18058] - Add new winlogbeat security dashboard {pull}18775[18775] +*Elastic Log Driver* +- Add support for `docker logs` command {pull}19531[19531] + ==== Deprecated *Affecting all Beats* diff --git a/Makefile b/Makefile index 0e21f801f10..11e9c72f72d 100644 --- a/Makefile +++ b/Makefile @@ -98,6 +98,8 @@ check: python-env @$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;) @$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) @$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false) + # check if vendor folder does not exists + [ ! -d vendor ] @# Validate that all updates were committed @$(MAKE) update @$(MAKE) check-headers diff --git a/NOTICE.txt b/NOTICE.txt index 6df219fffc1..1ed31b1ddb0 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -11014,6 +11014,25 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice +-------------------------------------------------------------------------------- +Dependency : github.com/hectane/go-acl +Version: v0.0.0-20190604041725-da78bae5fc95 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/hectane/go-acl@v0.0.0-20190604041725-da78bae5fc95/LICENSE.txt: + +The MIT License (MIT) + +Copyright (c) 2015 Nathan Osman + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/dhcp Version: v0.0.0-20200227161230-57ec251c7eb3 @@ -15626,11 +15645,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/tools -Version: v0.0.0-20200615191743-991d59a616de +Version: v0.0.0-20200602230032-c00d67ef29d0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.0.0-20200615191743-991d59a616de/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.0.0-20200602230032-c00d67ef29d0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 18a9344addf..194289595e2 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.10 +FROM golang:1.14.4 RUN \ apt-get update \ diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index d6ac3eb631f..2efe61502ae 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -118,16 +118,6 @@ func Build(params BuildArgs) error { } env["CGO_ENABLED"] = cgoEnabled - if UseVendor { - var goFlags string - goFlags, ok := env["GOFLAGS"] - if !ok { - env["GOFLAGS"] = "-mod=vendor" - } else { - env["GOFLAGS"] = strings.Join([]string{goFlags, "-mod=vendor"}, " ") - } - } - // Spec args := []string{ "build", diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index 91d291f5387..e4b93625b75 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -49,8 +49,6 @@ import ( "github.com/magefile/mage/sh" "github.com/magefile/mage/target" "github.com/pkg/errors" - - "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) // Expand expands the given Go text/template string. @@ -806,16 +804,6 @@ func ParseVersion(version string) (major, minor, patch int, err error) { return } -// listModuleDir calls gotool.ListModuleVendorDir or -// gotool.ListModuleCacheDir, depending on the value of -// UseVendor. -func listModuleDir(modpath string) (string, error) { - if UseVendor { - return gotool.ListModuleVendorDir(modpath) - } - return gotool.ListModuleCacheDir(modpath) -} - // ListMatchingEnvVars returns all of the environment variables names that begin // with prefix. func ListMatchingEnvVars(prefixes ...string) []string { diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index d80276a7974..5362426aba2 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -255,9 +255,6 @@ func (b GolangCrossBuilder) Build() error { if versionQualified { args = append(args, "--env", "VERSION_QUALIFIER="+versionQualifier) } - if UseVendor { - args = append(args, "--env", "GOFLAGS=-mod=vendor") - } if CrossBuildMountModcache { // Mount $GOPATH/pkg/mod into the container, read-only. hostDir := filepath.Join(build.Default.GOPATH, "pkg", "mod") diff --git a/dev-tools/mage/fields.go b/dev-tools/mage/fields.go index cb2b6ef7685..94c3f899e9e 100644 --- a/dev-tools/mage/fields.go +++ b/dev-tools/mage/fields.go @@ -101,16 +101,12 @@ func generateFieldsYAML(baseDir, output string, moduleDirs ...string) error { return err } - cmd := []string{"run"} - if UseVendor { - cmd = append(cmd, "-mod", "vendor") - } - cmd = append(cmd, + cmd := []string{"run", filepath.Join(beatsDir, globalFieldsCmdPath), "-es_beats_path", beatsDir, "-beat_path", baseDir, "-out", CreateDir(output), - ) + } globalFieldsCmd := sh.RunCmd("go", cmd...) return globalFieldsCmd(moduleDirs...) @@ -130,18 +126,14 @@ func GenerateFieldsGo(fieldsYML, out string) error { return err } - cmd := []string{"run"} - if UseVendor { - cmd = append(cmd, "-mod", "vendor") - } - cmd = append(cmd, + cmd := []string{"run", filepath.Join(beatsDir, assetCmdPath), "-pkg", "include", "-in", fieldsYML, "-out", CreateDir(out), "-license", toLibbeatLicenseName(BeatLicense), BeatName, - ) + } assetCmd := sh.RunCmd("go", cmd...) return assetCmd() @@ -162,16 +154,12 @@ func GenerateModuleFieldsGo(moduleDir string) error { moduleDir = CWD(moduleDir) } - cmd := []string{"run"} - if UseVendor { - cmd = append(cmd, "-mod", "vendor") - } - cmd = append(cmd, + cmd := []string{"run", filepath.Join(beatsDir, moduleFieldsCmdPath), "-beat", BeatName, "-license", toLibbeatLicenseName(BeatLicense), moduleDir, - ) + } moduleFieldsCmd := sh.RunCmd("go", cmd...) return moduleFieldsCmd() @@ -194,16 +182,12 @@ func GenerateIncludeListGo(options IncludeListOptions) error { return err } - cmd := []string{"run"} - if UseVendor { - cmd = append(cmd, "-mod", "vendor") - } - cmd = append(cmd, + cmd := []string{"run", filepath.Join(beatsDir, moduleIncludeListCmdPath), "-license", toLibbeatLicenseName(BeatLicense), "-out", options.Outfile, "-buildTags", options.BuildTags, "-pkg", options.Pkg, - ) + } includeListCmd := sh.RunCmd("go", cmd...) diff --git a/dev-tools/mage/fmt.go b/dev-tools/mage/fmt.go index b4463696b81..ed034e3fc9a 100644 --- a/dev-tools/mage/fmt.go +++ b/dev-tools/mage/fmt.go @@ -66,19 +66,10 @@ func GoImports() error { } fmt.Println(">> fmt - goimports: Formatting Go code") - if UseVendor { - if err := gotool.Install( - gotool.Install.Vendored(), - gotool.Install.Package(filepath.Join(GoImportsImportPath)), - ); err != nil { - return err - } - } else { - if err := gotool.Install( - gotool.Install.Package(filepath.Join(GoImportsImportPath)), - ); err != nil { - return err - } + if err := gotool.Install( + gotool.Install.Package(filepath.Join(GoImportsImportPath)), + ); err != nil { + return err } args := append( diff --git a/dev-tools/mage/godaemon.go b/dev-tools/mage/godaemon.go index 7397dac74d9..b580bc3ed5e 100644 --- a/dev-tools/mage/godaemon.go +++ b/dev-tools/mage/godaemon.go @@ -22,6 +22,8 @@ import ( "log" "os" "path/filepath" + + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) var ( @@ -43,7 +45,7 @@ func BuildGoDaemon() error { } // Test if binaries are up-to-date. - godaemonDir, err := listModuleDir("github.com/tsg/go-daemon") + godaemonDir, err := gotool.ListModuleCacheDir("github.com/tsg/go-daemon") if err != nil { return err } diff --git a/dev-tools/mage/install.go b/dev-tools/mage/install.go index 67483d23bb9..5220eb25be3 100644 --- a/dev-tools/mage/install.go +++ b/dev-tools/mage/install.go @@ -37,9 +37,6 @@ func InstallVendored(importPath string) error { // InstallGoLicenser target installs go-licenser func InstallGoLicenser() error { - if UseVendor { - return InstallVendored(GoLicenserImportPath) - } return gotool.Get( gotool.Get.Package(GoLicenserImportPath), ) diff --git a/dev-tools/mage/integtest.go b/dev-tools/mage/integtest.go index 064dc7d8102..e932a55c45d 100644 --- a/dev-tools/mage/integtest.go +++ b/dev-tools/mage/integtest.go @@ -245,9 +245,6 @@ func initRunner(tester IntegrationTester, dir string, passInEnv map[string]strin if mg.Verbose() { env["MAGEFILE_VERBOSE"] = "1" } - if UseVendor { - env["GOFLAGS"] = "-mod=vendor" - } runner := &IntegrationRunner{ steps: runnerSteps, diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index fb7644a90b2..2473202648e 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -60,7 +60,6 @@ var ( XPackDir = "../x-pack" RaceDetector = false TestCoverage = false - UseVendor = false // CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into // the crossbuild images at /go/pkg/mod, read-only. @@ -110,10 +109,6 @@ func init() { if err != nil { panic(errors.Wrap(err, "failed to parse TEST_COVERAGE env value")) } - UseVendor, err = strconv.ParseBool(EnvOr("USE_VENDOR", "false")) - if err != nil { - panic(errors.Wrap(err, "failed to parse USE_VENDOR env value")) - } Snapshot, err = strconv.ParseBool(EnvOr("SNAPSHOT", "false")) if err != nil { @@ -286,7 +281,7 @@ func findElasticBeatsDir() (string, error) { if repo.IsElasticBeats() { return repo.RootDir, nil } - return listModuleDir(elasticBeatsModulePath) + return gotool.ListModuleCacheDir(elasticBeatsModulePath) } var ( diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index d6d9026f055..a81b683f6e8 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -72,8 +72,8 @@ shared: - &macos_agent_pkg_spec <<: *common extra_vars: - # OS X 10.11 Mountain Lion is the oldest supported by Go 1.13. - # https://golang.org/doc/go1.13#ports + # OS X 10.11 El Capitan is the oldest supported by Go 1.14. + # https://golang.org/doc/go1.14#ports min_supported_osx_version: 10.11 identifier: 'co.{{.BeatVendor | tolower}}.beats.{{.BeatName}}' install_path: /Library/Application Support diff --git a/docs/devguide/newbeat.asciidoc b/docs/devguide/newbeat.asciidoc index 5c9d427e0f9..a7f239f22fc 100644 --- a/docs/devguide/newbeat.asciidoc +++ b/docs/devguide/newbeat.asciidoc @@ -515,9 +515,6 @@ To depend on the latest `master` of `github.com/elastic/beats` run the following go get github.com/elastic/beats@master ---------------------------------------------------------------------- -To move the dependencies to vendor, you need to manually fetch the new -`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`. - We suggest you read the following section to learn about maintaining dependencies using go modules: * https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies] @@ -526,6 +523,5 @@ Please note that it is your choice whether you put dependencies in the folder ve If you choose to abandon the vendor folder, you have to adjust the following things in your Beat: -* add `devtools.UseVendor = false` to the `magefile.go` of your Beat * add `devtools.CrossBuildMountModcache = true` to the `magefile.go` of your Beat * modify the path to `ES_BEATS` in the `Makefile` so it points to the folder under the module cache (`go list -m -f '{{.Dir}}' $ES_BEATS_IMPORT_PATH`) diff --git a/filebeat/Dockerfile b/filebeat/Dockerfile index 2800e5b1398..e4aec49417d 100644 --- a/filebeat/Dockerfile +++ b/filebeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.10 +FROM golang:1.14.4 RUN \ apt-get update \ diff --git a/generator/_templates/beat/{beat}/Makefile b/generator/_templates/beat/{beat}/Makefile index 6ad41d81456..94fe4131503 100644 --- a/generator/_templates/beat/{beat}/Makefile +++ b/generator/_templates/beat/{beat}/Makefile @@ -4,7 +4,7 @@ BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH})) SYSTEM_TESTS=false TEST_ENVIRONMENT=false ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7 -ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH} +ES_BEATS?=$(shell go list -m -f '{{.Dir}}' ${ES_BEATS_IMPORT_PATH}) LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools) GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)" diff --git a/generator/_templates/beat/{beat}/magefile.go b/generator/_templates/beat/{beat}/magefile.go index 28638df0f0a..b8dd789f418 100644 --- a/generator/_templates/beat/{beat}/magefile.go +++ b/generator/_templates/beat/{beat}/magefile.go @@ -14,7 +14,6 @@ import ( "github.com/elastic/beats/v7/dev-tools/mage/target/common" "github.com/elastic/beats/v7/dev-tools/mage/target/pkg" "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" - "github.com/elastic/beats/v7/generator/common/beatgen" ) func init() { @@ -23,11 +22,7 @@ func init() { devtools.BeatDescription = "One sentence description of the Beat." devtools.BeatVendor = "{full_name}" devtools.BeatProjectType = devtools.CommunityProject -} - -// VendorUpdate updates the vendor dir -func VendorUpdate() error { - return beatgen.VendorUpdate() + devtools.CrossBuildMountModcache = true } // Package packages the Beat for distribution. diff --git a/generator/_templates/metricbeat/{beat}/Makefile b/generator/_templates/metricbeat/{beat}/Makefile index 453344c8feb..ebdc51b72fb 100644 --- a/generator/_templates/metricbeat/{beat}/Makefile +++ b/generator/_templates/metricbeat/{beat}/Makefile @@ -4,7 +4,7 @@ BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH})) SYSTEM_TESTS=false TEST_ENVIRONMENT=false ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7 -ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH} +ES_BEATS?=$(shell go list -m -f '{{.Dir}}' ${ES_BEATS_IMPORT_PATH}) GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools) GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)" MAGE_IMPORT_PATH=github.com/magefile/mage @@ -12,7 +12,3 @@ CHECK_HEADERS_DISABLED=true # Path to the libbeat Makefile -include $(ES_BEATS)/metricbeat/Makefile - -.PHONY: copy-vendor -copy-vendor: - mage vendorUpdate diff --git a/generator/_templates/metricbeat/{beat}/magefile.go b/generator/_templates/metricbeat/{beat}/magefile.go index 8805fc36a32..c05ac7623b3 100644 --- a/generator/_templates/metricbeat/{beat}/magefile.go +++ b/generator/_templates/metricbeat/{beat}/magefile.go @@ -14,7 +14,6 @@ import ( "github.com/elastic/beats/v7/dev-tools/mage/target/common" "github.com/elastic/beats/v7/dev-tools/mage/target/pkg" "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" - "github.com/elastic/beats/v7/generator/common/beatgen" metricbeat "github.com/elastic/beats/v7/metricbeat/scripts/mage" // mage:import @@ -26,11 +25,7 @@ func init() { devtools.BeatDescription = "One sentence description of the Beat." devtools.BeatVendor = "{full_name}" -} - -// VendorUpdate updates elastic/beats/v7 in the vendor dir -func VendorUpdate() error { - return beatgen.VendorUpdate() + devtools.CrossBuildMountModcache = true } // CollectAll generates the docs and the fields. diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go index 72c25c7f30c..3cb101aa625 100644 --- a/generator/common/beatgen/beatgen.go +++ b/generator/common/beatgen/beatgen.go @@ -126,7 +126,6 @@ func Generate() error { return errors.Wrap(err, "error while getting required beats version") } - mg.Deps(setup.CopyVendor) mg.Deps(setup.GitInit) if cfg["type"] == "metricbeat" { diff --git a/go.mod b/go.mod index 6262a2a5c32..5f7655f143c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/beats/v7 -go 1.13 +go 1.14 require ( 4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b @@ -100,6 +100,7 @@ require ( github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/go-retryablehttp v0.6.6 github.com/hashicorp/golang-lru v0.5.2-0.20190520140433-59383c442f7d // indirect + github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 github.com/insomniacslk/dhcp v0.0.0-20180716145214-633285ba52b2 github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 @@ -190,5 +191,5 @@ replace ( github.com/google/gopacket => github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41 github.com/insomniacslk/dhcp => github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 // indirect github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c - golang.org/x/tools => golang.org/x/tools v0.0.0-20200615191743-991d59a616de // release 1.13 + golang.org/x/tools => golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0 // release 1.14 ) diff --git a/go.sum b/go.sum index 8efe86a2da6..fd1e638b44b 100644 --- a/go.sum +++ b/go.sum @@ -427,6 +427,8 @@ github.com/haya14busa/go-actions-toolkit v0.0.0-20200105081403-ca0307860f01 h1:H github.com/haya14busa/go-actions-toolkit v0.0.0-20200105081403-ca0307860f01/go.mod h1:1DWDZmeYf0LX30zscWb7K9rUMeirNeBMd5Dum+seUhc= github.com/haya14busa/go-checkstyle v0.0.0-20170303121022-5e9d09f51fa1/go.mod h1:RsN5RGgVYeXpcXNtWyztD5VIe7VNSEqpJvF2iEH7QvI= github.com/haya14busa/secretbox v0.0.0-20180525171038-07c7ecf409f5/go.mod h1:FGO/dXIFZnan7KvvUSFk1hYMnoVNzB6NTMPrmke8SSI= +github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 h1:S4qyfL2sEm5Budr4KVMyEniCy+PbS55651I/a+Kn/NQ= +github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -818,6 +820,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190529164535-6a60838ec259/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -843,8 +846,9 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20200615191743-991d59a616de h1:kFKSx8iHlOzmtGWtfJW+b2UzcJ+rMWHHyUBpjrZq8To= -golang.org/x/tools v0.0.0-20200615191743-991d59a616de/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0 h1:6txNFSnY+tteYoO+hf01EpdYcYZiurdC9MDIrcUzEu4= +golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index f1f920d0f00..1ea4bda3cc6 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.10 +FROM golang:1.14.4 RUN \ apt-get update \ diff --git a/journalbeat/Dockerfile b/journalbeat/Dockerfile index 770849908b8..b003f0da5c5 100644 --- a/journalbeat/Dockerfile +++ b/journalbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.10 +FROM golang:1.14.4 RUN \ apt-get update \ diff --git a/libbeat/Dockerfile b/libbeat/Dockerfile index 9dcad5b3735..9b89e87b685 100644 --- a/libbeat/Dockerfile +++ b/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.10 +FROM golang:1.14.4 RUN \ apt-get update \ diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index f273b6a8fac..a4936388f9e 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 8.0.0 :doc-branch: master -:go-version: 1.13.10 +:go-version: 1.14.4 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 508a6edc915..e3bd6006dc5 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.10 +FROM golang:1.14.4 RUN \ apt update \ diff --git a/metricbeat/module/apache/status/status_test.go b/metricbeat/module/apache/status/status_test.go index 182b41e1af3..c7346278c9e 100644 --- a/metricbeat/module/apache/status/status_test.go +++ b/metricbeat/module/apache/status/status_test.go @@ -178,12 +178,12 @@ func TestFetchTimeout(t *testing.T) { elapsed := time.Since(start) var found bool for _, err := range errs { - if strings.Contains(err.Error(), "request canceled (Client.Timeout exceeded") { + if strings.Contains(err.Error(), "context deadline exceeded (Client.Timeout exceeded") { found = true } } if !found { - assert.Failf(t, "", "expected an error containing 'request canceled (Client.Timeout exceeded'. Got %v", errs) + assert.Failf(t, "", "expected an error containing 'context deadline exceeded (Client.Timeout exceeded'. Got %v", errs) } // Elapsed should be ~50ms, sometimes it can be up to 1s diff --git a/metricbeat/module/envoyproxy/server/server_test.go b/metricbeat/module/envoyproxy/server/server_test.go index c4f8eb77594..df3b00886c9 100644 --- a/metricbeat/module/envoyproxy/server/server_test.go +++ b/metricbeat/module/envoyproxy/server/server_test.go @@ -184,12 +184,12 @@ func TestFetchTimeout(t *testing.T) { elapsed := time.Since(start) var found bool for _, err := range errs { - if strings.Contains(err.Error(), "request canceled (Client.Timeout exceeded") { + if strings.Contains(err.Error(), "context deadline exceeded (Client.Timeout exceeded") { found = true } } if !found { - assert.Failf(t, "", "expected an error containing 'request canceled (Client.Timeout exceeded'. Got %v", errs) + assert.Failf(t, "", "expected an error containing 'context deadline exceeded (Client.Timeout exceeded'. Got %v", errs) } assert.True(t, elapsed < 5*time.Second, "elapsed time: %s", elapsed.String()) diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 973035bd8ef..5d36c7f7b19 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.10 +FROM golang:1.14.4 COPY test/main.go main.go diff --git a/metricbeat/scripts/mage/docs_collector.go b/metricbeat/scripts/mage/docs_collector.go index 427975a7cf1..b58bad3edae 100644 --- a/metricbeat/scripts/mage/docs_collector.go +++ b/metricbeat/scripts/mage/docs_collector.go @@ -147,9 +147,6 @@ func getDefaultMetricsets() (map[string][]string, error) { } cmd := []string{"run"} - if mage.UseVendor { - cmd = append(cmd, "-mod", "vendor") - } for _, dir := range runpaths { rawMap, err := sh.OutCmd("go", append(cmd, dir)...)() if err != nil { diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 9a5e17eaa02..2711edfeeff 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.10 +FROM golang:1.14.4 RUN \ apt-get update \ diff --git a/vendor/modules.txt b/vendor/modules.txt deleted file mode 100644 index 7ee98f1384e..00000000000 --- a/vendor/modules.txt +++ /dev/null @@ -1,1457 +0,0 @@ -# 4d63.com/embedfiles v0.0.0-20190311033909-995e0740726f -4d63.com/embedfiles -# 4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b -4d63.com/tz -# cloud.google.com/go v0.51.0 -cloud.google.com/go -cloud.google.com/go/compute/metadata -cloud.google.com/go/functions/metadata -cloud.google.com/go/iam -cloud.google.com/go/internal -cloud.google.com/go/internal/fields -cloud.google.com/go/internal/optional -cloud.google.com/go/internal/trace -cloud.google.com/go/internal/version -cloud.google.com/go/monitoring/apiv3 -# cloud.google.com/go/datastore v1.0.0 -cloud.google.com/go/datastore -cloud.google.com/go/datastore/internal/gaepb -# cloud.google.com/go/pubsub v1.0.1 -cloud.google.com/go/pubsub -cloud.google.com/go/pubsub/apiv1 -cloud.google.com/go/pubsub/internal/distribution -# cloud.google.com/go/storage v1.0.0 -cloud.google.com/go/storage -# code.cloudfoundry.org/go-diodes v0.0.0-20190809170250-f77fb823c7ee -code.cloudfoundry.org/go-diodes -# code.cloudfoundry.org/go-loggregator v7.4.0+incompatible -code.cloudfoundry.org/go-loggregator -code.cloudfoundry.org/go-loggregator/conversion -code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2 -# code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f -code.cloudfoundry.org/gofileutils/fileutils -# code.cloudfoundry.org/rfc5424 v0.0.0-20180905210152-236a6d29298a -code.cloudfoundry.org/rfc5424 -# github.com/Azure/azure-amqp-common-go/v3 v3.0.0 -github.com/Azure/azure-amqp-common-go/v3 -github.com/Azure/azure-amqp-common-go/v3/aad -github.com/Azure/azure-amqp-common-go/v3/auth -github.com/Azure/azure-amqp-common-go/v3/cbs -github.com/Azure/azure-amqp-common-go/v3/conn -github.com/Azure/azure-amqp-common-go/v3/internal -github.com/Azure/azure-amqp-common-go/v3/internal/tracing -github.com/Azure/azure-amqp-common-go/v3/rpc -github.com/Azure/azure-amqp-common-go/v3/sas -github.com/Azure/azure-amqp-common-go/v3/uuid -# github.com/Azure/azure-event-hubs-go/v3 v3.1.2 -github.com/Azure/azure-event-hubs-go/v3 -github.com/Azure/azure-event-hubs-go/v3/atom -github.com/Azure/azure-event-hubs-go/v3/eph -github.com/Azure/azure-event-hubs-go/v3/persist -github.com/Azure/azure-event-hubs-go/v3/storage -# github.com/Azure/azure-pipeline-go v0.2.1 -github.com/Azure/azure-pipeline-go/pipeline -# github.com/Azure/azure-sdk-for-go v37.1.0+incompatible -github.com/Azure/azure-sdk-for-go/services/eventhub/mgmt/2017-04-01/eventhub -github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights -github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources -github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage -github.com/Azure/azure-sdk-for-go/version -# github.com/Azure/azure-storage-blob-go v0.8.0 -github.com/Azure/azure-storage-blob-go/azblob -# github.com/Azure/go-amqp v0.12.6 -github.com/Azure/go-amqp -github.com/Azure/go-amqp/internal/testconn -# github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 -github.com/Azure/go-ansiterm -github.com/Azure/go-ansiterm/winterm -# github.com/Azure/go-autorest/autorest v0.9.4 -github.com/Azure/go-autorest/autorest -github.com/Azure/go-autorest/autorest/azure -# github.com/Azure/go-autorest/autorest/adal v0.8.1 -github.com/Azure/go-autorest/autorest/adal -# github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 -github.com/Azure/go-autorest/autorest/azure/auth -# github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 -github.com/Azure/go-autorest/autorest/azure/cli -# github.com/Azure/go-autorest/autorest/date v0.2.0 -github.com/Azure/go-autorest/autorest/date -# github.com/Azure/go-autorest/autorest/to v0.3.0 -github.com/Azure/go-autorest/autorest/to -# github.com/Azure/go-autorest/autorest/validation v0.2.0 -github.com/Azure/go-autorest/autorest/validation -# github.com/Azure/go-autorest/logger v0.1.0 -github.com/Azure/go-autorest/logger -# github.com/Azure/go-autorest/tracing v0.5.0 -github.com/Azure/go-autorest/tracing -# github.com/BurntSushi/toml v0.3.1 -github.com/BurntSushi/toml -# github.com/Masterminds/semver v1.4.2 -github.com/Masterminds/semver -# github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 -github.com/Microsoft/go-winio -github.com/Microsoft/go-winio/pkg/guid -# github.com/Microsoft/hcsshim v0.8.7 -github.com/Microsoft/hcsshim/osversion -# github.com/Shopify/sarama v0.0.0-00010101000000-000000000000 => github.com/elastic/sarama v1.24.1-elastic.0.20200519143807-cbc80333a91e -github.com/Shopify/sarama -# github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 -github.com/StackExchange/wmi -# github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc -github.com/aerospike/aerospike-client-go -github.com/aerospike/aerospike-client-go/internal/lua -github.com/aerospike/aerospike-client-go/internal/lua/resources -github.com/aerospike/aerospike-client-go/logger -github.com/aerospike/aerospike-client-go/pkg/bcrypt -github.com/aerospike/aerospike-client-go/pkg/ripemd160 -github.com/aerospike/aerospike-client-go/types -github.com/aerospike/aerospike-client-go/types/atomic -github.com/aerospike/aerospike-client-go/types/particle_type -github.com/aerospike/aerospike-client-go/types/rand -github.com/aerospike/aerospike-client-go/utils/buffer -# github.com/akavel/rsrc v0.8.0 -github.com/akavel/rsrc/binutil -github.com/akavel/rsrc/coff -github.com/akavel/rsrc/ico -# github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 -github.com/andrewkroh/sys/windows/svc/eventlog -# github.com/antlr/antlr4 v0.0.0-20200225173536-225249fdaef5 -github.com/antlr/antlr4/runtime/Go/antlr -# github.com/armon/go-radix v1.0.0 -github.com/armon/go-radix -# github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 -github.com/armon/go-socks5 -# github.com/aws/aws-lambda-go v1.6.0 -github.com/aws/aws-lambda-go/events -github.com/aws/aws-lambda-go/lambda -github.com/aws/aws-lambda-go/lambda/messages -github.com/aws/aws-lambda-go/lambdacontext -# github.com/aws/aws-sdk-go-v2 v0.9.0 -github.com/aws/aws-sdk-go-v2/aws -github.com/aws/aws-sdk-go-v2/aws/arn -github.com/aws/aws-sdk-go-v2/aws/awserr -github.com/aws/aws-sdk-go-v2/aws/defaults -github.com/aws/aws-sdk-go-v2/aws/ec2metadata -github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds -github.com/aws/aws-sdk-go-v2/aws/endpointcreds -github.com/aws/aws-sdk-go-v2/aws/endpoints -github.com/aws/aws-sdk-go-v2/aws/external -github.com/aws/aws-sdk-go-v2/aws/signer/v4 -github.com/aws/aws-sdk-go-v2/aws/stscreds -github.com/aws/aws-sdk-go-v2/internal/awsutil -github.com/aws/aws-sdk-go-v2/internal/ini -github.com/aws/aws-sdk-go-v2/internal/sdk -github.com/aws/aws-sdk-go-v2/private/protocol -github.com/aws/aws-sdk-go-v2/private/protocol/ec2query -github.com/aws/aws-sdk-go-v2/private/protocol/json/jsonutil -github.com/aws/aws-sdk-go-v2/private/protocol/jsonrpc -github.com/aws/aws-sdk-go-v2/private/protocol/query -github.com/aws/aws-sdk-go-v2/private/protocol/query/queryutil -github.com/aws/aws-sdk-go-v2/private/protocol/rest -github.com/aws/aws-sdk-go-v2/private/protocol/restxml -github.com/aws/aws-sdk-go-v2/private/protocol/xml -github.com/aws/aws-sdk-go-v2/private/protocol/xml/xmlutil -github.com/aws/aws-sdk-go-v2/service/cloudformation -github.com/aws/aws-sdk-go-v2/service/cloudformation/cloudformationiface -github.com/aws/aws-sdk-go-v2/service/cloudwatch -github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface -github.com/aws/aws-sdk-go-v2/service/ec2 -github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/elasticloadbalancingv2iface -github.com/aws/aws-sdk-go-v2/service/iam -github.com/aws/aws-sdk-go-v2/service/rds -github.com/aws/aws-sdk-go-v2/service/rds/rdsiface -github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi -github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface -github.com/aws/aws-sdk-go-v2/service/s3 -github.com/aws/aws-sdk-go-v2/service/s3/s3iface -github.com/aws/aws-sdk-go-v2/service/sqs -github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface -github.com/aws/aws-sdk-go-v2/service/sts -# github.com/awslabs/goformation/v4 v4.1.0 -github.com/awslabs/goformation/v4/cloudformation -github.com/awslabs/goformation/v4/cloudformation/accessanalyzer -github.com/awslabs/goformation/v4/cloudformation/amazonmq -github.com/awslabs/goformation/v4/cloudformation/amplify -github.com/awslabs/goformation/v4/cloudformation/apigateway -github.com/awslabs/goformation/v4/cloudformation/apigatewayv2 -github.com/awslabs/goformation/v4/cloudformation/applicationautoscaling -github.com/awslabs/goformation/v4/cloudformation/appmesh -github.com/awslabs/goformation/v4/cloudformation/appstream -github.com/awslabs/goformation/v4/cloudformation/appsync -github.com/awslabs/goformation/v4/cloudformation/ask -github.com/awslabs/goformation/v4/cloudformation/athena -github.com/awslabs/goformation/v4/cloudformation/autoscaling -github.com/awslabs/goformation/v4/cloudformation/autoscalingplans -github.com/awslabs/goformation/v4/cloudformation/backup -github.com/awslabs/goformation/v4/cloudformation/batch -github.com/awslabs/goformation/v4/cloudformation/budgets -github.com/awslabs/goformation/v4/cloudformation/certificatemanager -github.com/awslabs/goformation/v4/cloudformation/cloud9 -github.com/awslabs/goformation/v4/cloudformation/cloudformation -github.com/awslabs/goformation/v4/cloudformation/cloudfront -github.com/awslabs/goformation/v4/cloudformation/cloudtrail -github.com/awslabs/goformation/v4/cloudformation/cloudwatch -github.com/awslabs/goformation/v4/cloudformation/codebuild -github.com/awslabs/goformation/v4/cloudformation/codecommit -github.com/awslabs/goformation/v4/cloudformation/codedeploy -github.com/awslabs/goformation/v4/cloudformation/codepipeline -github.com/awslabs/goformation/v4/cloudformation/codestar -github.com/awslabs/goformation/v4/cloudformation/codestarnotifications -github.com/awslabs/goformation/v4/cloudformation/cognito -github.com/awslabs/goformation/v4/cloudformation/config -github.com/awslabs/goformation/v4/cloudformation/datapipeline -github.com/awslabs/goformation/v4/cloudformation/dax -github.com/awslabs/goformation/v4/cloudformation/directoryservice -github.com/awslabs/goformation/v4/cloudformation/dlm -github.com/awslabs/goformation/v4/cloudformation/dms -github.com/awslabs/goformation/v4/cloudformation/docdb -github.com/awslabs/goformation/v4/cloudformation/dynamodb -github.com/awslabs/goformation/v4/cloudformation/ec2 -github.com/awslabs/goformation/v4/cloudformation/ecr -github.com/awslabs/goformation/v4/cloudformation/ecs -github.com/awslabs/goformation/v4/cloudformation/efs -github.com/awslabs/goformation/v4/cloudformation/eks -github.com/awslabs/goformation/v4/cloudformation/elasticache -github.com/awslabs/goformation/v4/cloudformation/elasticbeanstalk -github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancing -github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2 -github.com/awslabs/goformation/v4/cloudformation/elasticsearch -github.com/awslabs/goformation/v4/cloudformation/emr -github.com/awslabs/goformation/v4/cloudformation/events -github.com/awslabs/goformation/v4/cloudformation/eventschemas -github.com/awslabs/goformation/v4/cloudformation/fsx -github.com/awslabs/goformation/v4/cloudformation/gamelift -github.com/awslabs/goformation/v4/cloudformation/glue -github.com/awslabs/goformation/v4/cloudformation/greengrass -github.com/awslabs/goformation/v4/cloudformation/guardduty -github.com/awslabs/goformation/v4/cloudformation/iam -github.com/awslabs/goformation/v4/cloudformation/inspector -github.com/awslabs/goformation/v4/cloudformation/iot -github.com/awslabs/goformation/v4/cloudformation/iot1click -github.com/awslabs/goformation/v4/cloudformation/iotanalytics -github.com/awslabs/goformation/v4/cloudformation/iotevents -github.com/awslabs/goformation/v4/cloudformation/iotthingsgraph -github.com/awslabs/goformation/v4/cloudformation/kinesis -github.com/awslabs/goformation/v4/cloudformation/kinesisanalytics -github.com/awslabs/goformation/v4/cloudformation/kinesisanalyticsv2 -github.com/awslabs/goformation/v4/cloudformation/kinesisfirehose -github.com/awslabs/goformation/v4/cloudformation/kms -github.com/awslabs/goformation/v4/cloudformation/lakeformation -github.com/awslabs/goformation/v4/cloudformation/lambda -github.com/awslabs/goformation/v4/cloudformation/logs -github.com/awslabs/goformation/v4/cloudformation/managedblockchain -github.com/awslabs/goformation/v4/cloudformation/mediaconvert -github.com/awslabs/goformation/v4/cloudformation/medialive -github.com/awslabs/goformation/v4/cloudformation/mediastore -github.com/awslabs/goformation/v4/cloudformation/msk -github.com/awslabs/goformation/v4/cloudformation/neptune -github.com/awslabs/goformation/v4/cloudformation/opsworks -github.com/awslabs/goformation/v4/cloudformation/opsworkscm -github.com/awslabs/goformation/v4/cloudformation/pinpoint -github.com/awslabs/goformation/v4/cloudformation/pinpointemail -github.com/awslabs/goformation/v4/cloudformation/policies -github.com/awslabs/goformation/v4/cloudformation/qldb -github.com/awslabs/goformation/v4/cloudformation/ram -github.com/awslabs/goformation/v4/cloudformation/rds -github.com/awslabs/goformation/v4/cloudformation/redshift -github.com/awslabs/goformation/v4/cloudformation/robomaker -github.com/awslabs/goformation/v4/cloudformation/route53 -github.com/awslabs/goformation/v4/cloudformation/route53resolver -github.com/awslabs/goformation/v4/cloudformation/s3 -github.com/awslabs/goformation/v4/cloudformation/sagemaker -github.com/awslabs/goformation/v4/cloudformation/sdb -github.com/awslabs/goformation/v4/cloudformation/secretsmanager -github.com/awslabs/goformation/v4/cloudformation/securityhub -github.com/awslabs/goformation/v4/cloudformation/serverless -github.com/awslabs/goformation/v4/cloudformation/servicecatalog -github.com/awslabs/goformation/v4/cloudformation/servicediscovery -github.com/awslabs/goformation/v4/cloudformation/ses -github.com/awslabs/goformation/v4/cloudformation/sns -github.com/awslabs/goformation/v4/cloudformation/sqs -github.com/awslabs/goformation/v4/cloudformation/ssm -github.com/awslabs/goformation/v4/cloudformation/stepfunctions -github.com/awslabs/goformation/v4/cloudformation/tags -github.com/awslabs/goformation/v4/cloudformation/transfer -github.com/awslabs/goformation/v4/cloudformation/utils -github.com/awslabs/goformation/v4/cloudformation/waf -github.com/awslabs/goformation/v4/cloudformation/wafregional -github.com/awslabs/goformation/v4/cloudformation/wafv2 -github.com/awslabs/goformation/v4/cloudformation/workspaces -github.com/awslabs/goformation/v4/intrinsics -# github.com/beorn7/perks v1.0.1 -github.com/beorn7/perks/quantile -# github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 -github.com/blakesmith/ar -# github.com/bradleyfalzon/ghinstallation v1.1.0 -github.com/bradleyfalzon/ghinstallation -# github.com/bsm/sarama-cluster v2.1.14-0.20180625083203-7e67d87a6b3f+incompatible -github.com/bsm/sarama-cluster -# github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e -github.com/cavaliercoder/go-rpm -github.com/cavaliercoder/go-rpm/version -# github.com/cespare/xxhash/v2 v2.1.1 -github.com/cespare/xxhash/v2 -# github.com/cloudfoundry-community/go-cfclient v0.0.0-20190808214049-35bcce23fc5f -github.com/cloudfoundry-community/go-cfclient -# github.com/cloudfoundry/noaa v2.1.0+incompatible -github.com/cloudfoundry/noaa -github.com/cloudfoundry/noaa/consumer -github.com/cloudfoundry/noaa/consumer/internal -github.com/cloudfoundry/noaa/errors -# github.com/cloudfoundry/sonde-go v0.0.0-20171206171820-b33733203bb4 -github.com/cloudfoundry/sonde-go/events -# github.com/containerd/containerd v1.3.3 -github.com/containerd/containerd/errdefs -# github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41 -github.com/containerd/continuity/fs -github.com/containerd/continuity/pathdriver -github.com/containerd/continuity/syscallx -github.com/containerd/continuity/sysx -# github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c -github.com/containerd/fifo -# github.com/coreos/bbolt v1.3.1-coreos.6.0.20180318001526-af9db2027c98 -github.com/coreos/bbolt -# github.com/coreos/go-systemd/v22 v22.0.0 -github.com/coreos/go-systemd/v22/activation -github.com/coreos/go-systemd/v22/dbus -github.com/coreos/go-systemd/v22/internal/dlopen -github.com/coreos/go-systemd/v22/sdjournal -# github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea -github.com/coreos/pkg/dlopen -# github.com/davecgh/go-spew v1.1.1 -github.com/davecgh/go-spew/spew -# github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 -github.com/davecgh/go-xdr/xdr2 -# github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e -github.com/denisenkom/go-mssqldb -github.com/denisenkom/go-mssqldb/internal/cp -github.com/denisenkom/go-mssqldb/internal/decimal -github.com/denisenkom/go-mssqldb/internal/querytext -# github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2 -github.com/devigned/tab -# github.com/dgrijalva/jwt-go v3.2.1-0.20190620180102-5e25c22bd5d6+incompatible -github.com/dgrijalva/jwt-go -# github.com/digitalocean/go-libvirt v0.0.0-20180301200012-6075ea3c39a1 -github.com/digitalocean/go-libvirt -github.com/digitalocean/go-libvirt/internal/constants -github.com/digitalocean/go-libvirt/libvirttest -# github.com/dimchansky/utfbom v1.1.0 -github.com/dimchansky/utfbom -# github.com/dlclark/regexp2 v1.1.7-0.20171009020623-7632a260cbaf -github.com/dlclark/regexp2 -github.com/dlclark/regexp2/syntax -# github.com/docker/distribution v2.7.1+incompatible -github.com/docker/distribution/digestset -github.com/docker/distribution/reference -github.com/docker/distribution/registry/api/errcode -# github.com/docker/docker v1.4.2-0.20170802015333-8af4db6f002a => github.com/docker/engine v0.0.0-20191113042239-ea84732a7725 -github.com/docker/docker/api -github.com/docker/docker/api/types -github.com/docker/docker/api/types/backend -github.com/docker/docker/api/types/blkiodev -github.com/docker/docker/api/types/container -github.com/docker/docker/api/types/events -github.com/docker/docker/api/types/filters -github.com/docker/docker/api/types/image -github.com/docker/docker/api/types/mount -github.com/docker/docker/api/types/network -github.com/docker/docker/api/types/plugins/logdriver -github.com/docker/docker/api/types/registry -github.com/docker/docker/api/types/strslice -github.com/docker/docker/api/types/swarm -github.com/docker/docker/api/types/swarm/runtime -github.com/docker/docker/api/types/time -github.com/docker/docker/api/types/versions -github.com/docker/docker/api/types/volume -github.com/docker/docker/client -github.com/docker/docker/daemon/logger -github.com/docker/docker/errdefs -github.com/docker/docker/pkg/archive -github.com/docker/docker/pkg/fileutils -github.com/docker/docker/pkg/idtools -github.com/docker/docker/pkg/ioutils -github.com/docker/docker/pkg/jsonmessage -github.com/docker/docker/pkg/longpath -github.com/docker/docker/pkg/mount -github.com/docker/docker/pkg/plugingetter -github.com/docker/docker/pkg/plugins -github.com/docker/docker/pkg/plugins/transport -github.com/docker/docker/pkg/pools -github.com/docker/docker/pkg/progress -github.com/docker/docker/pkg/streamformatter -github.com/docker/docker/pkg/stringid -github.com/docker/docker/pkg/system -github.com/docker/docker/pkg/term -github.com/docker/docker/pkg/term/windows -# github.com/docker/go-connections v0.4.0 -github.com/docker/go-connections/nat -github.com/docker/go-connections/sockets -github.com/docker/go-connections/tlsconfig -# github.com/docker/go-metrics v0.0.1 -github.com/docker/go-metrics -# github.com/docker/go-plugins-helpers v0.0.0-20181025120712-1e6269c305b8 => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f -github.com/docker/go-plugins-helpers/sdk -# github.com/docker/go-units v0.4.0 -github.com/docker/go-units -# github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 -github.com/docker/spdystream -github.com/docker/spdystream/spdy -# github.com/dop251/goja v0.0.0-00010101000000-000000000000 => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 -github.com/dop251/goja -github.com/dop251/goja/ast -github.com/dop251/goja/file -github.com/dop251/goja/parser -github.com/dop251/goja/token -# github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 -github.com/dop251/goja_nodejs/require -github.com/dop251/goja_nodejs/util -# github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 -github.com/dustin/go-humanize -# github.com/eapache/go-resiliency v1.2.0 -github.com/eapache/go-resiliency/breaker -# github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 -github.com/eapache/go-xerial-snappy -# github.com/eapache/queue v1.1.0 -github.com/eapache/queue -# github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 -github.com/eclipse/paho.mqtt.golang -github.com/eclipse/paho.mqtt.golang/packets -# github.com/elastic/ecs v1.5.0 -github.com/elastic/ecs/code/go/ecs -# github.com/elastic/elastic-agent-client/v7 v7.0.0-20200601155656-d6a9eb4f6d07 -github.com/elastic/elastic-agent-client/v7/pkg/client -github.com/elastic/elastic-agent-client/v7/pkg/proto -github.com/elastic/elastic-agent-client/v7/pkg/utils -# github.com/elastic/go-concert v0.0.2 -github.com/elastic/go-concert -github.com/elastic/go-concert/atomic -github.com/elastic/go-concert/chorus -github.com/elastic/go-concert/unison -# github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8 -github.com/elastic/go-libaudit/v2 -github.com/elastic/go-libaudit/v2/aucoalesce -github.com/elastic/go-libaudit/v2/auparse -github.com/elastic/go-libaudit/v2/rule -github.com/elastic/go-libaudit/v2/rule/flags -github.com/elastic/go-libaudit/v2/sys -# github.com/elastic/go-licenser v0.2.1 -github.com/elastic/go-licenser -github.com/elastic/go-licenser/licensing -# github.com/elastic/go-lookslike v0.3.0 -github.com/elastic/go-lookslike -github.com/elastic/go-lookslike/internal/llreflect -github.com/elastic/go-lookslike/isdef -github.com/elastic/go-lookslike/llpath -github.com/elastic/go-lookslike/llresult -github.com/elastic/go-lookslike/testslike -github.com/elastic/go-lookslike/validator -# github.com/elastic/go-lumber v0.1.0 -github.com/elastic/go-lumber/client/v2 -github.com/elastic/go-lumber/lj -github.com/elastic/go-lumber/log -github.com/elastic/go-lumber/protocol/v2 -github.com/elastic/go-lumber/server/internal -github.com/elastic/go-lumber/server/v2 -# github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 -github.com/elastic/go-perf -# github.com/elastic/go-seccomp-bpf v1.1.0 -github.com/elastic/go-seccomp-bpf -github.com/elastic/go-seccomp-bpf/arch -# github.com/elastic/go-structform v0.0.7 -github.com/elastic/go-structform -github.com/elastic/go-structform/cborl -github.com/elastic/go-structform/gotype -github.com/elastic/go-structform/internal/unsafe -github.com/elastic/go-structform/json -github.com/elastic/go-structform/ubjson -github.com/elastic/go-structform/visitors -# github.com/elastic/go-sysinfo v1.3.0 -github.com/elastic/go-sysinfo -github.com/elastic/go-sysinfo/internal/registry -github.com/elastic/go-sysinfo/providers/darwin -github.com/elastic/go-sysinfo/providers/linux -github.com/elastic/go-sysinfo/providers/shared -github.com/elastic/go-sysinfo/providers/windows -github.com/elastic/go-sysinfo/types -# github.com/elastic/go-txfile v0.0.7 -github.com/elastic/go-txfile -github.com/elastic/go-txfile/dev-tools/lib/mage/xbuild -github.com/elastic/go-txfile/internal/cleanup -github.com/elastic/go-txfile/internal/invariant -github.com/elastic/go-txfile/internal/iter -github.com/elastic/go-txfile/internal/strbld -github.com/elastic/go-txfile/internal/tracelog -github.com/elastic/go-txfile/internal/vfs -github.com/elastic/go-txfile/internal/vfs/osfs -github.com/elastic/go-txfile/internal/vfs/osfs/osfstest -github.com/elastic/go-txfile/pq -github.com/elastic/go-txfile/txerr -github.com/elastic/go-txfile/txfiletest -# github.com/elastic/go-ucfg v0.8.3 -github.com/elastic/go-ucfg -github.com/elastic/go-ucfg/cfgutil -github.com/elastic/go-ucfg/flag -github.com/elastic/go-ucfg/json -github.com/elastic/go-ucfg/parse -github.com/elastic/go-ucfg/yaml -# github.com/elastic/go-windows v1.0.1 -github.com/elastic/go-windows -# github.com/elastic/gosigar v0.10.5 -github.com/elastic/gosigar -github.com/elastic/gosigar/cgroup -github.com/elastic/gosigar/sys -github.com/elastic/gosigar/sys/linux -github.com/elastic/gosigar/sys/windows -# github.com/evanphx/json-patch v4.2.0+incompatible -github.com/evanphx/json-patch -# github.com/fatih/color v1.5.0 -github.com/fatih/color -# github.com/fsnotify/fsevents v0.0.0-00010101000000-000000000000 => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 -github.com/fsnotify/fsevents -# github.com/fsnotify/fsnotify v1.4.7 => github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d -github.com/fsnotify/fsnotify -# github.com/garyburd/redigo v1.0.1-0.20160525165706-b8dc90050f24 -github.com/garyburd/redigo/internal -github.com/garyburd/redigo/redis -# github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647 -github.com/go-ole/go-ole -github.com/go-ole/go-ole/oleutil -# github.com/go-sourcemap/sourcemap v2.1.2+incompatible -github.com/go-sourcemap/sourcemap -github.com/go-sourcemap/sourcemap/internal/base64vlq -# github.com/go-sql-driver/mysql v1.4.1 -github.com/go-sql-driver/mysql -# github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be -github.com/gocarina/gocsv -# github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e -github.com/godbus/dbus -# github.com/godbus/dbus/v5 v5.0.3 -github.com/godbus/dbus/v5 -# github.com/godror/godror v0.10.4 -github.com/godror/godror -# github.com/gofrs/flock v0.7.2-0.20190320160742-5135e617513b -github.com/gofrs/flock -# github.com/gofrs/uuid v3.3.0+incompatible -github.com/gofrs/uuid -# github.com/gogo/protobuf v1.3.1 -github.com/gogo/protobuf/gogoproto -github.com/gogo/protobuf/proto -github.com/gogo/protobuf/protoc-gen-gogo/descriptor -github.com/gogo/protobuf/sortkeys -github.com/gogo/protobuf/types -# github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe -github.com/golang-sql/civil -# github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 -github.com/golang/groupcache/lru -# github.com/golang/protobuf v1.4.2 -github.com/golang/protobuf/descriptor -github.com/golang/protobuf/internal/gengogrpc -github.com/golang/protobuf/jsonpb -github.com/golang/protobuf/proto -github.com/golang/protobuf/protoc-gen-go -github.com/golang/protobuf/protoc-gen-go/descriptor -github.com/golang/protobuf/ptypes -github.com/golang/protobuf/ptypes/any -github.com/golang/protobuf/ptypes/duration -github.com/golang/protobuf/ptypes/empty -github.com/golang/protobuf/ptypes/struct -github.com/golang/protobuf/ptypes/timestamp -github.com/golang/protobuf/ptypes/wrappers -# github.com/golang/snappy v0.0.1 -github.com/golang/snappy -# github.com/google/flatbuffers v1.7.2-0.20170925184458-7a6b2bf521e9 -github.com/google/flatbuffers/go -# github.com/google/go-cmp v0.4.0 -github.com/google/go-cmp/cmp -github.com/google/go-cmp/cmp/internal/diff -github.com/google/go-cmp/cmp/internal/flags -github.com/google/go-cmp/cmp/internal/function -github.com/google/go-cmp/cmp/internal/value -# github.com/google/go-github/v28 v28.1.1 -github.com/google/go-github/v28/github -# github.com/google/go-github/v29 v29.0.2 -github.com/google/go-github/v29/github -# github.com/google/go-querystring v1.0.0 -github.com/google/go-querystring/query -# github.com/google/gofuzz v1.1.0 -github.com/google/gofuzz -# github.com/google/gopacket v1.1.18-0.20191009163724-0ad7f2610e34 => github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41 -github.com/google/gopacket -github.com/google/gopacket/afpacket -github.com/google/gopacket/layers -# github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f -github.com/google/uuid -# github.com/googleapis/gax-go/v2 v2.0.5 -github.com/googleapis/gax-go/v2 -# github.com/googleapis/gnostic v0.3.1-0.20190624222214-25d8b0b66985 -github.com/googleapis/gnostic/OpenAPIv2 -github.com/googleapis/gnostic/compiler -github.com/googleapis/gnostic/extensions -# github.com/gorhill/cronexpr v0.0.0-20161205141322-d520615e531a -github.com/gorhill/cronexpr -# github.com/gorilla/websocket v1.4.1 -github.com/gorilla/websocket -# github.com/grpc-ecosystem/grpc-gateway v1.13.0 -github.com/grpc-ecosystem/grpc-gateway/internal -github.com/grpc-ecosystem/grpc-gateway/runtime -github.com/grpc-ecosystem/grpc-gateway/utilities -# github.com/h2non/filetype v1.0.12 -github.com/h2non/filetype -github.com/h2non/filetype/matchers -github.com/h2non/filetype/matchers/isobmff -github.com/h2non/filetype/types -# github.com/hashicorp/errwrap v1.0.0 -github.com/hashicorp/errwrap -# github.com/hashicorp/go-cleanhttp v0.5.1 -github.com/hashicorp/go-cleanhttp -# github.com/hashicorp/go-multierror v1.1.0 -github.com/hashicorp/go-multierror -# github.com/hashicorp/go-retryablehttp v0.6.6 -github.com/hashicorp/go-retryablehttp -# github.com/hashicorp/go-uuid v1.0.2 -github.com/hashicorp/go-uuid -# github.com/hashicorp/go-version v1.0.0 -github.com/hashicorp/go-version -# github.com/hashicorp/golang-lru v0.5.2-0.20190520140433-59383c442f7d -github.com/hashicorp/golang-lru -github.com/hashicorp/golang-lru/simplelru -# github.com/haya14busa/go-actions-toolkit v0.0.0-20200105081403-ca0307860f01 -github.com/haya14busa/go-actions-toolkit/core -# github.com/imdario/mergo v0.3.6 -github.com/imdario/mergo -# github.com/inconshreveable/mousetrap v1.0.0 -github.com/inconshreveable/mousetrap -# github.com/insomniacslk/dhcp v0.0.0-20180716145214-633285ba52b2 => github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 -github.com/insomniacslk/dhcp/dhcpv4 -github.com/insomniacslk/dhcp/iana -# github.com/jcmturner/gofork v1.0.0 -github.com/jcmturner/gofork/encoding/asn1 -github.com/jcmturner/gofork/x/crypto/pbkdf2 -# github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af -github.com/jmespath/go-jmespath -# github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 -github.com/jmoiron/sqlx -github.com/jmoiron/sqlx/reflectx -# github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 -github.com/joeshaw/multierror -# github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd -github.com/josephspurrier/goversioninfo -# github.com/jpillora/backoff v1.0.0 -github.com/jpillora/backoff -# github.com/json-iterator/go v1.1.8 -github.com/json-iterator/go -# github.com/jstemmer/go-junit-report v0.9.1 -github.com/jstemmer/go-junit-report -github.com/jstemmer/go-junit-report/formatter -github.com/jstemmer/go-junit-report/parser -# github.com/klauspost/compress v1.9.8 -github.com/klauspost/compress/flate -github.com/klauspost/compress/fse -github.com/klauspost/compress/huff0 -github.com/klauspost/compress/snappy -github.com/klauspost/compress/zlib -github.com/klauspost/compress/zstd -github.com/klauspost/compress/zstd/internal/xxhash -# github.com/konsorten/go-windows-terminal-sequences v1.0.2 -github.com/konsorten/go-windows-terminal-sequences -# github.com/lib/pq v1.1.2-0.20190507191818-2ff3cb3adc01 -github.com/lib/pq -github.com/lib/pq/oid -github.com/lib/pq/scram -# github.com/magefile/mage v1.9.0 -github.com/magefile/mage -github.com/magefile/mage/internal -github.com/magefile/mage/mage -github.com/magefile/mage/mg -github.com/magefile/mage/parse -github.com/magefile/mage/sh -github.com/magefile/mage/target -# github.com/mailru/easyjson v0.7.1 -github.com/mailru/easyjson -github.com/mailru/easyjson/buffer -github.com/mailru/easyjson/jlexer -github.com/mailru/easyjson/jwriter -# github.com/mattn/go-colorable v0.0.8 -github.com/mattn/go-colorable -# github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe -github.com/mattn/go-ieproxy -# github.com/mattn/go-isatty v0.0.2 -github.com/mattn/go-isatty -# github.com/mattn/go-shellwords v1.0.7 -github.com/mattn/go-shellwords -# github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 -github.com/matttproud/golang_protobuf_extensions/pbutil -# github.com/miekg/dns v1.1.15 -github.com/miekg/dns -# github.com/mitchellh/go-homedir v1.1.0 -github.com/mitchellh/go-homedir -# github.com/mitchellh/gox v1.0.1 -github.com/mitchellh/gox -# github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51 -github.com/mitchellh/hashstructure -# github.com/mitchellh/iochan v1.0.0 -github.com/mitchellh/iochan -# github.com/mitchellh/mapstructure v1.1.2 -github.com/mitchellh/mapstructure -# github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd -github.com/modern-go/concurrent -# github.com/modern-go/reflect2 v1.0.1 -github.com/modern-go/reflect2 -# github.com/morikuni/aec v1.0.0 -github.com/morikuni/aec -# github.com/oklog/ulid v1.3.1 -github.com/oklog/ulid -# github.com/opencontainers/go-digest v1.0.0-rc1.0.20190228220655-ac19fd6e7483 -github.com/opencontainers/go-digest -# github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 -github.com/opencontainers/image-spec/specs-go -github.com/opencontainers/image-spec/specs-go/v1 -# github.com/opencontainers/runc v1.0.0-rc9 -github.com/opencontainers/runc/libcontainer/system -github.com/opencontainers/runc/libcontainer/user -# github.com/pierrec/lz4 v2.4.1+incompatible -github.com/pierrec/lz4 -github.com/pierrec/lz4/internal/xxh32 -# github.com/pierrre/gotestcover v0.0.0-20160113212533-7b94f124d338 -github.com/pierrre/gotestcover -# github.com/pkg/errors v0.9.1 -github.com/pkg/errors -# github.com/pmezard/go-difflib v1.0.0 -github.com/pmezard/go-difflib/difflib -# github.com/prometheus/client_golang v1.1.1-0.20190913103102-20428fa0bffc -github.com/prometheus/client_golang/prometheus -github.com/prometheus/client_golang/prometheus/internal -github.com/prometheus/client_golang/prometheus/promhttp -# github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 -github.com/prometheus/client_model/go -# github.com/prometheus/common v0.7.0 -github.com/prometheus/common/expfmt -github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg -github.com/prometheus/common/model -# github.com/prometheus/procfs v0.0.11 -github.com/prometheus/procfs -github.com/prometheus/procfs/internal/fs -github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v2.5.0+incompatible -github.com/prometheus/prometheus/prompb -# github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 -github.com/rcrowley/go-metrics -# github.com/reviewdog/errorformat v0.0.0-20200109134752-8983be9bc7dd -github.com/reviewdog/errorformat -github.com/reviewdog/errorformat/fmts -# github.com/reviewdog/reviewdog v0.9.17 -github.com/reviewdog/reviewdog -github.com/reviewdog/reviewdog/cienv -github.com/reviewdog/reviewdog/cmd/reviewdog -github.com/reviewdog/reviewdog/commands -github.com/reviewdog/reviewdog/diff -github.com/reviewdog/reviewdog/doghouse -github.com/reviewdog/reviewdog/doghouse/client -github.com/reviewdog/reviewdog/doghouse/server -github.com/reviewdog/reviewdog/doghouse/server/storage -github.com/reviewdog/reviewdog/project -github.com/reviewdog/reviewdog/service/github -github.com/reviewdog/reviewdog/service/github/githubutils -github.com/reviewdog/reviewdog/service/gitlab -github.com/reviewdog/reviewdog/service/serviceutil -# github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e -github.com/samuel/go-parser/parser -# github.com/samuel/go-thrift v0.0.0-20140522043831-2187045faa54 -github.com/samuel/go-thrift/parser -# github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b -github.com/sanathkr/go-yaml -# github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522 -github.com/sanathkr/yaml -# github.com/santhosh-tekuri/jsonschema v1.2.4 -github.com/santhosh-tekuri/jsonschema -github.com/santhosh-tekuri/jsonschema/decoders -github.com/santhosh-tekuri/jsonschema/formats -github.com/santhosh-tekuri/jsonschema/loader -github.com/santhosh-tekuri/jsonschema/mediatypes -# github.com/shirou/gopsutil v2.19.11+incompatible -github.com/shirou/gopsutil/disk -github.com/shirou/gopsutil/internal/common -github.com/shirou/gopsutil/net -# github.com/sirupsen/logrus v1.4.2 -github.com/sirupsen/logrus -# github.com/spf13/cobra v0.0.3 -github.com/spf13/cobra -# github.com/spf13/pflag v1.0.5 -github.com/spf13/pflag -# github.com/stretchr/objx v0.2.0 -github.com/stretchr/objx -# github.com/stretchr/testify v1.5.1 -github.com/stretchr/testify/assert -github.com/stretchr/testify/mock -github.com/stretchr/testify/require -github.com/stretchr/testify/suite -# github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b -github.com/tsg/go-daemon -# github.com/tsg/gopacket v0.0.0-20190320122513-dd3d0e41124a -github.com/tsg/gopacket -github.com/tsg/gopacket/afpacket -github.com/tsg/gopacket/layers -github.com/tsg/gopacket/pcap -# github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 -github.com/urso/diag -github.com/urso/diag/ctxfmt -# github.com/urso/go-bin v0.0.0-20180220135811-781c575c9f0e -github.com/urso/go-bin -# github.com/urso/magetools v0.0.0-20200125210132-c2e338f92f3a -github.com/urso/magetools/clitool -github.com/urso/magetools/ctrl -github.com/urso/magetools/fs -github.com/urso/magetools/gotool -github.com/urso/magetools/mgenv -# github.com/urso/sderr v0.0.0-20200210124243-c2a16f3d43ec -github.com/urso/sderr -# github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 -github.com/vmware/govmomi -github.com/vmware/govmomi/find -github.com/vmware/govmomi/list -github.com/vmware/govmomi/nfc -github.com/vmware/govmomi/object -github.com/vmware/govmomi/property -github.com/vmware/govmomi/session -github.com/vmware/govmomi/simulator -github.com/vmware/govmomi/simulator/esx -github.com/vmware/govmomi/simulator/vpx -github.com/vmware/govmomi/task -github.com/vmware/govmomi/view -github.com/vmware/govmomi/vim25 -github.com/vmware/govmomi/vim25/debug -github.com/vmware/govmomi/vim25/methods -github.com/vmware/govmomi/vim25/mo -github.com/vmware/govmomi/vim25/progress -github.com/vmware/govmomi/vim25/soap -github.com/vmware/govmomi/vim25/types -github.com/vmware/govmomi/vim25/xml -# github.com/xanzy/go-gitlab v0.22.3 -github.com/xanzy/go-gitlab -# github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c -github.com/xdg/scram -# github.com/xdg/stringprep v1.0.0 -github.com/xdg/stringprep -# github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 -github.com/yuin/gopher-lua -github.com/yuin/gopher-lua/ast -github.com/yuin/gopher-lua/parse -github.com/yuin/gopher-lua/pm -# go.elastic.co/apm v1.7.2 -go.elastic.co/apm -go.elastic.co/apm/apmconfig -go.elastic.co/apm/apmtest -go.elastic.co/apm/internal/apmcontext -go.elastic.co/apm/internal/apmhostutil -go.elastic.co/apm/internal/apmhttputil -go.elastic.co/apm/internal/apmlog -go.elastic.co/apm/internal/apmschema -go.elastic.co/apm/internal/apmstrings -go.elastic.co/apm/internal/apmversion -go.elastic.co/apm/internal/configutil -go.elastic.co/apm/internal/iochan -go.elastic.co/apm/internal/pkgerrorsutil -go.elastic.co/apm/internal/ringbuffer -go.elastic.co/apm/internal/wildcard -go.elastic.co/apm/model -go.elastic.co/apm/stacktrace -go.elastic.co/apm/transport -go.elastic.co/apm/transport/transporttest -# go.elastic.co/apm/module/apmelasticsearch v1.7.2 -go.elastic.co/apm/module/apmelasticsearch -# go.elastic.co/apm/module/apmhttp v1.7.2 -go.elastic.co/apm/module/apmhttp -# go.elastic.co/ecszap v0.2.0 -go.elastic.co/ecszap -go.elastic.co/ecszap/internal -# go.elastic.co/fastjson v1.0.0 -go.elastic.co/fastjson -# go.opencensus.io v0.22.2 -go.opencensus.io -go.opencensus.io/internal -go.opencensus.io/internal/tagencoding -go.opencensus.io/metric/metricdata -go.opencensus.io/metric/metricproducer -go.opencensus.io/plugin/ocgrpc -go.opencensus.io/plugin/ochttp -go.opencensus.io/plugin/ochttp/propagation/b3 -go.opencensus.io/resource -go.opencensus.io/stats -go.opencensus.io/stats/internal -go.opencensus.io/stats/view -go.opencensus.io/tag -go.opencensus.io/trace -go.opencensus.io/trace/internal -go.opencensus.io/trace/propagation -go.opencensus.io/trace/tracestate -# go.uber.org/atomic v1.5.0 -go.uber.org/atomic -# go.uber.org/multierr v1.3.0 -go.uber.org/multierr -# go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee -go.uber.org/tools/update-license -# go.uber.org/zap v1.14.0 -go.uber.org/zap -go.uber.org/zap/buffer -go.uber.org/zap/internal/bufferpool -go.uber.org/zap/internal/color -go.uber.org/zap/internal/exit -go.uber.org/zap/zapcore -go.uber.org/zap/zaptest/observer -# golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 -golang.org/x/crypto/blake2b -golang.org/x/crypto/blowfish -golang.org/x/crypto/cast5 -golang.org/x/crypto/chacha20 -golang.org/x/crypto/curve25519 -golang.org/x/crypto/ed25519 -golang.org/x/crypto/ed25519/internal/edwards25519 -golang.org/x/crypto/internal/subtle -golang.org/x/crypto/md4 -golang.org/x/crypto/openpgp -golang.org/x/crypto/openpgp/armor -golang.org/x/crypto/openpgp/elgamal -golang.org/x/crypto/openpgp/errors -golang.org/x/crypto/openpgp/packet -golang.org/x/crypto/openpgp/s2k -golang.org/x/crypto/pbkdf2 -golang.org/x/crypto/pkcs12 -golang.org/x/crypto/pkcs12/internal/rc2 -golang.org/x/crypto/poly1305 -golang.org/x/crypto/sha3 -golang.org/x/crypto/ssh -golang.org/x/crypto/ssh/internal/bcrypt_pbkdf -golang.org/x/crypto/ssh/terminal -# golang.org/x/exp v0.0.0-20191227195350-da58074b4299 -golang.org/x/exp/apidiff -golang.org/x/exp/cmd/apidiff -# golang.org/x/lint v0.0.0-20200130185559-910be7a94367 -golang.org/x/lint -golang.org/x/lint/golint -# golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee -golang.org/x/mod/module -golang.org/x/mod/semver -# golang.org/x/net v0.0.0-20200202094626-16171245cfb2 -golang.org/x/net/bpf -golang.org/x/net/context -golang.org/x/net/context/ctxhttp -golang.org/x/net/http/httpguts -golang.org/x/net/http/httpproxy -golang.org/x/net/http2 -golang.org/x/net/http2/hpack -golang.org/x/net/icmp -golang.org/x/net/idna -golang.org/x/net/internal/iana -golang.org/x/net/internal/socket -golang.org/x/net/internal/socks -golang.org/x/net/internal/timeseries -golang.org/x/net/ipv4 -golang.org/x/net/ipv6 -golang.org/x/net/netutil -golang.org/x/net/proxy -golang.org/x/net/publicsuffix -golang.org/x/net/trace -golang.org/x/net/websocket -# golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d -golang.org/x/oauth2 -golang.org/x/oauth2/clientcredentials -golang.org/x/oauth2/endpoints -golang.org/x/oauth2/google -golang.org/x/oauth2/internal -golang.org/x/oauth2/jws -golang.org/x/oauth2/jwt -# golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e -golang.org/x/sync/errgroup -golang.org/x/sync/semaphore -# golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e -golang.org/x/sys/cpu -golang.org/x/sys/unix -golang.org/x/sys/windows -golang.org/x/sys/windows/registry -golang.org/x/sys/windows/svc -golang.org/x/sys/windows/svc/debug -golang.org/x/sys/windows/svc/eventlog -# golang.org/x/text v0.3.2 -golang.org/x/text/cases -golang.org/x/text/collate -golang.org/x/text/encoding -golang.org/x/text/encoding/charmap -golang.org/x/text/encoding/htmlindex -golang.org/x/text/encoding/internal -golang.org/x/text/encoding/internal/identifier -golang.org/x/text/encoding/japanese -golang.org/x/text/encoding/korean -golang.org/x/text/encoding/simplifiedchinese -golang.org/x/text/encoding/traditionalchinese -golang.org/x/text/encoding/unicode -golang.org/x/text/internal -golang.org/x/text/internal/colltab -golang.org/x/text/internal/language -golang.org/x/text/internal/language/compact -golang.org/x/text/internal/tag -golang.org/x/text/internal/utf8internal -golang.org/x/text/language -golang.org/x/text/runes -golang.org/x/text/secure/bidirule -golang.org/x/text/transform -golang.org/x/text/unicode/bidi -golang.org/x/text/unicode/norm -# golang.org/x/time v0.0.0-20191024005414-555d28b269f0 -golang.org/x/time/rate -# golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2 -golang.org/x/tools/cmd/goimports -golang.org/x/tools/cmd/stringer -golang.org/x/tools/go/analysis -golang.org/x/tools/go/analysis/passes/inspect -golang.org/x/tools/go/ast/astutil -golang.org/x/tools/go/ast/inspector -golang.org/x/tools/go/buildutil -golang.org/x/tools/go/gcexportdata -golang.org/x/tools/go/internal/gcimporter -golang.org/x/tools/go/internal/packagesdriver -golang.org/x/tools/go/packages -golang.org/x/tools/go/types/objectpath -golang.org/x/tools/go/types/typeutil -golang.org/x/tools/go/vcs -golang.org/x/tools/internal/fastwalk -golang.org/x/tools/internal/gopathwalk -golang.org/x/tools/internal/imports -golang.org/x/tools/internal/packagesinternal -# golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 -golang.org/x/xerrors -golang.org/x/xerrors/internal -# google.golang.org/api v0.15.0 -google.golang.org/api/cloudfunctions/v1 -google.golang.org/api/compute/v1 -google.golang.org/api/googleapi -google.golang.org/api/googleapi/transport -google.golang.org/api/internal -google.golang.org/api/internal/gensupport -google.golang.org/api/internal/third_party/uritemplates -google.golang.org/api/iterator -google.golang.org/api/option -google.golang.org/api/storage/v1 -google.golang.org/api/support/bundler -google.golang.org/api/transport -google.golang.org/api/transport/grpc -google.golang.org/api/transport/http -google.golang.org/api/transport/http/internal/propagation -# google.golang.org/appengine v1.6.5 -google.golang.org/appengine -google.golang.org/appengine/cloudsql -google.golang.org/appengine/internal -google.golang.org/appengine/internal/app_identity -google.golang.org/appengine/internal/base -google.golang.org/appengine/internal/datastore -google.golang.org/appengine/internal/log -google.golang.org/appengine/internal/modules -google.golang.org/appengine/internal/remote_api -google.golang.org/appengine/internal/socket -google.golang.org/appengine/internal/urlfetch -google.golang.org/appengine/socket -google.golang.org/appengine/urlfetch -# google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb -google.golang.org/genproto/googleapis/api -google.golang.org/genproto/googleapis/api/annotations -google.golang.org/genproto/googleapis/api/distribution -google.golang.org/genproto/googleapis/api/httpbody -google.golang.org/genproto/googleapis/api/label -google.golang.org/genproto/googleapis/api/metric -google.golang.org/genproto/googleapis/api/monitoredres -google.golang.org/genproto/googleapis/datastore/v1 -google.golang.org/genproto/googleapis/iam/v1 -google.golang.org/genproto/googleapis/monitoring/v3 -google.golang.org/genproto/googleapis/pubsub/v1 -google.golang.org/genproto/googleapis/rpc/code -google.golang.org/genproto/googleapis/rpc/status -google.golang.org/genproto/googleapis/type/calendarperiod -google.golang.org/genproto/googleapis/type/expr -google.golang.org/genproto/googleapis/type/latlng -google.golang.org/genproto/protobuf/field_mask -# google.golang.org/grpc v1.29.1 -google.golang.org/grpc -google.golang.org/grpc/attributes -google.golang.org/grpc/backoff -google.golang.org/grpc/balancer -google.golang.org/grpc/balancer/base -google.golang.org/grpc/balancer/grpclb -google.golang.org/grpc/balancer/grpclb/grpc_lb_v1 -google.golang.org/grpc/balancer/roundrobin -google.golang.org/grpc/binarylog/grpc_binarylog_v1 -google.golang.org/grpc/codes -google.golang.org/grpc/connectivity -google.golang.org/grpc/credentials -google.golang.org/grpc/credentials/alts -google.golang.org/grpc/credentials/alts/internal -google.golang.org/grpc/credentials/alts/internal/authinfo -google.golang.org/grpc/credentials/alts/internal/conn -google.golang.org/grpc/credentials/alts/internal/handshaker -google.golang.org/grpc/credentials/alts/internal/handshaker/service -google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp -google.golang.org/grpc/credentials/google -google.golang.org/grpc/credentials/internal -google.golang.org/grpc/credentials/oauth -google.golang.org/grpc/encoding -google.golang.org/grpc/encoding/proto -google.golang.org/grpc/grpclog -google.golang.org/grpc/internal -google.golang.org/grpc/internal/backoff -google.golang.org/grpc/internal/balancerload -google.golang.org/grpc/internal/binarylog -google.golang.org/grpc/internal/buffer -google.golang.org/grpc/internal/channelz -google.golang.org/grpc/internal/envconfig -google.golang.org/grpc/internal/grpclog -google.golang.org/grpc/internal/grpcrand -google.golang.org/grpc/internal/grpcsync -google.golang.org/grpc/internal/grpcutil -google.golang.org/grpc/internal/resolver/dns -google.golang.org/grpc/internal/resolver/passthrough -google.golang.org/grpc/internal/status -google.golang.org/grpc/internal/syscall -google.golang.org/grpc/internal/transport -google.golang.org/grpc/keepalive -google.golang.org/grpc/metadata -google.golang.org/grpc/naming -google.golang.org/grpc/peer -google.golang.org/grpc/resolver -google.golang.org/grpc/serviceconfig -google.golang.org/grpc/stats -google.golang.org/grpc/status -google.golang.org/grpc/tap -# google.golang.org/protobuf v1.23.0 -google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo -google.golang.org/protobuf/compiler/protogen -google.golang.org/protobuf/encoding/protojson -google.golang.org/protobuf/encoding/prototext -google.golang.org/protobuf/encoding/protowire -google.golang.org/protobuf/internal/descfmt -google.golang.org/protobuf/internal/descopts -google.golang.org/protobuf/internal/detectknown -google.golang.org/protobuf/internal/detrand -google.golang.org/protobuf/internal/encoding/defval -google.golang.org/protobuf/internal/encoding/json -google.golang.org/protobuf/internal/encoding/messageset -google.golang.org/protobuf/internal/encoding/tag -google.golang.org/protobuf/internal/encoding/text -google.golang.org/protobuf/internal/errors -google.golang.org/protobuf/internal/fieldnum -google.golang.org/protobuf/internal/fieldsort -google.golang.org/protobuf/internal/filedesc -google.golang.org/protobuf/internal/filetype -google.golang.org/protobuf/internal/flags -google.golang.org/protobuf/internal/genname -google.golang.org/protobuf/internal/impl -google.golang.org/protobuf/internal/mapsort -google.golang.org/protobuf/internal/pragma -google.golang.org/protobuf/internal/set -google.golang.org/protobuf/internal/strs -google.golang.org/protobuf/internal/version -google.golang.org/protobuf/proto -google.golang.org/protobuf/reflect/protodesc -google.golang.org/protobuf/reflect/protoreflect -google.golang.org/protobuf/reflect/protoregistry -google.golang.org/protobuf/runtime/protoiface -google.golang.org/protobuf/runtime/protoimpl -google.golang.org/protobuf/types/descriptorpb -google.golang.org/protobuf/types/known/anypb -google.golang.org/protobuf/types/known/durationpb -google.golang.org/protobuf/types/known/emptypb -google.golang.org/protobuf/types/known/structpb -google.golang.org/protobuf/types/known/timestamppb -google.golang.org/protobuf/types/known/wrapperspb -google.golang.org/protobuf/types/pluginpb -# gopkg.in/inf.v0 v0.9.1 -gopkg.in/inf.v0 -# gopkg.in/jcmturner/aescts.v1 v1.0.1 -gopkg.in/jcmturner/aescts.v1 -# gopkg.in/jcmturner/dnsutils.v1 v1.0.1 -gopkg.in/jcmturner/dnsutils.v1 -# gopkg.in/jcmturner/goidentity.v3 v3.0.0 -gopkg.in/jcmturner/goidentity.v3 -# gopkg.in/jcmturner/gokrb5.v7 v7.5.0 -gopkg.in/jcmturner/gokrb5.v7/asn1tools -gopkg.in/jcmturner/gokrb5.v7/client -gopkg.in/jcmturner/gokrb5.v7/config -gopkg.in/jcmturner/gokrb5.v7/credentials -gopkg.in/jcmturner/gokrb5.v7/crypto -gopkg.in/jcmturner/gokrb5.v7/crypto/common -gopkg.in/jcmturner/gokrb5.v7/crypto/etype -gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3961 -gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3962 -gopkg.in/jcmturner/gokrb5.v7/crypto/rfc4757 -gopkg.in/jcmturner/gokrb5.v7/crypto/rfc8009 -gopkg.in/jcmturner/gokrb5.v7/gssapi -gopkg.in/jcmturner/gokrb5.v7/iana -gopkg.in/jcmturner/gokrb5.v7/iana/addrtype -gopkg.in/jcmturner/gokrb5.v7/iana/adtype -gopkg.in/jcmturner/gokrb5.v7/iana/asnAppTag -gopkg.in/jcmturner/gokrb5.v7/iana/chksumtype -gopkg.in/jcmturner/gokrb5.v7/iana/errorcode -gopkg.in/jcmturner/gokrb5.v7/iana/etypeID -gopkg.in/jcmturner/gokrb5.v7/iana/flags -gopkg.in/jcmturner/gokrb5.v7/iana/keyusage -gopkg.in/jcmturner/gokrb5.v7/iana/msgtype -gopkg.in/jcmturner/gokrb5.v7/iana/nametype -gopkg.in/jcmturner/gokrb5.v7/iana/patype -gopkg.in/jcmturner/gokrb5.v7/kadmin -gopkg.in/jcmturner/gokrb5.v7/keytab -gopkg.in/jcmturner/gokrb5.v7/krberror -gopkg.in/jcmturner/gokrb5.v7/messages -gopkg.in/jcmturner/gokrb5.v7/pac -gopkg.in/jcmturner/gokrb5.v7/service -gopkg.in/jcmturner/gokrb5.v7/spnego -gopkg.in/jcmturner/gokrb5.v7/types -# gopkg.in/jcmturner/rpc.v1 v1.1.0 -gopkg.in/jcmturner/rpc.v1/mstypes -gopkg.in/jcmturner/rpc.v1/ndr -# gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 -gopkg.in/mgo.v2 -gopkg.in/mgo.v2/bson -gopkg.in/mgo.v2/internal/json -gopkg.in/mgo.v2/internal/sasl -gopkg.in/mgo.v2/internal/scram -# gopkg.in/yaml.v2 v2.3.0 -gopkg.in/yaml.v2 -# honnef.co/go/tools v0.0.1-2019.2.3 -honnef.co/go/tools/arg -honnef.co/go/tools/cmd/staticcheck -honnef.co/go/tools/config -honnef.co/go/tools/deprecated -honnef.co/go/tools/facts -honnef.co/go/tools/functions -honnef.co/go/tools/go/types/typeutil -honnef.co/go/tools/internal/cache -honnef.co/go/tools/internal/passes/buildssa -honnef.co/go/tools/internal/renameio -honnef.co/go/tools/internal/sharedcheck -honnef.co/go/tools/lint -honnef.co/go/tools/lint/lintdsl -honnef.co/go/tools/lint/lintutil -honnef.co/go/tools/lint/lintutil/format -honnef.co/go/tools/loader -honnef.co/go/tools/printf -honnef.co/go/tools/simple -honnef.co/go/tools/ssa -honnef.co/go/tools/ssautil -honnef.co/go/tools/staticcheck -honnef.co/go/tools/staticcheck/vrp -honnef.co/go/tools/stylecheck -honnef.co/go/tools/unused -honnef.co/go/tools/version -# howett.net/plist v0.0.0-20181124034731-591f970eefbb -howett.net/plist -# k8s.io/api v0.18.3 -k8s.io/api/admissionregistration/v1 -k8s.io/api/admissionregistration/v1beta1 -k8s.io/api/apps/v1 -k8s.io/api/apps/v1beta1 -k8s.io/api/apps/v1beta2 -k8s.io/api/auditregistration/v1alpha1 -k8s.io/api/authentication/v1 -k8s.io/api/authentication/v1beta1 -k8s.io/api/authorization/v1 -k8s.io/api/authorization/v1beta1 -k8s.io/api/autoscaling/v1 -k8s.io/api/autoscaling/v2beta1 -k8s.io/api/autoscaling/v2beta2 -k8s.io/api/batch/v1 -k8s.io/api/batch/v1beta1 -k8s.io/api/batch/v2alpha1 -k8s.io/api/certificates/v1beta1 -k8s.io/api/coordination/v1 -k8s.io/api/coordination/v1beta1 -k8s.io/api/core/v1 -k8s.io/api/discovery/v1alpha1 -k8s.io/api/discovery/v1beta1 -k8s.io/api/events/v1beta1 -k8s.io/api/extensions/v1beta1 -k8s.io/api/flowcontrol/v1alpha1 -k8s.io/api/networking/v1 -k8s.io/api/networking/v1beta1 -k8s.io/api/node/v1alpha1 -k8s.io/api/node/v1beta1 -k8s.io/api/policy/v1beta1 -k8s.io/api/rbac/v1 -k8s.io/api/rbac/v1alpha1 -k8s.io/api/rbac/v1beta1 -k8s.io/api/scheduling/v1 -k8s.io/api/scheduling/v1alpha1 -k8s.io/api/scheduling/v1beta1 -k8s.io/api/settings/v1alpha1 -k8s.io/api/storage/v1 -k8s.io/api/storage/v1alpha1 -k8s.io/api/storage/v1beta1 -# k8s.io/apimachinery v0.18.3 -k8s.io/apimachinery/pkg/api/errors -k8s.io/apimachinery/pkg/api/meta -k8s.io/apimachinery/pkg/api/resource -k8s.io/apimachinery/pkg/apis/meta/internalversion -k8s.io/apimachinery/pkg/apis/meta/v1 -k8s.io/apimachinery/pkg/apis/meta/v1/unstructured -k8s.io/apimachinery/pkg/apis/meta/v1beta1 -k8s.io/apimachinery/pkg/conversion -k8s.io/apimachinery/pkg/conversion/queryparams -k8s.io/apimachinery/pkg/fields -k8s.io/apimachinery/pkg/labels -k8s.io/apimachinery/pkg/runtime -k8s.io/apimachinery/pkg/runtime/schema -k8s.io/apimachinery/pkg/runtime/serializer -k8s.io/apimachinery/pkg/runtime/serializer/json -k8s.io/apimachinery/pkg/runtime/serializer/protobuf -k8s.io/apimachinery/pkg/runtime/serializer/recognizer -k8s.io/apimachinery/pkg/runtime/serializer/streaming -k8s.io/apimachinery/pkg/runtime/serializer/versioning -k8s.io/apimachinery/pkg/selection -k8s.io/apimachinery/pkg/types -k8s.io/apimachinery/pkg/util/cache -k8s.io/apimachinery/pkg/util/clock -k8s.io/apimachinery/pkg/util/diff -k8s.io/apimachinery/pkg/util/errors -k8s.io/apimachinery/pkg/util/framer -k8s.io/apimachinery/pkg/util/httpstream -k8s.io/apimachinery/pkg/util/httpstream/spdy -k8s.io/apimachinery/pkg/util/intstr -k8s.io/apimachinery/pkg/util/json -k8s.io/apimachinery/pkg/util/mergepatch -k8s.io/apimachinery/pkg/util/naming -k8s.io/apimachinery/pkg/util/net -k8s.io/apimachinery/pkg/util/runtime -k8s.io/apimachinery/pkg/util/sets -k8s.io/apimachinery/pkg/util/strategicpatch -k8s.io/apimachinery/pkg/util/validation -k8s.io/apimachinery/pkg/util/validation/field -k8s.io/apimachinery/pkg/util/wait -k8s.io/apimachinery/pkg/util/yaml -k8s.io/apimachinery/pkg/version -k8s.io/apimachinery/pkg/watch -k8s.io/apimachinery/third_party/forked/golang/json -k8s.io/apimachinery/third_party/forked/golang/netutil -k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/client-go v0.18.3 -k8s.io/client-go/discovery -k8s.io/client-go/discovery/fake -k8s.io/client-go/kubernetes -k8s.io/client-go/kubernetes/fake -k8s.io/client-go/kubernetes/scheme -k8s.io/client-go/kubernetes/typed/admissionregistration/v1 -k8s.io/client-go/kubernetes/typed/admissionregistration/v1/fake -k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 -k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/fake -k8s.io/client-go/kubernetes/typed/apps/v1 -k8s.io/client-go/kubernetes/typed/apps/v1/fake -k8s.io/client-go/kubernetes/typed/apps/v1beta1 -k8s.io/client-go/kubernetes/typed/apps/v1beta1/fake -k8s.io/client-go/kubernetes/typed/apps/v1beta2 -k8s.io/client-go/kubernetes/typed/apps/v1beta2/fake -k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1 -k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/authentication/v1 -k8s.io/client-go/kubernetes/typed/authentication/v1/fake -k8s.io/client-go/kubernetes/typed/authentication/v1beta1 -k8s.io/client-go/kubernetes/typed/authentication/v1beta1/fake -k8s.io/client-go/kubernetes/typed/authorization/v1 -k8s.io/client-go/kubernetes/typed/authorization/v1/fake -k8s.io/client-go/kubernetes/typed/authorization/v1beta1 -k8s.io/client-go/kubernetes/typed/authorization/v1beta1/fake -k8s.io/client-go/kubernetes/typed/autoscaling/v1 -k8s.io/client-go/kubernetes/typed/autoscaling/v1/fake -k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1 -k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1/fake -k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2 -k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2/fake -k8s.io/client-go/kubernetes/typed/batch/v1 -k8s.io/client-go/kubernetes/typed/batch/v1/fake -k8s.io/client-go/kubernetes/typed/batch/v1beta1 -k8s.io/client-go/kubernetes/typed/batch/v1beta1/fake -k8s.io/client-go/kubernetes/typed/batch/v2alpha1 -k8s.io/client-go/kubernetes/typed/batch/v2alpha1/fake -k8s.io/client-go/kubernetes/typed/certificates/v1beta1 -k8s.io/client-go/kubernetes/typed/certificates/v1beta1/fake -k8s.io/client-go/kubernetes/typed/coordination/v1 -k8s.io/client-go/kubernetes/typed/coordination/v1/fake -k8s.io/client-go/kubernetes/typed/coordination/v1beta1 -k8s.io/client-go/kubernetes/typed/coordination/v1beta1/fake -k8s.io/client-go/kubernetes/typed/core/v1 -k8s.io/client-go/kubernetes/typed/core/v1/fake -k8s.io/client-go/kubernetes/typed/discovery/v1alpha1 -k8s.io/client-go/kubernetes/typed/discovery/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/discovery/v1beta1 -k8s.io/client-go/kubernetes/typed/discovery/v1beta1/fake -k8s.io/client-go/kubernetes/typed/events/v1beta1 -k8s.io/client-go/kubernetes/typed/events/v1beta1/fake -k8s.io/client-go/kubernetes/typed/extensions/v1beta1 -k8s.io/client-go/kubernetes/typed/extensions/v1beta1/fake -k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1 -k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/networking/v1 -k8s.io/client-go/kubernetes/typed/networking/v1/fake -k8s.io/client-go/kubernetes/typed/networking/v1beta1 -k8s.io/client-go/kubernetes/typed/networking/v1beta1/fake -k8s.io/client-go/kubernetes/typed/node/v1alpha1 -k8s.io/client-go/kubernetes/typed/node/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/node/v1beta1 -k8s.io/client-go/kubernetes/typed/node/v1beta1/fake -k8s.io/client-go/kubernetes/typed/policy/v1beta1 -k8s.io/client-go/kubernetes/typed/policy/v1beta1/fake -k8s.io/client-go/kubernetes/typed/rbac/v1 -k8s.io/client-go/kubernetes/typed/rbac/v1/fake -k8s.io/client-go/kubernetes/typed/rbac/v1alpha1 -k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/rbac/v1beta1 -k8s.io/client-go/kubernetes/typed/rbac/v1beta1/fake -k8s.io/client-go/kubernetes/typed/scheduling/v1 -k8s.io/client-go/kubernetes/typed/scheduling/v1/fake -k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1 -k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/scheduling/v1beta1 -k8s.io/client-go/kubernetes/typed/scheduling/v1beta1/fake -k8s.io/client-go/kubernetes/typed/settings/v1alpha1 -k8s.io/client-go/kubernetes/typed/settings/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/storage/v1 -k8s.io/client-go/kubernetes/typed/storage/v1/fake -k8s.io/client-go/kubernetes/typed/storage/v1alpha1 -k8s.io/client-go/kubernetes/typed/storage/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/storage/v1beta1 -k8s.io/client-go/kubernetes/typed/storage/v1beta1/fake -k8s.io/client-go/pkg/apis/clientauthentication -k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1 -k8s.io/client-go/pkg/apis/clientauthentication/v1beta1 -k8s.io/client-go/pkg/version -k8s.io/client-go/plugin/pkg/client/auth/exec -k8s.io/client-go/rest -k8s.io/client-go/rest/watch -k8s.io/client-go/testing -k8s.io/client-go/tools/auth -k8s.io/client-go/tools/cache -k8s.io/client-go/tools/clientcmd -k8s.io/client-go/tools/clientcmd/api -k8s.io/client-go/tools/clientcmd/api/latest -k8s.io/client-go/tools/clientcmd/api/v1 -k8s.io/client-go/tools/metrics -k8s.io/client-go/tools/pager -k8s.io/client-go/tools/portforward -k8s.io/client-go/tools/reference -k8s.io/client-go/tools/watch -k8s.io/client-go/transport -k8s.io/client-go/transport/spdy -k8s.io/client-go/util/cert -k8s.io/client-go/util/connrotation -k8s.io/client-go/util/flowcontrol -k8s.io/client-go/util/homedir -k8s.io/client-go/util/keyutil -k8s.io/client-go/util/workqueue -# k8s.io/klog v1.0.0 -k8s.io/klog -# k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 -k8s.io/kube-openapi/pkg/util/proto -# k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 -k8s.io/utils/buffer -k8s.io/utils/integer -k8s.io/utils/trace -# sigs.k8s.io/structured-merge-diff/v3 v3.0.0 -sigs.k8s.io/structured-merge-diff/v3/value -# sigs.k8s.io/yaml v1.2.0 -sigs.k8s.io/yaml diff --git a/x-pack/auditbeat/module/system/socket/state.go b/x-pack/auditbeat/module/system/socket/state.go index afd72e853fc..c7b3ac761a7 100644 --- a/x-pack/auditbeat/module/system/socket/state.go +++ b/x-pack/auditbeat/module/system/socket/state.go @@ -523,7 +523,7 @@ func (s *state) ExpireOlder() { deadline = s.clock().Add(-s.socketTimeout) for item := s.socketLRU.peek(); item != nil && item.Timestamp().Before(deadline); { if sock, ok := item.(*socket); ok { - s.onSockDestroyed(sock.sock, 0) + s.onSockDestroyed(sock.sock, sock, 0) } else { s.socketLRU.get() } @@ -704,13 +704,16 @@ func (s *state) OnSockDestroyed(ptr uintptr, pid uint32) error { s.Lock() defer s.Unlock() - return s.onSockDestroyed(ptr, pid) + return s.onSockDestroyed(ptr, nil, pid) } -func (s *state) onSockDestroyed(ptr uintptr, pid uint32) error { - sock, found := s.socks[ptr] - if !found { - return nil +func (s *state) onSockDestroyed(ptr uintptr, sock *socket, pid uint32) error { + var found bool + if sock == nil { + sock, found = s.socks[ptr] + if !found { + return nil + } } // Enrich with pid if sock.pid == 0 && pid != 0 { diff --git a/x-pack/auditbeat/module/system/socket/state_test.go b/x-pack/auditbeat/module/system/socket/state_test.go index 9b36c8b5dd4..75b73a374d1 100644 --- a/x-pack/auditbeat/module/system/socket/state_test.go +++ b/x-pack/auditbeat/module/system/socket/state_test.go @@ -152,7 +152,6 @@ func TestTCPConnWithProcessSocketTimeouts(t *testing.T) { lPort, rPort := be16(localPort), be16(remotePort) lAddr, rAddr := ipv4(localIP), ipv4(remoteIP) evs := []event{ - callExecve(meta(1234, 1234, 1), []string{"/usr/bin/curl", "https://example.net/", "-o", "/tmp/site.html"}), &commitCreds{Meta: meta(1234, 1234, 2), UID: 501, GID: 20, EUID: 501, EGID: 20}, &execveRet{Meta: meta(1234, 1234, 2), Retval: 1234}, @@ -302,6 +301,32 @@ func TestTCPConnWithProcessSocketTimeouts(t *testing.T) { } } +func TestSocketExpirationWithOverwrittenSockets(t *testing.T) { + const ( + sock uintptr = 0xff1234 + flowTimeout = time.Hour + socketTimeout = time.Minute * 3 + closeTimeout = time.Minute + ) + st := makeState(nil, (*logWrapper)(t), flowTimeout, socketTimeout, closeTimeout, time.Second) + now := time.Now() + st.clock = func() time.Time { + return now + } + if err := feedEvents([]event{ + &inetCreate{Meta: meta(1234, 1236, 5), Proto: 0}, + &sockInitData{Meta: meta(1234, 1236, 5), Sock: sock}, + &inetCreate{Meta: meta(1234, 1237, 5), Proto: 0}, + &sockInitData{Meta: meta(1234, 1237, 5), Sock: sock}, + }, st, t); err != nil { + t.Fatal(err) + } + now = now.Add(closeTimeout + 1) + st.ExpireOlder() + now = now.Add(socketTimeout + 1) + st.ExpireOlder() +} + func TestUDPOutgoingSinglePacketWithProcess(t *testing.T) { const ( localIP = "192.168.33.10" diff --git a/x-pack/dockerlogbeat/config.json b/x-pack/dockerlogbeat/config.json index d3072a841bc..5d7d1d3c745 100644 --- a/x-pack/dockerlogbeat/config.json +++ b/x-pack/dockerlogbeat/config.json @@ -13,7 +13,31 @@ ], "socket": "beatSocket.sock" }, + "mounts": [ + { + "name": "LOG_DIR", + "description": "Mount for local log cache", + "destination": "/var/log/docker", + "source": "/var/lib/docker", + "type": "none", + "options": [ + "rw", + "rbind" + ], + "Settable": [ + "source" + ] + } + ], "env": [ + { + "description": "Destroy logs after a container has stopped", + "name": "DESTROY_LOGS_ON_STOP", + "value": "false", + "Settable": [ + "value" + ] + }, { "description": "debug level", "name": "LOG_DRIVER_LEVEL", diff --git a/x-pack/dockerlogbeat/docs/configuration.asciidoc b/x-pack/dockerlogbeat/docs/configuration.asciidoc index 708eb941a91..f1bf6821489 100644 --- a/x-pack/dockerlogbeat/docs/configuration.asciidoc +++ b/x-pack/dockerlogbeat/docs/configuration.asciidoc @@ -49,10 +49,6 @@ format is `"username:password"`. [[es-output-options]] === {es} output options -// TODO: Add the following settings. Syntax is a little different so we might -// need to add deameon examples that show how to specify these settings: -// `output.elasticsearch.indices -// `output.elasticsearch.pipelines` [options="header"] |===== @@ -117,3 +113,72 @@ for more information about the environment variables. |===== + + +[float] +[[local-log-opts]] +=== Configuring the local log +This plugin fully supports `docker logs`, and it maintains a local copy of logs that can be read without a connection to Elasticsearch. The plugin mounts the `/var/lib/docker` directory on the host to write logs to `/var/log/containers` on the host. If you want to change the log location on the host, you must change the mount inside the plugin: + +1. Disable the plugin: ++ +["source","sh",subs="attributes"] +---- +docker plugin disable elastic/{log-driver-alias}:{version} +---- + +2. Set the bindmount directory: ++ +["source","sh",subs="attributes"] +---- +docker plugin set elastic/{log-driver-alias}:{version} LOG_DIR.source=NEW_LOG_LOCATION +---- ++ + +3. Enable the plugin: ++ +["source","sh",subs="attributes"] +---- +docker plugin enable elastic/{log-driver-alias}:{version} +---- + + +The local log also supports the `max-file`, `max-size` and `compress` options that are https://docs.docker.com/config/containers/logging/json-file/#options[a part of the Docker default file logger]. For example: + +["source","sh",subs="attributes"] +---- +docker run --log-driver=elastic/{log-driver-alias}:{version} \ + --log-opt endpoint="myhost:9200" \ + --log-opt user="myusername" \ + --log-opt password="mypassword" \ + --log-opt max-file=10 \ + --log-opt max-size=5M \ + --log-opt compress=true \ + -it debian:jessie /bin/bash +---- + + +In situations where logs can't be easily managed, for example, you can also configure the plugin to remove log files when a container is stopped. This will prevent you from reading logs on a stopped container, but it will rotate logs without user intervention. To enable removal of logs for stopped containers, you must change the `DESTROY_LOGS_ON_STOP` environment variable: + +1. Disable the plugin: ++ +["source","sh",subs="attributes"] +---- +docker plugin disable elastic/{log-driver-alias}:{version} +---- + +2. Enable log removal: ++ +["source","sh",subs="attributes"] +---- +docker plugin set elastic/{log-driver-alias}:{version} DESTROY_LOGS_ON_STOP=true +---- ++ + +3. Enable the plugin: ++ +["source","sh",subs="attributes"] +---- +docker plugin enable elastic/{log-driver-alias}:{version} +---- + diff --git a/x-pack/dockerlogbeat/handlers.go b/x-pack/dockerlogbeat/handlers.go index 604c029e601..8b3a771a741 100644 --- a/x-pack/dockerlogbeat/handlers.go +++ b/x-pack/dockerlogbeat/handlers.go @@ -6,12 +6,14 @@ package main import ( "encoding/json" + "io" "net/http" "github.com/docker/docker/daemon/logger" "github.com/elastic/beats/v7/x-pack/dockerlogbeat/pipelinemanager" + "github.com/docker/docker/pkg/ioutils" "github.com/pkg/errors" ) @@ -26,6 +28,26 @@ type StopLoggingRequest struct { File string } +// capabilitiesResponse represents the response to a capabilities request +type capabilitiesResponse struct { + Err string + Cap logger.Capability +} + +// logsRequest represents the request object we get from a `docker logs` call +type logsRequest struct { + Info logger.Info + Config logger.ReadConfig +} + +func reportCaps() func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(&capabilitiesResponse{ + Cap: logger.Capability{ReadLogs: true}, + }) + } +} + // This gets called when a container starts that requests the log driver func startLoggingHandler(pm *pipelinemanager.PipelineManager) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { @@ -36,7 +58,7 @@ func startLoggingHandler(pm *pipelinemanager.PipelineManager) func(w http.Respon return } - pm.Logger.Infof("Got start request object from container %#v\n", startReq.Info.ContainerName) + pm.Logger.Debugf("Got start request object from container %#v\n", startReq.Info.ContainerName) pm.Logger.Debugf("Got a container with the following labels: %#v\n", startReq.Info.ContainerLabels) pm.Logger.Debugf("Got a container with the following log opts: %#v\n", startReq.Info.Config) @@ -67,7 +89,7 @@ func stopLoggingHandler(pm *pipelinemanager.PipelineManager) func(w http.Respons http.Error(w, errors.Wrap(err, "error decoding json request").Error(), http.StatusBadRequest) return } - pm.Logger.Infof("Got stop request object %#v\n", stopReq) + pm.Logger.Debugf("Got stop request object %#v\n", stopReq) // Run the stop async, since nothing 'depends' on it, // and we can break people's docker automation if this times out. go func() { @@ -81,6 +103,30 @@ func stopLoggingHandler(pm *pipelinemanager.PipelineManager) func(w http.Respons } // end func } +func readLogHandler(pm *pipelinemanager.PipelineManager) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + var logReq logsRequest + err := json.NewDecoder(r.Body).Decode(&logReq) + if err != nil { + http.Error(w, errors.Wrap(err, "error decoding json request").Error(), http.StatusBadRequest) + return + } + + pm.Logger.Debugf("Got logging request for container %s\n", logReq.Info.ContainerName) + stream, err := pm.CreateReaderForContainer(logReq.Info, logReq.Config) + if err != nil { + http.Error(w, errors.Wrap(err, "error creating log reader").Error(), http.StatusBadRequest) + return + } + defer stream.Close() + w.Header().Set("Content-Type", "application/x-json-stream") + wf := ioutils.NewWriteFlusher(w) + defer wf.Close() + io.Copy(wf, stream) + + } //end func +} + // For the start/stop handler, the daemon expects back an error object. If the body is empty, then all is well. func respondOK(w http.ResponseWriter) { res := struct { diff --git a/x-pack/dockerlogbeat/main.go b/x-pack/dockerlogbeat/main.go index 360fd265caa..e3a5b8d0310 100644 --- a/x-pack/dockerlogbeat/main.go +++ b/x-pack/dockerlogbeat/main.go @@ -7,6 +7,7 @@ package main import ( "fmt" "os" + "strconv" "github.com/docker/go-plugins-helpers/sdk" @@ -41,6 +42,14 @@ func genNewMonitoringConfig() (*common.Config, error) { return cfg, nil } +func setDestroyLogsOnStop() (bool, error) { + setting, ok := os.LookupEnv("DESTROY_LOGS_ON_STOP") + if !ok { + return false, nil + } + return strconv.ParseBool(setting) +} + func fatal(format string, vs ...interface{}) { fmt.Fprintf(os.Stderr, format, vs...) os.Exit(1) @@ -60,12 +69,18 @@ func main() { fatal("error starting log handler: %s", err) } - pipelines := pipelinemanager.NewPipelineManager(logcfg) + logDestroy, err := setDestroyLogsOnStop() + if err != nil { + fatal("DESTROY_LOGS_ON_STOP must be 'true' or 'false': %s", err) + } + pipelines := pipelinemanager.NewPipelineManager(logDestroy) sdkHandler := sdk.NewHandler(`{"Implements": ["LoggingDriver"]}`) // Create handlers for startup and shutdown of the log driver sdkHandler.HandleFunc("/LogDriver.StartLogging", startLoggingHandler(pipelines)) sdkHandler.HandleFunc("/LogDriver.StopLogging", stopLoggingHandler(pipelines)) + sdkHandler.HandleFunc("/LogDriver.Capabilities", reportCaps()) + sdkHandler.HandleFunc("/LogDriver.ReadLogs", readLogHandler(pipelines)) err = sdkHandler.ServeUnix("beatSocket", 0) if err != nil { diff --git a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go index 62d54c88b6a..1a82dd214e5 100644 --- a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go +++ b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go @@ -5,13 +5,15 @@ package pipelinemanager import ( - "os" + "io" "strings" "time" "github.com/docker/docker/api/types/plugins/logdriver" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/api/types/backend" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/acker" @@ -20,20 +22,26 @@ import ( "github.com/elastic/beats/v7/x-pack/dockerlogbeat/pipereader" ) -// ClientLogger is an instance of a pipeline logger client meant for reading from a single log stream -// There's a many-to-one relationship between clients and pipelines. -// Each container with the same config will get its own client to the same pipeline. +// ClientLogger collects logs for a docker container logging to stdout and stderr, using the FIFO provided by the docker daemon. +// Each log line is written to a local log file for retrieval via "docker logs", and forwarded to the beats publisher pipeline. +// The local log storage is based on the docker json-file logger and supports the same settings. If "max-size" is not configured, we will rotate the log file every 10MB. type ClientLogger struct { - logFile *pipereader.PipeReader - client beat.Client - pipelineHash uint64 - closer chan struct{} - containerMeta logger.Info - logger *logp.Logger + // pipelineHash is a hash of the libbeat publisher pipeline config + pipelineHash uint64 + // logger is the internal error message logger + logger *logp.Logger + // ContainerMeta is the metadata object for the container we get from docker + ContainerMeta logger.Info + // logFile is the FIFO reader that reads from the docker container stdio + logFile *pipereader.PipeReader + // client is the libbeat client object that sends logs upstream + client beat.Client + // localLog manages the local JSON logs for containers + localLog logger.Logger } // newClientFromPipeline creates a new Client logger with a FIFO reader and beat client -func newClientFromPipeline(pipeline beat.PipelineConnector, inputFile *pipereader.PipeReader, hash uint64, info logger.Info) (*ClientLogger, error) { +func newClientFromPipeline(pipeline beat.PipelineConnector, inputFile *pipereader.PipeReader, hash uint64, info logger.Info, localLog logger.Logger) (*ClientLogger, error) { // setup the beat client settings := beat.ClientConfig{ WaitClose: 0, @@ -50,7 +58,12 @@ func newClientFromPipeline(pipeline beat.PipelineConnector, inputFile *pipereade clientLogger.Debugf("Created new logger for %d", hash) - return &ClientLogger{logFile: inputFile, client: client, pipelineHash: hash, closer: make(chan struct{}), containerMeta: info, logger: clientLogger}, nil + return &ClientLogger{logFile: inputFile, + client: client, + pipelineHash: hash, + ContainerMeta: info, + localLog: localLog, + logger: clientLogger}, nil } // Close closes the pipeline client and reader @@ -64,7 +77,6 @@ func (cl *ClientLogger) Close() error { // ConsumePipelineAndSend consumes events from the FIFO pipe and sends them to the pipeline client func (cl *ClientLogger) ConsumePipelineAndSend() { publishWriter := make(chan logdriver.LogEntry, 500) - go cl.publishLoop(publishWriter) // Clean up the reader after we're done defer func() { @@ -76,7 +88,10 @@ func (cl *ClientLogger) ConsumePipelineAndSend() { for { err := cl.logFile.ReadMessage(&log) if err != nil { - cl.logger.Error(os.Stderr, "Error getting message: %s\n", err) + if err == io.EOF { + return + } + cl.logger.Errorf("Error getting message: %s\n", err) return } publishWriter <- log @@ -96,6 +111,7 @@ func (cl *ClientLogger) publishLoop(reader chan logdriver.LogEntry) { return } + cl.localLog.Log(constructLogSpoolMsg(entry)) line := strings.TrimSpace(string(entry.Line)) cl.client.Publish(beat.Event{ @@ -103,11 +119,11 @@ func (cl *ClientLogger) publishLoop(reader chan logdriver.LogEntry) { Fields: common.MapStr{ "message": line, "container": common.MapStr{ - "labels": helper.DeDotLabels(cl.containerMeta.ContainerLabels, true), - "id": cl.containerMeta.ContainerID, - "name": helper.ExtractContainerName([]string{cl.containerMeta.ContainerName}), + "labels": helper.DeDotLabels(cl.ContainerMeta.ContainerLabels, true), + "id": cl.ContainerMeta.ContainerID, + "name": helper.ExtractContainerName([]string{cl.ContainerMeta.ContainerName}), "image": common.MapStr{ - "name": cl.containerMeta.ContainerImageName, + "name": cl.ContainerMeta.ContainerImageName, }, }, }, @@ -116,3 +132,18 @@ func (cl *ClientLogger) publishLoop(reader chan logdriver.LogEntry) { } } + +func constructLogSpoolMsg(line logdriver.LogEntry) *logger.Message { + var msg logger.Message + + msg.Line = line.Line + msg.Source = line.Source + msg.Timestamp = time.Unix(0, line.TimeNano) + if line.PartialLogMetadata != nil { + msg.PLogMetaData = &backend.PartialLogMetaData{} + msg.PLogMetaData.ID = line.PartialLogMetadata.Id + msg.PLogMetaData.Last = line.PartialLogMetadata.Last + msg.PLogMetaData.Ordinal = int(line.PartialLogMetadata.Ordinal) + } + return &msg +} diff --git a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader_test.go b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader_test.go index 4f396b194be..b53d26e234d 100644 --- a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader_test.go +++ b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader_test.go @@ -5,13 +5,18 @@ package pipelinemanager import ( + "os" + "path/filepath" "sync" "testing" + "time" "github.com/docker/docker/daemon/logger" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/docker/docker/daemon/logger/jsonfilelog" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/x-pack/dockerlogbeat/pipelinemock" @@ -85,7 +90,17 @@ func createNewClient(t *testing.T, logString string, mockConnector *pipelinemock reader, err := pipereader.NewReaderFromReadCloser(pipelinemock.CreateTestInputFromLine(t, logString)) require.NoError(t, err) - client, err := newClientFromPipeline(mockConnector, reader, 123, cfgObject) + info := logger.Info{ + ContainerID: "b87d3b0379f816a5f2f7070f28cc05e2f564a3fb549a67c64ec30fc5b04142ed", + LogPath: filepath.Join("/tmp/dockerbeattest/", string(time.Now().Unix())), + } + + err = os.MkdirAll(filepath.Dir(info.LogPath), 0755) + assert.NoError(t, err) + localLog, err := jsonfilelog.New(info) + assert.NoError(t, err) + + client, err := newClientFromPipeline(mockConnector, reader, 123, cfgObject, localLog) require.NoError(t, err) return client diff --git a/x-pack/dockerlogbeat/pipelinemanager/config.go b/x-pack/dockerlogbeat/pipelinemanager/config.go index 3da18bc8546..92d6e98ee9f 100644 --- a/x-pack/dockerlogbeat/pipelinemanager/config.go +++ b/x-pack/dockerlogbeat/pipelinemanager/config.go @@ -35,7 +35,7 @@ func NewCfgFromRaw(input map[string]string) (ContainerOutputConfig, error) { newCfg := ContainerOutputConfig{} endpoint, ok := input["hosts"] if !ok { - return newCfg, errors.New("An endpoint flag is required") + return newCfg, errors.New("A hosts flag is required") } endpointList := strings.Split(endpoint, ",") diff --git a/x-pack/dockerlogbeat/pipelinemanager/pipelineManager.go b/x-pack/dockerlogbeat/pipelinemanager/pipelineManager.go index e96caa77863..b1d04d16541 100644 --- a/x-pack/dockerlogbeat/pipelinemanager/pipelineManager.go +++ b/x-pack/dockerlogbeat/pipelinemanager/pipelineManager.go @@ -5,7 +5,11 @@ package pipelinemanager import ( + "encoding/binary" "fmt" + "io" + "os" + "path/filepath" "sync" "github.com/mitchellh/hashstructure" @@ -14,7 +18,11 @@ import ( "github.com/pkg/errors" + "github.com/docker/docker/api/types/plugins/logdriver" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/jsonfilelog" + + protoio "github.com/gogo/protobuf/io" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" @@ -41,14 +49,23 @@ type PipelineManager struct { pipelines map[uint64]*Pipeline // clients config: filepath clients map[string]*ClientLogger + // Client Logger key: container hash + clientLogger map[string]logger.Logger + // logDirectory is the bindmount for local container logsd + logDirectory string + // destroyLogsOnStop indicates for the client to remove log files when a container stops + destroyLogsOnStop bool } // NewPipelineManager creates a new Pipeline map -func NewPipelineManager(logCfg *common.Config) *PipelineManager { +func NewPipelineManager(logDestroy bool) *PipelineManager { return &PipelineManager{ - Logger: logp.NewLogger("PipelineManager"), - pipelines: make(map[uint64]*Pipeline), - clients: make(map[string]*ClientLogger), + Logger: logp.NewLogger("PipelineManager"), + pipelines: make(map[uint64]*Pipeline), + clients: make(map[string]*ClientLogger), + clientLogger: make(map[string]logger.Logger), + logDirectory: "/var/log/docker/containers", + destroyLogsOnStop: logDestroy, } } @@ -62,13 +79,16 @@ func (pm *PipelineManager) CloseClientWithFile(file string) error { hash := cl.pipelineHash + // remove the logger + pm.removeLogger(cl.ContainerMeta) + pm.Logger.Debugf("Closing Client first from pipelineManager") err = cl.Close() if err != nil { return errors.Wrap(err, "error closing client") } - //if the pipeline is no longer in use, clean up + // if the pipeline is no longer in use, clean up pm.removePipelineIfNeeded(hash) return nil @@ -89,20 +109,90 @@ func (pm *PipelineManager) CreateClientWithConfig(containerConfig ContainerOutpu reader, err := pipereader.NewReaderFromPath(file) if err != nil { - return nil, errors.Wrap(err, "") + return nil, errors.Wrap(err, "error creating reader for docker log stream") + } + + // Why is this empty by default? What should be here? Who knows! + if info.LogPath == "" { + info.LogPath = filepath.Join(pm.logDirectory, info.ContainerID, fmt.Sprintf("%s-json.log", info.ContainerID)) + } + err = os.MkdirAll(filepath.Dir(info.LogPath), 0755) + if err != nil { + return nil, errors.Wrap(err, "error creating directory for local logs") + } + // set a default log size + if _, ok := info.Config["max-size"]; !ok { + info.Config["max-size"] = "10M" + } + // set a default log count + if _, ok := info.Config["max-file"]; !ok { + info.Config["max-file"] = "5" + } + + localLog, err := jsonfilelog.New(info) + if err != nil { + return nil, errors.Wrap(err, "error creating local log") } //actually get to crafting the new client. - cl, err := newClientFromPipeline(pipeline.pipeline, reader, hashstring, info) + cl, err := newClientFromPipeline(pipeline.pipeline, reader, hashstring, info, localLog) if err != nil { return nil, errors.Wrap(err, "error creating client") } pm.registerClient(cl, hashstring, file) - + pm.registerLogger(localLog, info) return cl, nil } +// CreateReaderForContainer responds to docker logs requests to pull local logs from the json logger +func (pm *PipelineManager) CreateReaderForContainer(info logger.Info, config logger.ReadConfig) (io.ReadCloser, error) { + logObject, exists := pm.getLogger(info) + if !exists { + return nil, fmt.Errorf("Could not find logger for %s", info.ContainerID) + } + pipeReader, pipeWriter := io.Pipe() + logReader, ok := logObject.(logger.LogReader) + if !ok { + return nil, fmt.Errorf("logger does not support reading") + } + + go func() { + watcher := logReader.ReadLogs(config) + + enc := protoio.NewUint32DelimitedWriter(pipeWriter, binary.BigEndian) + defer enc.Close() + defer watcher.ConsumerGone() + var rawLog logdriver.LogEntry + for { + select { + case msg, ok := <-watcher.Msg: + if !ok { + pipeWriter.Close() + return + } + rawLog.Line = msg.Line + rawLog.Partial = msg.PLogMetaData != nil + rawLog.TimeNano = msg.Timestamp.UnixNano() + rawLog.Source = msg.Source + + if err := enc.WriteMsg(&rawLog); err != nil { + pipeWriter.CloseWithError(err) + return + } + + case err := <-watcher.Err: + pipeWriter.CloseWithError(err) + return + + } + } + + }() + + return pipeReader, nil +} + //=================== // Private methods @@ -134,6 +224,13 @@ func (pm *PipelineManager) getClient(file string) (*ClientLogger, bool) { return cli, exists } +func (pm *PipelineManager) getLogger(info logger.Info) (logger.Logger, bool) { + pm.mu.Lock() + defer pm.mu.Unlock() + logger, exists := pm.clientLogger[info.ContainerID] + return logger, exists +} + // removePipeline removes a pipeline from the manager if it's refcount is zero. func (pm *PipelineManager) removePipelineIfNeeded(hash uint64) { pm.mu.Lock() @@ -161,6 +258,35 @@ func (pm *PipelineManager) registerClient(cl *ClientLogger, hash uint64, clientF pm.pipelines[hash].refCount++ } +// registerLogger registers a local logger used for reading back logs +func (pm *PipelineManager) registerLogger(log logger.Logger, info logger.Info) { + pm.mu.Lock() + defer pm.mu.Unlock() + pm.clientLogger[info.ContainerID] = log +} + +// removeLogger removes a logging instace +func (pm *PipelineManager) removeLogger(info logger.Info) { + pm.mu.Lock() + defer pm.mu.Unlock() + logger, exists := pm.clientLogger[info.ContainerID] + if !exists { + return + } + logger.Close() + delete(pm.clientLogger, info.ContainerID) + if pm.destroyLogsOnStop { + pm.removeLogFile(info.ContainerID) + } +} + +// removeLogFile removes a log file for a given container. Disabled by default. +func (pm *PipelineManager) removeLogFile(id string) error { + toRemove := filepath.Join(pm.logDirectory, id) + + return os.Remove(toRemove) +} + // removeClient deregisters a client func (pm *PipelineManager) removeClient(file string) (*ClientLogger, error) { pm.mu.Lock() diff --git a/x-pack/dockerlogbeat/pipereader/reader.go b/x-pack/dockerlogbeat/pipereader/reader.go index f622fb03ce6..d1f8eb05c21 100644 --- a/x-pack/dockerlogbeat/pipereader/reader.go +++ b/x-pack/dockerlogbeat/pipereader/reader.go @@ -54,7 +54,7 @@ func (reader *PipeReader) ReadMessage(log *logdriver.LogEntry) error { for { lenFrame, err = reader.getValidLengthFrame() if err != nil { - return errors.Wrap(err, "error getting length frame") + return err } if lenFrame <= reader.maxSize { break diff --git a/x-pack/dockerlogbeat/readme.md b/x-pack/dockerlogbeat/readme.md index b6c97035c53..be06d96daa9 100644 --- a/x-pack/dockerlogbeat/readme.md +++ b/x-pack/dockerlogbeat/readme.md @@ -48,3 +48,13 @@ The location of the logs AND the container base directory in the docker docs is You can use this to find the list of plugins running on the host: `runc --root /containers/services/docker/rootfs/run/docker/plugins/runtime-root/plugins.moby/ list` The logs are in `/var/log/docker`. If you want to make the logs useful, you need to find the ID of the plugin. Back on the darwin host, run `docker plugin inspect $name_of_plugin | grep Id` use the hash ID to grep through the logs: `grep 22bb02c1506677cd48cc1cfccc0847c1b602f48f735e51e4933001804f86e2e docker.*` + + +## Local logs + +This plugin fully supports `docker logs`, and it maintains a local copy of logs that can be read without a connection to Elasticsearch. Unfortunately, due to the limitations in the docker plugin API, we can't "clean up" log files when a container is destroyed. The plugin mounts the `/var/lib/docker` directory on the host to write logs. This mount point can be changed via [Docker](https://docs.docker.com/engine/reference/commandline/plugin_set/#change-the-source-of-a-mount). The plugin can also be configured to do a "hard" cleanup and destroy logs when a container stops. To enable this, set the `DESTROY_LOGS_ON_STOP` environment var inside the plugin: +``` +docker plugin set d805664c550e DESTROY_LOGS_ON_STOP=true +``` + +You can also set `max-file`, `max-size` and `compress` via `--log-opts` \ No newline at end of file diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index c5c0892c016..a11165e2890 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -45,6 +45,7 @@ - Guard against empty stream.datasource and namespace {pull}18769[18769] - Fix install service script for windows {pull}18814[18814] - Properly stops subprocess on shutdown {pull}19567[19567] +- Forward revision number of the configuration to the endpoint. {pull}19759[19759] ==== New features @@ -77,3 +78,4 @@ - Rename input.type logs to logfile {pull}19360[19360] - Agent now installs/uninstalls Elastic Endpoint {pull}19248[19248] - Agent now downloads Elastic Endpoint {pull}19503[19503] +- Agent now load balances across multiple Kibana instances {pull}19628[19628] diff --git a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl index 81dbde65419..3cb0bcdac46 100644 --- a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl @@ -42,7 +42,7 @@ inputs: # access_token: "" # kibana: # # kibana minimal configuration -# host: "localhost:5601" +# hosts: ["localhost:5601"] # ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # optional values diff --git a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl index 59ea36c70bc..97290ee4dae 100644 --- a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl @@ -34,7 +34,7 @@ inputs: # access_token: "" # kibana: # # kibana minimal configuration -# host: "localhost:5601" +# hosts: ["localhost:5601"] # ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # optional values diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl index 212d6944b6e..70b4f496c3c 100644 --- a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl @@ -34,7 +34,7 @@ inputs: # access_token: "" # kibana: # # kibana minimal configuration -# host: "localhost:5601" +# hosts: ["localhost:5601"] # ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # optional values diff --git a/x-pack/elastic-agent/_meta/elastic-agent.yml b/x-pack/elastic-agent/_meta/elastic-agent.yml index e8d47c9ea75..8c3c518194d 100644 --- a/x-pack/elastic-agent/_meta/elastic-agent.yml +++ b/x-pack/elastic-agent/_meta/elastic-agent.yml @@ -34,7 +34,7 @@ inputs: # access_token: "" # kibana: # # kibana minimal configuration -# host: "localhost:5601" +# hosts: ["localhost:5601"] # ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # optional values diff --git a/x-pack/elastic-agent/elastic-agent.docker.yml b/x-pack/elastic-agent/elastic-agent.docker.yml index 8c51ecd6120..6f800c1dc08 100644 --- a/x-pack/elastic-agent/elastic-agent.docker.yml +++ b/x-pack/elastic-agent/elastic-agent.docker.yml @@ -34,7 +34,7 @@ inputs: # access_token: "" # kibana: # # kibana minimal configuration -# host: "localhost:5601" +# hosts: ["localhost:5601"] # ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # optional values diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml index d5a5966dfb4..0607ef8d226 100644 --- a/x-pack/elastic-agent/elastic-agent.reference.yml +++ b/x-pack/elastic-agent/elastic-agent.reference.yml @@ -40,7 +40,7 @@ inputs: # access_token: "" # kibana: # # kibana minimal configuration -# host: "localhost:5601" +# hosts: ["localhost:5601"] # ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # optional values diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml index d1c7c6220be..b709e9c8ab5 100644 --- a/x-pack/elastic-agent/elastic-agent.yml +++ b/x-pack/elastic-agent/elastic-agent.yml @@ -48,7 +48,7 @@ inputs: # access_token: "" # kibana: # # kibana minimal configuration -# host: "localhost:5601" +# hosts: ["localhost:5601"] # ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # optional values diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index baaf9eb741c..46175c0c64e 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -20,7 +20,7 @@ func init() { // spec/endpoint.yml // spec/filebeat.yml // spec/metricbeat.yml - unpacked := packer.MustUnpack("eJy8mFuToziWx9/3Y9TrbuyCKOc2E9EPhmxudpJlnCkJvSHJBmwJuwt8gYn57hPiYoOzqrurZ6YfMjJNCulc/uecn/z3T+Vxw/5vU/DjIS+q/62l+PS3T1Q6FXk7pBGa7ZlrHmmxSt8B3HEcHLm3X8RA37/mlqAyulAgTtzWG4JCnUmhbVbHjBXRkUhnx58PKbnvUREXArsIBSvIMQbvT/5zbLw+p4sYZCIG1TZBs4a7TkmfD4vl2hIbF+4wIEfqvj/Z+Tz1besS4+jwms/z8b7sblver8uY5M1rekh9e54u1/OcS1gniMz8/hl3RUWQqSsbX5r5grlmwx21X6jF6Fq+pofKd+FngsItkaIkb4eFes/3rIy76ZNvB9/2f+1361ynJsZLb/e88u3gtrc/smu51nXm8jpGkXh4XhMcnjkOdgS/5KN9vnPuZP1pI8XlW76Gu/nFLqyaQFOnUpyYEWXUvTzZuZYSnIlYN2WCrmKIHXMdLXk+pL6EJ+JZ5wTNtCUORWzAOsHRLZ4xDgrW9DEaYo5mH3z+aEugUxc2XbzJceOYDfcCESPtyfcq0+6fUy8STJggRled4CGuVkPQVcRGdGa7Q5qg2YXjqOn/95Xg/ZPvRTPmvve5Ixn1oLjbqY31uWhjIEXJXVhj42GtFwrqwh13zfo1t460sHTuvfS5rsTmrdV6FsurIPPeV+mUHMGRDi2NFVBMfNp93H9in9oPr26fpz7OK981de5ZOp8fFhxkgu4O6dp1mhWazRY2l9SFgtvs6Nux9H/JMqYJjSC9Wdjw/xEQJ+ZBjem6uVjPZZBbfozD1xjpghlWFoP3QwD6/W1W+jYX1HUa7oodAzBjMjwE9X7x6X+6NrLNxYZukg9tRJUPCkSMV0PraCUWS5jx+bEr1dyifq47fn5J/SIU3IOXpRQlXc8ElU5OXbj/glRKQtGueVxbRIJiq4xxJJYSnmIUlAStTCKdkoH3fGnP8+V795si5xQjLiiCJ27PKgoi8QWnFXOdXVLrfcj80rf9Klqr30EVo1lGAKyIkv9of+4FOllP1pYU8CJBs2Ipr4JLWH5BkYgLWPhCW8Q40BJEsthYPfmuiknULNsWB3OCHO13yyFvJfArQaGGgTgRF34e5MU9cVHxpq5ZsEvbQo9UHlV5bJkR1QQ5FTasmoJQMCM836TnmicMwjOVpExQqHXyVm062saIaAT3La0rpSffvZ6J8dKWC0XO5bFVPJRhzdF1UnIxMC8baGbUvW65a26pKxr+fG8dvm1ptDmkg83sckhv//to64kC89KXQ/tDcLYj2NJaTRWhxiTMKH5pc5+gVfv7VqptnoMLk+aO4LChRtCoPD3YqlHdLBMcaoOvnS2qlFReRjEtXv6sH/eYSyipEXRtwlPtvq2jPlekpkB78t2+dC9DW/3p/sy4+bzo27rG1Dh2Oh8w6FrA9/L2aG+CI0HfPvoxObPTWxXj+cOofmiV3k3f95YonRMD14zfxvx8Yler69U4dnrGPOveOm/PW112+hzH2p63NdGP8fbv8Tm+bak6PXG7H325VVLAHvaGoK19I9SIK06vudVwNzrSdHoOwUTQYnXmbnh5zS0tNubf2QfumfINRZcHW9TZZ+KauwRANWr2FIRfbzU49rU2a4KiI9PNhrqmofx6za3u2Tf8XxrhjIHrmbSaFM0oDp3OkAkINNuxObYpAXD2mls6A3BLDaipfP/Ae/sYRxkDf+6dpZxlFMFG9eZBA8u18lE/cwm37fpJTfaj2HW0GIeC27OTGmf/aYyYxrLTyP0z3LP5Xe93TWjpkOdxbIgrmtFeXc4ut/ebl/vfJ4IjqXJ/x4FM4571KwPm/Xz3pybcwYyj6MjvdmSbAtYEdXpMPHhHTDcsqQH3RCpEjOrNPe4FBWZBXXjhKGg48u9YAhyZgF/+YoQKz8wb9GNJJs3qY7+JzlPeaPuU+qzm2sf8FyNkd0dz+SNuDfZ8JVg0yieyOhqJC1VvUBgVcBdWrE7FRq/u59uz3r6Qb9f7NMjjFtfeVI/H4SVGoVjY/MxxdOF4VSzteUHQNWNGdIyNUMQ42CUdinV1XrNS2RaAKiOyyoL6kgZKL0ao9D/BM7mpvubsG4D2hqDGpNj1QLajSMGJLrgXHGPQg1vH9ymqb6DTEBzpTDnkaqdb4p51GaNr8wBFw1o1eC7UdTTye2An9YxKpyBIVw31RJG5J2/65yVWXFpWPRf/Ftjd98dRzdEDBKqBoACqnpVt037W9wQFOqkDbkun5q6QcTeYWwGw2qwIjuoEhT2oWWdmRJM7ayeQrjFN7oyTe5R+Jh4s1R2N2GazwaGCiNMG6bc7kgIHFW+CV+qeqQC1HZpLuTqrxq2EtSxERe3ZPsHhADILVrT7PvVF+BkPIPB2SDeGNgHCBM32BKdDAWgbbKnBN/jYdAUmTomEChx6CNW3zAvOMYANA2Y9FCUFs20MzBOR12MHtuLEAKy5Y2akiAR7BNRebz001Eo7FL3/kUIeimewp7dTz9jzw/38GzD0HQDZUcOaYeCU1PkO6HVn388cDayPvs/OdDL0LbFxQ8G81ZPvlPkNouq2Lo49gN602l1qJkCZ49WDrUZ0xuB6ZMZqOjwHUBvlaAICP+THLYc5QeTM5PtfDZN9kw62wxDABj9yN9syCQuCs9vFo4eGVsPDpUPpDOefvy5B38eMl/1vDvt/FRDc26Xwu5DQXhZX/w5QhlsOhJY4CgC52HjzPwbOXqBqfLN4NldfOqD572VeHj/GqPtpz3g+pMEYfmwtZe0FWBdT8OovJpO190uQ6t/qIngfpHqWALiNcVDHjxDYa+TWJ+7wONLKYHMobjHrbKuoQQQGLVCM7ejzNHpv/gNA+QAmPwShf+adAVz/QmjmOBQY3L9PGvrNsJYUwVnl5UH/fX7HDPGbfX0yR5fyWxfZ24ycAOWtxwBYEhRq6rI+qo/pfP5DwDb9zmytdIdfDgGu6MSf7kuith4x1kbfpY3ivE6L4XLUQRgsYxxqapYS5NQxSItla8Md3Hybf40R+Rqv279LCrhiiSax2dFOf/750z/+658BAAD//+UP9yY=") + unpacked := packer.MustUnpack("eJy8mFt3qziWx9/nY5zXmTUD4jhT9Fr1YEiBwQ45Jokk9IYkG7AlTAV8gV793XuJiw3OSVWd6u56yEpMhLQv/733T/77l7LYsP/b5Lw4ZHn1v7UUX/72hUqnIq+HJESzPXPNgubr5A3AHcd+wRf7ZQT0/XNmCSrDMwXiyG29ISjQmRTaZl2kLA8LIp0dfzwk5LZHRVwI7DwQLCdFBN4evMfIeH5MlhFIRQSqbYxmDXedkj4elqsXS2xcuMOAFNR9e7CzeeLZ1jnC4eE5m2fjfdnNtqxflzLJm+fkkHj2PFm9zDMuYR0jMvP6Z9wVFUGmrmx8auZL5poNd9R+gRahS/mcHCrPhV8JCrZEipK8HpbqPW9hpdxNHjzb/77/L163znVqYjz1ds8rz/ave3sju1Yvus5cXkcoFHfPa4KDE8f+juCnbLTPJ+dO1h83Upy/52uwm5/t3KoJNHUqxZEZYUrd84OdaQnBqYh0U8boIobYMdfR4sdD4kl4JAvrFKOZtsKBiAxYxzi8xjPCfs6aPkZDzNHsg88fbfF16sKmizcpNo7Z8IUvIqQ9eIvKtPvndBEKJkwQoYtO8BBXqyHoIiIjPLHdIYnR7Mxx2PT/eyd4/+Atwhlz3/rckZQuoLjZqY31uWxjIEXJXVhj427tIhDUhTvumvVzZhU0t3S+eOpzXYnNa6v1NJIXQea9r9IpOYIjHVoay6GY+LT7uP/EPrUfXl8/T32cV55r6nxh6Xx+PaMmmBQMiBNNDksOUkF3h+TFdZo1ms2WNpfUhYLbrPDsSHq/pCnThEaQ3ixt+P8IiCNbQI3purl8mUs/s7wIB88R0gUzrDQCbwcf9GfarPRsLqjrNNwVOwZgymRw8Ov98sv/dK1lm4kN3cQfWosqKeSLCK+HdtLKLpIw5fOiK9/Mol6mO152Trw8EHwBzyspSvoyE1Q6GXXh/htSaQpEu+Z+bR4Kiq0ywqFYSXiMkF8StDaJdEoG3rKVPc9Wb91vipxjhLigCB65PasoCMU3nFTMdXZxrfch80rP9qrwRf32qwjNUgJgRVRJjPbnC18nL5O1JQU8j9EsX8mL4BKW31AoohzmntCWEfa1GJE0MtYPnqtiEjartu3BjCBH+90SyVpZ/EpQoGEgjsSFXwfJ8YU4q3hT18zZuW2rBZWFKpktM8KaIKfChlVTEAhmBKerHF3ziEFwopKUMQq0TvKqdYfbCBGN4L7NdeX14LmXEzGe2hKiyDnft4+70qw5ukzKMALmeQPNlLqXLXfNLXVFwx9v7cSzLY02h2SwmZ0PyfV/H209UmCe+xJpfwhOdwRbWqupPNCYhCnFT23uY7Ruf1/Lt82zf2bS3BEcNNTwG5WnO1s1qptljANt8LWzRZWSyssopvnTn/XjFnMJJTX8rnUs1Aho66jPFakp0B48ty/d89Bqf7o9M64+L/tWrzE1op3OBwy6FvBZ3u7tjXEo6OtHPyZndnqrIjy/G9937XNx1fetTUrnyMAl5dfRP5/Y1ep6PY6dnrKFdWun1+etLjt9jmNtz9ua6Ed7+/f4HM+2VJ0eud2Pw8wqKWB3e0PQ1r4RaMQVx+fMargbFjSZnkMwETRfn7gbnJ8zS4uM+Sf7wD1TvqHwfGeLOvtEXHMXA6jGz56C4P1ag2Nfa7MmKCyYbjbUNQ3l13Nmdc++4//KCGYMXE6k1aRoRnHodIZMQKDZjtKxTTGAs+fM0hmAW2pATeX7B97bRzhMGfhz76zkLKUINqo3Dxpoxx7ST1zCbbt+UpP9eHYdLcKB4PbsqMbZfxotprHsNHL7DPfsNrJHmtCSIc/j2BBXNKO9upydr+83T7e/jwSHUuX+hgipxhfWrwyYt/Pdn5pgB1OOwmKEDukmhzVBnR7jBbxhpxuU1IB7IhU2hvXmFvecAjOnLjxz5DcceTdUAY6MwS9/MVYFJ7YY9GNJJs3qY78JT1PeaPuU+qzm2sf85yOMd0dz+XMEeydYNMonsi6M2IWqNyiM8rkLK1YnYqNXt/PtWW9fwLcv+8TPohbXXlWPx8E5QoFY2vzEcXjmeJ2v7HlO0CVlRlhERiAi7O/iDsW6Oq9ZqWzzQZUSWaV+fU58pRcjUPqf4JncVO8Z+w6gvSKoMSl2PZDtKFJwogu+8IsI9ODWMX+C6ivoNASHOlMOudrxmrhHXUbo0txB0bBWDZ4zdR2N/B7YST2l0skJ0lVDPVJk7smr/nWFFZeWVc/KvwV2t/1xWHN0B4FqICiAqmdl27Qf9T1Bvk5qn9vSqbkrZNQN5lYArDYrgsM6RkEPataJGeHkHtsJpGtMk3vk5G6ln8gClureRmyz2eBAQcRxg/TrvUmBg4o3wWt191SA2g7NlVyfVONWwlrloqL2bB/jYACZJcvbfR/6IvyKBxB4PSQbQ5sAYYxme4KToQC0DbbU4Bt8bLoCE8dYQgUOPYTqW7bwTxGADQNmPRQlBbNtBMwjkZeiA1txZADW3DFTkoeC3QNqr7ceGmqlHYre/kghD8Uz2NPbqafs8e7O/h0Y+gRAdtSwZhg4JXU+Ab3u7NuZo4H10ffZiU6GviU2biDYYv3gOWV2hai6rYuiB9CrVrtLzQQoM7y+s9UITxhcCmasp8NzALVRjiYg8EN+XHOYEUROTL791TDZN2l/OwwBbPCCu+mWSZgTnF4vHj00tBoeLh1KZzj7+r4CfR8znva/Oez/VUBwr5fCTyGhvSyu/x2gDLccCC12FABysVnM/xg4L3xV45vlo7n+1gHNf6+ysvgYo+6nPePxkPhj+LG1hLUXYF1Mwau/mEzW3i5Bqn+ri+BtkOppDOA2wn4d3UNgr5Frn7jB40grg82BuMass62iBhEYtEAxtqPP0+i9+Q8A5R2Y/BCE/pl3BnD9C6GZ40BgcAWc5dBvhrUk908qL3f67/M7Zojf7OuTObqS37vIXmfkBCivPQbAkqBAU5f1UX1M5/MfArbpd2YvSnf46eDjik786b4kausRY230Xdoozi9JPlyOOgiDZYQDTc1Sgpw6Akm+am24gZtn8/cIkffopf27pIArlmhimxV28vPPX/7xX/8MAAD//3jh/kY=") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml index 96cb960c119..f1ad8c1c646 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml @@ -1,3 +1,4 @@ +revision: 5 fleet: agent: id: fleet-agent-id diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml index 5e0e23276a5..38e06698393 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml @@ -1,3 +1,4 @@ +revision: 5 name: Endpoint Host fleet: agent: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_no_fleet.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_no_fleet.yml index 18401859458..6a7aea5099d 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_no_fleet.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_no_fleet.yml @@ -1,5 +1,5 @@ name: Endpoint Host - +revision: 5 outputs: default: type: elasticsearch diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_unknown_output.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_unknown_output.yml index 4272c418560..7c813b3732d 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_unknown_output.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_unknown_output.yml @@ -1,4 +1,5 @@ name: Endpoint Host +revision: 5 fleet: agent: id: fleet-agent-id diff --git a/x-pack/elastic-agent/pkg/agent/storage/storage.go b/x-pack/elastic-agent/pkg/agent/storage/storage.go index 0c8ff360b87..b37c6e06ab3 100644 --- a/x-pack/elastic-agent/pkg/agent/storage/storage.go +++ b/x-pack/elastic-agent/pkg/agent/storage/storage.go @@ -12,6 +12,8 @@ import ( "os" "time" + "github.com/hectane/go-acl" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" ) @@ -129,6 +131,13 @@ func (r *ReplaceOnSuccessStore) Save(in io.Reader) error { } } + if err := acl.Chmod(r.target, perms); err != nil { + return errors.New(err, + fmt.Sprintf("could not set permissions target file %s", r.target), + errors.TypeFilesystem, + errors.M(errors.MetaKeyPath, r.target)) + } + return nil } @@ -178,6 +187,13 @@ func (d *DiskStore) Save(in io.Reader) error { errors.M(errors.MetaKeyPath, d.target)) } + if err := acl.Chmod(d.target, perms); err != nil { + return errors.New(err, + fmt.Sprintf("could not set permissions target file %s", d.target), + errors.TypeFilesystem, + errors.M(errors.MetaKeyPath, d.target)) + } + return nil } diff --git a/x-pack/elastic-agent/pkg/agent/storage/storage_test.go b/x-pack/elastic-agent/pkg/agent/storage/storage_test.go index 3d53f43cdb6..52cd1960b22 100644 --- a/x-pack/elastic-agent/pkg/agent/storage/storage_test.go +++ b/x-pack/elastic-agent/pkg/agent/storage/storage_test.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "testing" "time" @@ -49,11 +50,7 @@ func TestReplaceOrRollbackStore(t *testing.T) { require.True(t, bytes.Equal(writtenContent, replaceWith)) requireFilesCount(t, dir, 2) - - info, err := os.Stat(target) - require.NoError(t, err) - - require.Equal(t, perms, info.Mode()) + checkPerms(t, target, perms) }) t.Run("when save is not successful", func(t *testing.T) { @@ -103,10 +100,6 @@ func TestReplaceOrRollbackStore(t *testing.T) { require.True(t, bytes.Equal(writtenContent, replaceWith)) requireFilesCount(t, dir, 1) - info, err := os.Stat(target) - require.NoError(t, err) - - require.Equal(t, perms, info.Mode()) }) t.Run("when target file do not exist", func(t *testing.T) { @@ -135,11 +128,7 @@ func TestDiskStore(t *testing.T) { require.NoError(t, err) require.Equal(t, msg, content) - - info, err := os.Stat(target) - require.NoError(t, err) - - require.Equal(t, perms, info.Mode()) + checkPerms(t, target, perms) }) t.Run("when the target do no exist", func(t *testing.T) { @@ -158,11 +147,7 @@ func TestDiskStore(t *testing.T) { require.NoError(t, err) require.Equal(t, msg, content) - - info, err := os.Stat(target) - require.NoError(t, err) - - require.Equal(t, perms, info.Mode()) + checkPerms(t, target, perms) }) t.Run("return an io.ReadCloser to the target file", func(t *testing.T) { @@ -178,11 +163,7 @@ func TestDiskStore(t *testing.T) { content, err := ioutil.ReadAll(r) require.NoError(t, err) require.Equal(t, msg, content) - - info, err := os.Stat(target) - require.NoError(t, err) - - require.Equal(t, perms, info.Mode()) + checkPerms(t, target, perms) }) } @@ -210,3 +191,14 @@ func requireFilesCount(t *testing.T, dir string, l int) { require.NoError(t, err) require.Equal(t, l, len(files)) } + +func checkPerms(t *testing.T, target string, expected os.FileMode) { + t.Helper() + if runtime.GOOS == "windows" { + // Windows API validation of ACL is skipped, as its very complicated. + return + } + info, err := os.Stat(target) + require.NoError(t, err) + require.Equal(t, expected, info.Mode()) +} diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index cc6c4c83895..cdd90b3a190 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -15,12 +15,10 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/operation/config" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/app" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/process" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/retry" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/server" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/tokenbucket" @@ -52,9 +50,7 @@ type Application struct { monitor monitoring.Monitor - processConfig *process.Config - downloadConfig *artifact.Config - retryConfig *retry.Config + processConfig *process.Config logger *logger.Logger @@ -84,22 +80,20 @@ func NewApplication( b, _ := tokenbucket.NewTokenBucket(ctx, 3, 3, 1*time.Second) return &Application{ - bgContext: ctx, - id: id, - name: appName, - pipelineID: pipelineID, - logLevel: logLevel, - spec: spec, - srv: srv, - processConfig: cfg.ProcessConfig, - downloadConfig: cfg.DownloadConfig, - retryConfig: cfg.RetryConfig, - logger: logger, - limiter: b, - reporter: reporter, - monitor: monitor, - uid: uid, - gid: gid, + bgContext: ctx, + id: id, + name: appName, + pipelineID: pipelineID, + logLevel: logLevel, + spec: spec, + srv: srv, + processConfig: cfg.ProcessConfig, + logger: logger, + limiter: b, + reporter: reporter, + monitor: monitor, + uid: uid, + gid: gid, }, nil } diff --git a/x-pack/elastic-agent/pkg/core/plugin/service/app.go b/x-pack/elastic-agent/pkg/core/plugin/service/app.go index 3c8a42a0db8..0832312053a 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/service/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/service/app.go @@ -18,12 +18,10 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/operation/config" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/app" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/process" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/retry" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/server" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/tokenbucket" @@ -55,9 +53,7 @@ type Application struct { monitor monitoring.Monitor - processConfig *process.Config - downloadConfig *artifact.Config - retryConfig *retry.Config + processConfig *process.Config logger *logger.Logger @@ -88,23 +84,21 @@ func NewApplication( b, _ := tokenbucket.NewTokenBucket(ctx, 3, 3, 1*time.Second) return &Application{ - bgContext: ctx, - id: id, - name: appName, - pipelineID: pipelineID, - logLevel: logLevel, - spec: spec, - srv: srv, - processConfig: cfg.ProcessConfig, - downloadConfig: cfg.DownloadConfig, - retryConfig: cfg.RetryConfig, - logger: logger, - limiter: b, - reporter: reporter, - monitor: monitor, - uid: uid, - gid: gid, - credsPort: credsPort, + bgContext: ctx, + id: id, + name: appName, + pipelineID: pipelineID, + logLevel: logLevel, + spec: spec, + srv: srv, + processConfig: cfg.ProcessConfig, + logger: logger, + limiter: b, + reporter: reporter, + monitor: monitor, + uid: uid, + gid: gid, + credsPort: credsPort, }, nil } diff --git a/x-pack/elastic-agent/pkg/kibana/client.go b/x-pack/elastic-agent/pkg/kibana/client.go index 5a20d1b077a..7411742d251 100644 --- a/x-pack/elastic-agent/pkg/kibana/client.go +++ b/x-pack/elastic-agent/pkg/kibana/client.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "strings" + "sync" "time" "github.com/pkg/errors" @@ -24,20 +25,19 @@ import ( const ( kibanaPort = 5601 kibanaHTTPSPort = 443 + + kibanaRetryOnBadConnTimeout = 5 * time.Minute ) type requestFunc func(string, string, url.Values, io.Reader) (*http.Request, error) type wrapperFunc func(rt http.RoundTripper) (http.RoundTripper, error) -type clienter interface { - Send( - method string, - path string, - params url.Values, - headers http.Header, - body io.Reader, - ) (*http.Response, error) - Close() error +type requestClient struct { + request requestFunc + client http.Client + lastUsed time.Time + lastErr error + lastErrOcc time.Time } // Client wraps an http.Client and takes care of making the raw calls to kibana, the client should @@ -46,27 +46,11 @@ type clienter interface { // implementations that will take care of the boiler plates. type Client struct { log *logger.Logger - request requestFunc - client http.Client + lock sync.Mutex + clients []*requestClient config *Config } -// New creates new Kibana API client. -func New( - log *logger.Logger, - factory requestFunc, - cfg *Config, - httpClient http.Client, -) (*Client, error) { - c := &Client{ - log: log, - request: factory, - client: httpClient, - config: cfg, - } - return c, nil -} - // NewConfigFromURL returns a Kibana Config based on a received host. func NewConfigFromURL(kURL string) (*Config, error) { u, err := url.Parse(kURL) @@ -112,29 +96,6 @@ func NewWithRawConfig(log *logger.Logger, config *config.Config, wrapper wrapper // NewWithConfig takes a Kibana Config and return a client. func NewWithConfig(log *logger.Logger, cfg *Config, wrapper wrapperFunc) (*Client, error) { - var transport http.RoundTripper - transport, err := makeTransport(cfg.Timeout, cfg.TLS) - if err != nil { - return nil, err - } - - if cfg.IsBasicAuth() { - // Pass basic auth credentials to all the underlying calls. - transport = NewBasicAuthRoundTripper(transport, cfg.Username, cfg.Password) - } - - if wrapper != nil { - transport, err = wrapper(transport) - if err != nil { - return nil, errors.Wrap(err, "fail to create transport client") - } - } - - httpClient := http.Client{ - Transport: transport, - Timeout: cfg.Timeout, - } - // Normalize the URL with the path any spaces configured. var p string if len(cfg.SpaceID) > 0 { @@ -152,22 +113,53 @@ func NewWithConfig(log *logger.Logger, cfg *Config, wrapper wrapperFunc) (*Clien usedDefaultPort = kibanaHTTPSPort } - kibanaURL, err := common.MakeURL(string(cfg.Protocol), p, cfg.Host, usedDefaultPort) - if err != nil { - return nil, errors.Wrap(err, "invalid Kibana endpoint") + hosts := cfg.GetHosts() + clients := make([]*requestClient, len(hosts)) + for i, host := range cfg.GetHosts() { + var transport http.RoundTripper + transport, err := makeTransport(cfg.Timeout, cfg.TLS) + if err != nil { + return nil, err + } + + if cfg.IsBasicAuth() { + // Pass basic auth credentials to all the underlying calls. + transport = NewBasicAuthRoundTripper(transport, cfg.Username, cfg.Password) + } + + if wrapper != nil { + transport, err = wrapper(transport) + if err != nil { + return nil, errors.Wrap(err, "fail to create transport client") + } + } + + httpClient := http.Client{ + Transport: transport, + Timeout: cfg.Timeout, + } + + kibanaURL, err := common.MakeURL(string(cfg.Protocol), p, host, usedDefaultPort) + if err != nil { + return nil, errors.Wrap(err, "invalid Kibana endpoint") + } + clients[i] = &requestClient{ + request: prefixRequestFactory(kibanaURL), + client: httpClient, + } } - return New(log, prefixRequestFactory(kibanaURL), cfg, httpClient) + return new(log, cfg, clients...) } -// Send executes a direct calls agains't the Kibana API, the method will takes cares of cloning +// Send executes a direct calls against the Kibana API, the method will takes cares of cloning // also add necessary headers for Kibana likes: "Content-Type", "Accept", and "kbn-xsrf". -// No assumptions is done on the response concerning the received format, this will be the responsability +// No assumptions is done on the response concerning the received format, this will be the responsibility // of the implementation to correctly unpack any received data. // // NOTE: -// - The caller of this method is free to overrides any values found in the headers. -// - The magic of unpack kibana errors is not done in the Send method, an helper methods is provided. +// - The caller of this method is free to override any value found in the headers. +// - The magic of unpack kibana errors is not done in the Send method, a helper method is provided. func (c *Client) Send( ctx context.Context, method, path string, @@ -176,8 +168,11 @@ func (c *Client) Send( body io.Reader, ) (*http.Response, error) { c.log.Debugf("Request method: %s, path: %s", method, path) + c.lock.Lock() + defer c.lock.Unlock() + requester := c.nextRequester() - req, err := c.request(method, path, params, body) + req, err := requester.request(method, path, params, body) if err != nil { return nil, errors.Wrapf(err, "fail to create HTTP request using method %s to %s", method, path) } @@ -195,12 +190,79 @@ func (c *Client) Send( } } - return c.client.Do(req.WithContext(ctx)) + requester.lastUsed = time.Now().UTC() + resp, err := requester.client.Do(req.WithContext(ctx)) + if err != nil { + requester.lastErr = err + requester.lastErrOcc = time.Now().UTC() + } else { + requester.lastErr = nil + requester.lastErrOcc = time.Time{} + } + return resp, err } // URI returns the remote URI. func (c *Client) URI() string { - return string(c.config.Protocol) + "://" + c.config.Host + "/" + c.config.Path + host := c.config.GetHosts()[0] + return string(c.config.Protocol) + "://" + host + "/" + c.config.Path +} + +// new creates new Kibana API client. +func new( + log *logger.Logger, + cfg *Config, + httpClients ...*requestClient, +) (*Client, error) { + c := &Client{ + log: log, + clients: httpClients, + config: cfg, + } + return c, nil +} + +// nextRequester returns the requester to use. +// +// It excludes clients that have errored in the last 5 minutes. +func (c *Client) nextRequester() *requestClient { + var selected *requestClient + + now := time.Now().UTC() + for _, requester := range c.clients { + if requester.lastErr != nil && now.Sub(requester.lastErrOcc) > kibanaRetryOnBadConnTimeout { + requester.lastErr = nil + requester.lastErrOcc = time.Time{} + } + if requester.lastErr != nil { + continue + } + if requester.lastUsed.IsZero() { + // never been used, instant winner! + selected = requester + break + } + if selected == nil { + selected = requester + continue + } + if requester.lastUsed.Before(selected.lastUsed) { + selected = requester + } + } + if selected == nil { + // all are erroring; select the oldest one that errored + for _, requester := range c.clients { + if selected == nil { + selected = requester + continue + } + if requester.lastErrOcc.Before(selected.lastErrOcc) { + selected = requester + } + } + } + return selected } func prefixRequestFactory(URL string) requestFunc { diff --git a/x-pack/elastic-agent/pkg/kibana/client_test.go b/x-pack/elastic-agent/pkg/kibana/client_test.go index 818b9a5ea2f..36d24b88a00 100644 --- a/x-pack/elastic-agent/pkg/kibana/client_test.go +++ b/x-pack/elastic-agent/pkg/kibana/client_test.go @@ -14,6 +14,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -57,7 +58,7 @@ func TestPortDefaults(t *testing.T) { c, err := NewWithConfig(l, cfg, nil) require.NoError(t, err) - r, err := c.request("GET", "/", nil, strings.NewReader("")) + r, err := c.nextRequester().request("GET", "/", nil, strings.NewReader("")) require.NoError(t, err) assert.True(t, strings.HasSuffix(r.Host, fmt.Sprintf(":%d", tc.ExpectedPort))) @@ -303,6 +304,90 @@ func TestHTTPClient(t *testing.T) { )) } +func TestNextRequester(t *testing.T) { + t.Run("Picks first requester on initial call", func(t *testing.T) { + one := &requestClient{} + two := &requestClient{} + client, err := new(nil, nil, one, two) + require.NoError(t, err) + assert.Equal(t, one, client.nextRequester()) + }) + + t.Run("Picks second requester when first has error", func(t *testing.T) { + one := &requestClient{ + lastErr: fmt.Errorf("fake error"), + lastErrOcc: time.Now().UTC(), + } + two := &requestClient{} + client, err := new(nil, nil, one, two) + require.NoError(t, err) + assert.Equal(t, two, client.nextRequester()) + }) + + t.Run("Picks second requester when first has used", func(t *testing.T) { + one := &requestClient{ + lastUsed: time.Now().UTC(), + } + two := &requestClient{} + client, err := new(nil, nil, one, two) + require.NoError(t, err) + assert.Equal(t, two, client.nextRequester()) + }) + + t.Run("Picks second requester when its oldest", func(t *testing.T) { + one := &requestClient{ + lastUsed: time.Now().UTC().Add(-time.Minute), + } + two := &requestClient{ + lastUsed: time.Now().UTC().Add(-3 * time.Minute), + } + three := &requestClient{ + lastUsed: time.Now().UTC().Add(-2 * time.Minute), + } + client, err := new(nil, nil, one, two, three) + require.NoError(t, err) + assert.Equal(t, two, client.nextRequester()) + }) + + t.Run("Picks third requester when its second has error and first is last used", func(t *testing.T) { + one := &requestClient{ + lastUsed: time.Now().UTC().Add(-time.Minute), + } + two := &requestClient{ + lastUsed: time.Now().UTC().Add(-3 * time.Minute), + lastErr: fmt.Errorf("fake error"), + lastErrOcc: time.Now().Add(-time.Minute), + } + three := &requestClient{ + lastUsed: time.Now().UTC().Add(-2 * time.Minute), + } + client, err := new(nil, nil, one, two, three) + require.NoError(t, err) + assert.Equal(t, three, client.nextRequester()) + }) + + t.Run("Picks second requester when its oldest and all have old errors", func(t *testing.T) { + one := &requestClient{ + lastUsed: time.Now().UTC().Add(-time.Minute), + lastErr: fmt.Errorf("fake error"), + lastErrOcc: time.Now().Add(-time.Minute), + } + two := &requestClient{ + lastUsed: time.Now().UTC().Add(-3 * time.Minute), + lastErr: fmt.Errorf("fake error"), + lastErrOcc: time.Now().Add(-3 * time.Minute), + } + three := &requestClient{ + lastUsed: time.Now().UTC().Add(-2 * time.Minute), + lastErr: fmt.Errorf("fake error"), + lastErrOcc: time.Now().Add(-2 * time.Minute), + } + client, err := new(nil, nil, one, two, three) + require.NoError(t, err) + assert.Equal(t, two, client.nextRequester()) + }) +} + func withServer(m func(t *testing.T) *http.ServeMux, test func(t *testing.T, host string)) func(t *testing.T) { return func(t *testing.T) { s := httptest.NewServer(m(t)) diff --git a/x-pack/elastic-agent/pkg/kibana/config.go b/x-pack/elastic-agent/pkg/kibana/config.go index 14182898bc5..928dff7106d 100644 --- a/x-pack/elastic-agent/pkg/kibana/config.go +++ b/x-pack/elastic-agent/pkg/kibana/config.go @@ -19,6 +19,7 @@ type Config struct { Password string `config:"password" yaml:"password,omitempty"` Path string `config:"path" yaml:"path,omitempty"` Host string `config:"host" yaml:"host,omitempty"` + Hosts []string `config:"hosts" yaml:"hosts,omitempty"` Timeout time.Duration `config:"timeout" yaml:"timeout,omitempty"` TLS *tlscommon.Config `config:"ssl" yaml:"ssl,omitempty"` } @@ -53,3 +54,13 @@ func defaultClientConfig() Config { func (c *Config) IsBasicAuth() bool { return len(c.Username) > 0 && len(c.Password) > 0 } + +// GetHosts returns the hosts to connect to kibana. +// +// This looks first at `Hosts` and then at `Host` when `Hosts` is not defined. +func (c *Config) GetHosts() []string { + if len(c.Hosts) > 0 { + return c.Hosts + } + return []string{c.Host} +} diff --git a/x-pack/elastic-agent/spec/endpoint.yml b/x-pack/elastic-agent/spec/endpoint.yml index d77be321654..8f52a3ac87a 100644 --- a/x-pack/elastic-agent/spec/endpoint.yml +++ b/x-pack/elastic-agent/spec/endpoint.yml @@ -43,5 +43,6 @@ rules: - fleet - inputs - output + - revision when: HasAny('fleet') && HasItems(%{[inputs]}) && HasNamespace('output', 'elasticsearch') diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index ae739d07510..7d20d33952d 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -10,7 +10,6 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awscloudwatch" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" - _ "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/googlepubsub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" diff --git a/x-pack/filebeat/input/cloudfoundry/input.go b/x-pack/filebeat/input/cloudfoundry/input.go index fc38d85fc34..036a61b9d1e 100644 --- a/x-pack/filebeat/input/cloudfoundry/input.go +++ b/x-pack/filebeat/input/cloudfoundry/input.go @@ -6,49 +6,51 @@ package cloudfoundry import ( "fmt" + "time" - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry" ) -func init() { - err := input.Register("cloudfoundry", NewInput) - if err != nil { - panic(err) - } +type cloudfoundryEvent interface { + Timestamp() time.Time + ToFields() common.MapStr } -// NewInput creates a new udp input -func NewInput( - cfg *common.Config, - outlet channel.Connector, - context input.Context, -) (input.Input, error) { - cfgwarn.Beta("The cloudfoundry input is beta") - - log := logp.NewLogger("cloudfoundry") - - out, err := outlet.Connect(cfg) - if err != nil { - return nil, err +func Plugin() v2.Plugin { + return v2.Plugin{ + Name: "cloudfoundry", + Stability: feature.Beta, + Deprecated: false, + Info: "collect logs from cloudfoundry loggregator", + Manager: stateless.NewInputManager(configure), } +} - var conf cloudfoundry.Config - if err = cfg.Unpack(&conf); err != nil { +func configure(cfg *common.Config) (stateless.Input, error) { + config := cloudfoundry.Config{} + if err := cfg.Unpack(&config); err != nil { return nil, err } - switch conf.Version { + switch config.Version { case cloudfoundry.ConsumerVersionV1: - return newInputV1(log, conf, out, context) + return configureV1(config) case cloudfoundry.ConsumerVersionV2: - return newInputV2(log, conf, out, context) + return configureV2(config) default: - return nil, fmt.Errorf("not supported consumer version: %s", conf.Version) + return nil, fmt.Errorf("not supported consumer version: %s", config.Version) + } +} + +func createEvent(evt cloudfoundryEvent) beat.Event { + return beat.Event{ + Timestamp: evt.Timestamp(), + Fields: evt.ToFields(), } } diff --git a/x-pack/filebeat/input/cloudfoundry/input_integration_test.go b/x-pack/filebeat/input/cloudfoundry/input_integration_test.go index e989c721631..ac53db9944c 100644 --- a/x-pack/filebeat/input/cloudfoundry/input_integration_test.go +++ b/x-pack/filebeat/input/cloudfoundry/input_integration_test.go @@ -11,16 +11,17 @@ import ( "context" "crypto/tls" "net/http" + "sync" "testing" "time" "github.com/stretchr/testify/require" - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" + pubtest "github.com/elastic/beats/v7/libbeat/publisher/testing" cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test" ) @@ -40,66 +41,53 @@ func testInput(t *testing.T, version string) { config := common.MustNewConfigFrom(cftest.GetConfigFromEnv(t)) config.SetString("version", -1, version) + input, err := Plugin().Manager.Create(config) + require.NoError(t, err) + + var wg sync.WaitGroup + defer wg.Wait() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() + // Ensure that there is something happening in the firehose apiAddress, err := config.String("api_address", -1) require.NoError(t, err) - - // Ensure that there is something happening in the firehose go makeApiRequests(t, ctx, apiAddress) - events := make(chan beat.Event) - connector := channel.ConnectorFunc(func(*common.Config, beat.ClientConfig) (channel.Outleter, error) { - return newOutleter(events), nil - }) - - inputCtx := input.Context{Done: make(chan struct{})} + ch := make(chan beat.Event) + client := &pubtest.FakeClient{ + PublishFunc: func(evt beat.Event) { + if ctx.Err() != nil { + return + } + + select { + case ch <- evt: + case <-ctx.Done(): + } + }, + } - input, err := NewInput(config, connector, inputCtx) - require.NoError(t, err) + wg.Add(1) + go func() { + defer wg.Done() - go input.Run() - defer input.Stop() + inputCtx := v2.Context{ + Logger: logp.NewLogger("test"), + Cancelation: ctx, + } + input.Run(inputCtx, pubtest.ConstClient(client)) + }() select { - case e := <-events: + case e := <-ch: t.Logf("Event received: %+v", e) case <-time.After(10 * time.Second): t.Fatal("timeout waiting for events") } } -type outleter struct { - events chan<- beat.Event - done chan struct{} -} - -func newOutleter(events chan<- beat.Event) *outleter { - return &outleter{ - events: events, - done: make(chan struct{}), - } -} - -func (o *outleter) Close() error { - close(o.done) - return nil -} - -func (o *outleter) Done() <-chan struct{} { - return o.done -} - -func (o *outleter) OnEvent(e beat.Event) bool { - select { - case o.events <- e: - return true - default: - return false - } -} - func makeApiRequests(t *testing.T, ctx context.Context, address string) { client := &http.Client{ Transport: &http.Transport{ diff --git a/x-pack/filebeat/input/cloudfoundry/v1.go b/x-pack/filebeat/input/cloudfoundry/v1.go index c1248545369..ffc5ccd4a75 100644 --- a/x-pack/filebeat/input/cloudfoundry/v1.go +++ b/x-pack/filebeat/input/cloudfoundry/v1.go @@ -5,83 +5,59 @@ package cloudfoundry import ( - "sync" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/harvester" - "github.com/elastic/beats/v7/filebeat/input" - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/logp" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry" + "github.com/elastic/go-concert/ctxtool" ) -// InputV1 defines a udp input to receive event on a specific host:port. -type InputV1 struct { - sync.Mutex - consumer *cloudfoundry.DopplerConsumer - started bool - log *logp.Logger - outlet channel.Outleter +// inputV1 defines a udp input to receive event on a specific host:port. +type inputV1 struct { + config cloudfoundry.Config +} + +func configureV1(config cloudfoundry.Config) (*inputV1, error) { + return &inputV1{config: config}, nil } -func newInputV1(log *logp.Logger, conf cloudfoundry.Config, out channel.Outleter, context input.Context) (*InputV1, error) { - hub := cloudfoundry.NewHub(&conf, "filebeat", log) - forwarder := harvester.NewForwarder(out) +func (i *inputV1) Name() string { return "cloudfoundry-v1" } + +func (i *inputV1) Test(ctx v2.TestContext) error { + hub := cloudfoundry.NewHub(&i.config, "filebeat", ctx.Logger) + _, err := hub.Client() + return err +} + +func (i *inputV1) Run(ctx v2.Context, publisher stateless.Publisher) error { + log := ctx.Logger + hub := cloudfoundry.NewHub(&i.config, "filebeat", log) + + log.Info("Starting cloudfoundry input") + defer log.Info("Stopped cloudfoundry input") callbacks := cloudfoundry.DopplerCallbacks{ Log: func(evt cloudfoundry.Event) { - forwarder.Send(beat.Event{ - Timestamp: evt.Timestamp(), - Fields: evt.ToFields(), - }) + publisher.Publish(createEvent(evt)) }, Error: func(evt cloudfoundry.EventError) { - forwarder.Send(beat.Event{ - Timestamp: evt.Timestamp(), - Fields: evt.ToFields(), - }) + publisher.Publish(createEvent(&evt)) }, } consumer, err := hub.DopplerConsumer(callbacks) if err != nil { - return nil, errors.Wrapf(err, "initializing doppler consumer") + return errors.Wrapf(err, "initializing doppler consumer") } - return &InputV1{ - outlet: out, - consumer: consumer, - started: false, - log: log, - }, nil -} - -// Run starts the consumer of cloudfoundry events -func (p *InputV1) Run() { - p.Lock() - defer p.Unlock() - if !p.started { - p.log.Info("starting cloudfoundry input") - p.consumer.Run() - p.started = true - } -} - -// Stop stops cloudfoundry doppler consumer -func (p *InputV1) Stop() { - defer p.outlet.Close() - p.Lock() - defer p.Unlock() - - p.log.Info("stopping cloudfoundry input") - p.consumer.Stop() - p.started = false -} + stopCtx, cancel := ctxtool.WithFunc(ctxtool.FromCanceller(ctx.Cancelation), func() { + // wait stops the consumer and waits for all internal go-routines to be stopped. + consumer.Wait() + }) + defer cancel() -// Wait waits for the input to finalize, and stops it -func (p *InputV1) Wait() { - p.Stop() - p.consumer.Wait() + consumer.Run() + <-stopCtx.Done() + return nil } diff --git a/x-pack/filebeat/input/cloudfoundry/v2.go b/x-pack/filebeat/input/cloudfoundry/v2.go index 0d21d361f9c..a8257e29a76 100644 --- a/x-pack/filebeat/input/cloudfoundry/v2.go +++ b/x-pack/filebeat/input/cloudfoundry/v2.go @@ -5,86 +5,51 @@ package cloudfoundry import ( - "context" - "sync" - - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/harvester" - "github.com/elastic/beats/v7/filebeat/input" - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/logp" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry" + "github.com/elastic/go-concert/ctxtool" ) -// InputV2 defines a Cloudfoundry input that uses the consumer V2 API -type InputV2 struct { - sync.Mutex - listener *cloudfoundry.RlpListener - started bool - log *logp.Logger - outlet channel.Outleter +// inputV2 defines a Cloudfoundry input that uses the consumer V2 API +type inputV2 struct { + config cloudfoundry.Config +} + +func configureV2(config cloudfoundry.Config) (*inputV2, error) { + return &inputV2{config: config}, nil +} + +func (i *inputV2) Name() string { return "cloudfoundry-v2" } + +func (i *inputV2) Test(ctx v2.TestContext) error { + hub := cloudfoundry.NewHub(&i.config, "filebeat", ctx.Logger) + _, err := hub.Client() + return err } -func newInputV2(log *logp.Logger, conf cloudfoundry.Config, out channel.Outleter, context input.Context) (*InputV2, error) { - hub := cloudfoundry.NewHub(&conf, "filebeat", log) - forwarder := harvester.NewForwarder(out) +func (i *inputV2) Run(ctx v2.Context, publisher stateless.Publisher) error { + log := ctx.Logger + hub := cloudfoundry.NewHub(&i.config, "filebeat", log) + callbacks := cloudfoundry.RlpListenerCallbacks{ HttpAccess: func(evt *cloudfoundry.EventHttpAccess) { - forwarder.Send(beat.Event{ - Timestamp: evt.Timestamp(), - Fields: evt.ToFields(), - }) + publisher.Publish(createEvent(evt)) }, Log: func(evt *cloudfoundry.EventLog) { - forwarder.Send(beat.Event{ - Timestamp: evt.Timestamp(), - Fields: evt.ToFields(), - }) + publisher.Publish(createEvent(evt)) }, Error: func(evt *cloudfoundry.EventError) { - forwarder.Send(beat.Event{ - Timestamp: evt.Timestamp(), - Fields: evt.ToFields(), - }) + publisher.Publish(createEvent(evt)) }, } listener, err := hub.RlpListener(callbacks) if err != nil { - return nil, err + return err } - return &InputV2{ - outlet: out, - listener: listener, - started: false, - log: log, - }, nil -} - -// Run starts the listener of cloudfoundry events -func (p *InputV2) Run() { - p.Lock() - defer p.Unlock() - - if !p.started { - p.log.Info("starting cloudfoundry input") - p.listener.Start(context.TODO()) - p.started = true - } -} - -// Stop stops cloudfoundry listener -func (p *InputV2) Stop() { - defer p.outlet.Close() - p.Lock() - defer p.Unlock() - - p.log.Info("stopping cloudfoundry input") - p.listener.Stop() - p.started = false -} -// Wait waits for the input to finalize, and stops it -func (p *InputV2) Wait() { - p.Stop() + listener.Start(ctxtool.FromCanceller(ctx.Cancelation)) + listener.Wait() + return nil } diff --git a/x-pack/filebeat/input/default-inputs/inputs.go b/x-pack/filebeat/input/default-inputs/inputs.go index 0af51f2c18a..afac3c2e61c 100644 --- a/x-pack/filebeat/input/default-inputs/inputs.go +++ b/x-pack/filebeat/input/default-inputs/inputs.go @@ -10,6 +10,7 @@ import ( v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" ) @@ -23,5 +24,6 @@ func Init(info beat.Info, log *logp.Logger, store beater.StateStore) []v2.Plugin func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2.Plugin { return []v2.Plugin{ o365audit.Plugin(log, store), + cloudfoundry.Plugin(), } } diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index 65c87f850c3..844d810830d 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.10 +FROM golang:1.14.4 RUN \ apt-get update \ diff --git a/x-pack/libbeat/Dockerfile b/x-pack/libbeat/Dockerfile index 19f2ddf5c3d..ac25e26ac2b 100644 --- a/x-pack/libbeat/Dockerfile +++ b/x-pack/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.10 +FROM golang:1.14.4 RUN \ apt-get update \ diff --git a/x-pack/libbeat/common/cloudfoundry/rlplistener.go b/x-pack/libbeat/common/cloudfoundry/rlplistener.go index f542f8eac7e..e80db747c8e 100644 --- a/x-pack/libbeat/common/cloudfoundry/rlplistener.go +++ b/x-pack/libbeat/common/cloudfoundry/rlplistener.go @@ -66,8 +66,8 @@ func (c *RlpListener) Start(ctx context.Context) { } es := rlpClient.Stream(ctx, l) + c.wg.Add(1) go func() { - c.wg.Add(1) defer c.wg.Done() for { select { @@ -110,6 +110,10 @@ func (c *RlpListener) Stop() { c.wg.Wait() } +func (c *RlpListener) Wait() { + c.wg.Wait() +} + // getSelectors returns the server side selectors based on the callbacks defined on the listener. func (c *RlpListener) getSelectors() []*loggregator_v2.Selector { selectors := make([]*loggregator_v2.Selector, 0)