From 82635039f9fabef1faa99cc7ec0e4182cc27289d Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 30 Oct 2018 08:24:03 -0400 Subject: [PATCH] The 'export config' subcommand should display field reference instead of values (#8769) Change the behavior of the export config to not display the values from the keystore or the environment. (cherry picked from commit 47c78ee5d764bfd938fa35b33c42f3afe6db275a) --- CHANGELOG-developer.asciidoc | 1 + CHANGELOG.asciidoc | 4 ++ NOTICE.txt | 9 +++-- libbeat/cmd/export/config.go | 2 + libbeat/cmd/instance/beat.go | 34 +++++++++++++++-- libbeat/cmd/instance/settings.go | 13 ++++--- libbeat/keystore/keystore.go | 11 ------ libbeat/tests/system/beat/beat.py | 24 ++++++++---- libbeat/tests/system/test_cmd.py | 18 +++++++++ libbeat/tests/system/test_keystore.py | 21 ++++++++++ .../github.com/elastic/go-ucfg/CHANGELOG.md | 28 ++++++++++++++ vendor/github.com/elastic/go-ucfg/opts.go | 13 +++++++ vendor/vendor.json | 38 +++++++++++++------ 13 files changed, 173 insertions(+), 43 deletions(-) diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index 6fdef04af5c..6ea407f4d68 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -44,3 +44,4 @@ The list below covers the major changes between 6.3.0 and master only. - Libbeat provides a new function `cmd.GenRootCmdWithSettings` that should be preferred over deprecated functions `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850] - You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449] +- Allow to disable config resolver using the `Settings.DisableConfigResolver` field when initializing libbeat. {pull}8769[8769] \ No newline at end of file diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 4cf76d2f111..08957aee206 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -31,6 +31,10 @@ https://github.com/elastic/beats/compare/v6.4.1...6.4[Check the HEAD diff] - Fix a race condition with the `add_host_metadata` and the event serialization. {pull}8223[8223] {pull}8653[8653] - Fix race condition when publishing monitoring data. {pull}8646[8646] +- Fix in-cluster kubernetes configuration on IPv6. {pull}8754[8754] +- Fix bug in loading dashboards from zip file. {issue}8051[8051] +- The export config subcommand should not display real value for field reference. {pull}xxx[xxx] +- The export config subcommand should not display real value for field reference. {pull}8769[8769] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index ee5c7a94094..4e1d19c45a3 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -447,8 +447,8 @@ Apache License 2.0 -------------------------------------------------------------------- Dependency: github.com/elastic/go-ucfg -Version: v0.6.1 -Revision: 581f7b1fe9d84f4c18ef0694d6e0eb944a925dae +Version: v0.6.5 +Revision: 92d43887f91851c9936621665af7f796f4d03412 License type (autodetected): Apache-2.0 ./vendor/github.com/elastic/go-ucfg/LICENSE: -------------------------------------------------------------------- @@ -2209,6 +2209,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------- Dependency: github.com/stretchr/objx +Revision: b8b73a35e9830ae509858c10dec5866b4d5c8bff License type (autodetected): MIT ./vendor/github.com/stretchr/objx/LICENSE.md: -------------------------------------------------------------------- @@ -2238,8 +2239,8 @@ SOFTWARE. -------------------------------------------------------------------- Dependency: github.com/stretchr/testify -Version: v1.2.0 -Revision: b91bfb9ebec76498946beb6af7c0230c7cc7ba6c +Version: v1.2.2 +Revision: f35b8ab0b5a2cef36673838d662e249dd9c94686 License type (autodetected): MIT ./vendor/github.com/stretchr/testify/LICENSE: -------------------------------------------------------------------- diff --git a/libbeat/cmd/export/config.go b/libbeat/cmd/export/config.go index 60d296a70a8..a678f63c560 100644 --- a/libbeat/cmd/export/config.go +++ b/libbeat/cmd/export/config.go @@ -45,6 +45,8 @@ func exportConfig(settings instance.Settings, name, idxPrefix, beatVersion strin return fmt.Errorf("error initializing beat: %s", err) } + settings.DisableConfigResolver = true + err = b.InitWithSettings(settings) if err != nil { return fmt.Errorf("error initializing beat: %s", err) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index dca68b381d2..4d4fd06c351 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -140,12 +140,13 @@ func init() { // CryptGenRandom is used. func initRand() { n, err := cryptRand.Int(cryptRand.Reader, big.NewInt(math.MaxInt64)) - seed := n.Int64() + var seed int64 if err != nil { // fallback to current timestamp seed = time.Now().UnixNano() + } else { + seed = n.Int64() } - rand.Seed(seed) } @@ -520,8 +521,13 @@ func (b *Beat) configure(settings Settings) error { return fmt.Errorf("could not initialize the keystore: %v", err) } - // TODO: Allow the options to be more flexible for dynamic changes - common.OverwriteConfigOpts(keystore.ConfigOpts(store)) + if settings.DisableConfigResolver { + common.OverwriteConfigOpts(obfuscateConfigOpts()) + } else { + // TODO: Allow the options to be more flexible for dynamic changes + common.OverwriteConfigOpts(configOpts(store)) + } + b.keystore = store err = cloudid.OverwriteSettings(cfg) if err != nil { @@ -828,3 +834,23 @@ func logSystemInfo(info beat.Info) { } } } + +// configOpts returns ucfg config options with a resolver linked to the current keystore. +// TODO: Refactor to allow insert into the config option array without having to redefine everything +func configOpts(store keystore.Keystore) []ucfg.Option { + return []ucfg.Option{ + ucfg.PathSep("."), + ucfg.Resolve(keystore.ResolverWrap(store)), + ucfg.ResolveEnv, + ucfg.VarExp, + } +} + +// obfuscateConfigOpts disables any resolvers in the configuration, instead we return the field +// reference string directly. +func obfuscateConfigOpts() []ucfg.Option { + return []ucfg.Option{ + ucfg.PathSep("."), + ucfg.ResolveNOOP, + } +} diff --git a/libbeat/cmd/instance/settings.go b/libbeat/cmd/instance/settings.go index 09c1180a56f..8275282daad 100644 --- a/libbeat/cmd/instance/settings.go +++ b/libbeat/cmd/instance/settings.go @@ -26,10 +26,11 @@ import ( // Settings contains basic settings for any beat to pass into GenRootCmd type Settings struct { - Name string - IndexPrefix string - Version string - Monitoring report.Settings - RunFlags *pflag.FlagSet - ConfigOverrides *common.Config + Name string + IndexPrefix string + Version string + Monitoring report.Settings + RunFlags *pflag.FlagSet + ConfigOverrides *common.Config + DisableConfigResolver bool } diff --git a/libbeat/keystore/keystore.go b/libbeat/keystore/keystore.go index b7ec9820834..eb95c3e041d 100644 --- a/libbeat/keystore/keystore.go +++ b/libbeat/keystore/keystore.go @@ -120,14 +120,3 @@ func ResolverWrap(keystore Keystore) func(string) (string, error) { return string(v), nil } } - -// ConfigOpts returns ucfg config options with a resolver linked to the current keystore. -// TODO: Refactor to allow insert into the config option array without having to redefine everything -func ConfigOpts(keystore Keystore) []ucfg.Option { - return []ucfg.Option{ - ucfg.PathSep("."), - ucfg.Resolve(ResolverWrap(keystore)), - ucfg.ResolveEnv, - ucfg.VarExp, - } -} diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 1626e6476fc..4958accc6ca 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -35,21 +35,27 @@ class Proc(object): the object gets collected. """ - def __init__(self, args, outputfile): + def __init__(self, args, outputfile, env={}): self.args = args self.output = open(outputfile, "ab") self.stdin_read, self.stdin_write = os.pipe() + self.env = env def start(self): if sys.platform.startswith("win"): + # ensure that the environment is inherited to the subprocess. + variables = os.environ.copy() + variables = variables.update(self.env) + self.proc = subprocess.Popen( self.args, stdin=self.stdin_read, stdout=self.output, stderr=subprocess.STDOUT, bufsize=0, - creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + env=variables) else: self.proc = subprocess.Popen( self.args, @@ -57,7 +63,9 @@ def start(self): stdout=self.output, stderr=subprocess.STDOUT, bufsize=0, - ) + env=self.env) + # If a "No such file or directory" error points you here, run + # "make metricbeat.test" on metricbeat folder return self.proc def kill(self): @@ -140,7 +148,8 @@ def run_beat(self, output=None, logging_args=["-e", "-v", "-d", "*"], extra_args=[], - exit_code=None): + exit_code=None, + env={}): """ Executes beat. Waits for the process to finish before returning to @@ -148,7 +157,7 @@ def run_beat(self, """ proc = self.start_beat(cmd=cmd, config=config, output=output, logging_args=logging_args, - extra_args=extra_args) + extra_args=extra_args, env=env) if exit_code != None: return proc.check_wait(exit_code) @@ -159,7 +168,8 @@ def start_beat(self, config=None, output=None, logging_args=["-e", "-v", "-d", "*"], - extra_args=[]): + extra_args=[], + env={}): """ Starts beat and returns the process handle. The caller is responsible for stopping / waiting for the @@ -190,7 +200,7 @@ def start_beat(self, if extra_args: args.extend(extra_args) - proc = Proc(args, os.path.join(self.working_dir, output)) + proc = Proc(args, os.path.join(self.working_dir, output), env) proc.start() return proc diff --git a/libbeat/tests/system/test_cmd.py b/libbeat/tests/system/test_cmd.py index ee9471b4f72..c657a3cad3c 100644 --- a/libbeat/tests/system/test_cmd.py +++ b/libbeat/tests/system/test_cmd.py @@ -149,6 +149,24 @@ def test_export_config(self): assert self.log_contains("filename: mockbeat") assert self.log_contains("period: 1234") + def test_export_config_environment_variable(self): + """ + Test export config works but doesn"t expose environment variable. + """ + self.render_config_template("mockbeat", + os.path.join(self.working_dir, + "libbeat.yml"), + metrics_period="${METRIC_PERIOD}") + + exit_code = self.run_beat( + logging_args=[], + extra_args=["export", "config"], + config="libbeat.yml", env={'METRIC_PERIOD': '1234'}) + + assert exit_code == 0 + assert self.log_contains("filename: mockbeat") + assert self.log_contains("period: ${METRIC_PERIOD}") + def test_export_template(self): """ Test export template works diff --git a/libbeat/tests/system/test_keystore.py b/libbeat/tests/system/test_keystore.py index 84f5ddb2f5a..7bd1f002e8c 100644 --- a/libbeat/tests/system/test_keystore.py +++ b/libbeat/tests/system/test_keystore.py @@ -70,3 +70,24 @@ def test_keystore_with_nested_key(self): self.wait_until(lambda: self.log_contains("no such host")) assert self.log_contains(secret) proc.check_kill_and_wait() + + def test_export_config_with_keystore(self): + """ + Test export config works and doesn't expose keystore value + """ + key = "asecret" + secret = "asecretvalue" + + self.render_config_template(keystore_path=self.keystore_path, elasticsearch={ + 'hosts': "${%s}" % key + }) + + exit_code = self.run_beat(extra_args=["keystore", "create"]) + assert exit_code == 0 + + self.add_secret(key, value=secret) + exit_code = self.run_beat(extra_args=["export", "config"]) + + assert exit_code == 0 + assert self.log_contains(secret) == False + assert self.log_contains("${%s}" % key) diff --git a/vendor/github.com/elastic/go-ucfg/CHANGELOG.md b/vendor/github.com/elastic/go-ucfg/CHANGELOG.md index 53a0e6169d6..c890e7be47d 100644 --- a/vendor/github.com/elastic/go-ucfg/CHANGELOG.md +++ b/vendor/github.com/elastic/go-ucfg/CHANGELOG.md @@ -14,6 +14,29 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed +## [0.6.5] + +### Added +- Added a NOOP Resolver that will return the key wrapped in the field reference syntax. #122 + +## [0.6.4] + +### Fixed +- Do not treat $ as escape char in plain strings/regexes #120 + +## [0.6.3] + +### Changed +- Remove UUID lib and use pseudo-random IDs instead. #118 + +## [0.6.2] + +### Changed +- New UUID lib: github.com/gofrs/uuid. #116 + +### Fixed +- Fix escape character not removed from escaped string #115 + ## [0.6.1] ### Fixed @@ -203,6 +226,11 @@ This project adheres to [Semantic Versioning](http://semver.org/). [Unreleased]: https://github.com/elastic/go-ucfg/compare/v0.6.1...HEAD +[Unreleased]: https://github.com/elastic/go-ucfg/compare/v0.6.5...HEAD +[0.6.4]: https://github.com/elastic/go-ucfg/compare/v0.6.4...v0.6.5 +[0.6.4]: https://github.com/elastic/go-ucfg/compare/v0.6.3...v0.6.4 +[0.6.3]: https://github.com/elastic/go-ucfg/compare/v0.6.2...v0.6.3 +[0.6.2]: https://github.com/elastic/go-ucfg/compare/v0.6.1...v0.6.2 [0.6.1]: https://github.com/elastic/go-ucfg/compare/v0.6.0...v0.6.1 [0.6.0]: https://github.com/elastic/go-ucfg/compare/v0.5.1...v0.6.0 [0.5.1]: https://github.com/elastic/go-ucfg/compare/v0.5.0...v0.5.1 diff --git a/vendor/github.com/elastic/go-ucfg/opts.go b/vendor/github.com/elastic/go-ucfg/opts.go index 80075d946d9..69937340c77 100644 --- a/vendor/github.com/elastic/go-ucfg/opts.go +++ b/vendor/github.com/elastic/go-ucfg/opts.go @@ -123,6 +123,19 @@ func doResolveEnv(o *options) { }) } +// ResolveNOOP option add a resolver that will not search the value but instead will return the +// provided key wrap with the field reference syntax. This is useful if you don't to expose values +// from envionment variable or other resolvers. +// +// Example: "mysecret" => ${mysecret}" +var ResolveNOOP Option = doResolveNOOP + +func doResolveNOOP(o *options) { + o.resolvers = append(o.resolvers, func(name string) (string, error) { + return "${" + name + "}", nil + }) +} + var ( // ReplacesValues option configures all merging and unpacking operations to // replace old dictionaries and arrays while merging. Value merging can be diff --git a/vendor/vendor.json b/vendor/vendor.json index 90723a01ae9..38f155654dd 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -672,12 +672,12 @@ "versionExact": "v0.0.3" }, { - "checksumSHA1": "MK8/w0Idj7kRBUiBabARPdm9hOo=", + "checksumSHA1": "Yb61Nqnh+3igFci61hv9WYgk/hc=", "path": "github.com/elastic/go-ucfg", - "revision": "581f7b1fe9d84f4c18ef0694d6e0eb944a925dae", - "revisionTime": "2018-07-13T14:04:29Z", - "version": "v0.6.1", - "versionExact": "v0.6.1" + "revision": "92d43887f91851c9936621665af7f796f4d03412", + "revisionTime": "2018-10-26T17:42:06Z", + "version": "v0.6.5", + "versionExact": "v0.6.5" }, { "checksumSHA1": "X+R/CD8SokJrmlxFTx2nSevRDhQ=", @@ -1388,18 +1388,34 @@ "revisionTime": "2017-05-08T18:43:26Z" }, { - "checksumSHA1": "Le1psgZO0t6mRg6oY5dmnjH13hk=", + "checksumSHA1": "XHH8+1ESYdBoyKSD4FXwdy1liUk=", + "path": "github.com/stretchr/objx", + "revision": "b8b73a35e9830ae509858c10dec5866b4d5c8bff", + "revisionTime": "2018-07-02T10:34:55Z" + }, + { + "checksumSHA1": "c6pbpF7eowwO59phRTpF8cQ80Z0=", "path": "github.com/stretchr/testify/assert", - "revision": "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c", - "revisionTime": "2017-12-30T17:54:59Z", - "version": "v1.2.0", - "versionExact": "v1.2.0" + "revision": "f35b8ab0b5a2cef36673838d662e249dd9c94686", + "revisionTime": "2018-05-06T18:05:49Z", + "version": "v1.2.2", + "versionExact": "v1.2.2" + }, + { + "checksumSHA1": "WmBSFdqpdYRIkp0I408JIZ3LDMY=", + "path": "github.com/stretchr/testify/mock", + "revision": "f35b8ab0b5a2cef36673838d662e249dd9c94686", + "revisionTime": "2018-05-06T18:05:49Z", + "version": "v1.2.2", + "versionExact": "v1.2.2" }, { "checksumSHA1": "wnEANt4k5X/KGwoFyfSSnpxULm4=", "path": "github.com/stretchr/testify/require", "revision": "f35b8ab0b5a2cef36673838d662e249dd9c94686", - "revisionTime": "2018-05-06T18:05:49Z" + "revisionTime": "2018-05-06T18:05:49Z", + "version": "v1.2.2", + "versionExact": "v1.2.2" }, { "checksumSHA1": "CpcG17Q/0k1g2uy8AL26Uu7TouU=",