Cherry-pick elastic#8769 to 6.4: The 'export config' subcommand shoul…

…d display field reference instead of values (elastic#8832)

Cherry-pick of PR elastic#8769 to 6.4 branch. Original message:

Change the behavior of the export config to not display the values from
the keystore or the environment.

CHANGELOG-developer.asciidoc

@@ -44,3 +44,4 @@ The list below covers the major changes between 6.3.0 and master only.
 - Libbeat provides a new function `cmd.GenRootCmdWithSettings` that should be preferred over deprecated functions
   `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850]
 - You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]
+- Allow to disable config resolver using the `Settings.DisableConfigResolver` field when initializing libbeat. {pull}8769[8769]

CHANGELOG.asciidoc

@@ -31,6 +31,7 @@ https://github.com/elastic/beats/compare/v6.4.1...6.4[Check the HEAD diff]
 
 - Fix a race condition with the `add_host_metadata` and the event serialization. {pull}8223[8223] {pull}8653[8653]
 - Fix race condition when publishing monitoring data. {pull}8646[8646]
+- The export config subcommand should not display real value for field reference. {pull}8769[8769]
 
 *Auditbeat*
 

NOTICE.txt

@@ -447,8 +447,8 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: github.com/elastic/go-ucfg
-Version: v0.6.1
-Revision: 581f7b1fe9d84f4c18ef0694d6e0eb944a925dae
+Version: v0.6.5
+Revision: 92d43887f91851c9936621665af7f796f4d03412
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/go-ucfg/LICENSE:
 --------------------------------------------------------------------

libbeat/cmd/export/config.go

@@ -45,6 +45,8 @@ func exportConfig(settings instance.Settings, name, idxPrefix, beatVersion strin
 		return fmt.Errorf("error initializing beat: %s", err)
 	}
 
+	settings.DisableConfigResolver = true
+
 	err = b.InitWithSettings(settings)
 	if err != nil {
 		return fmt.Errorf("error initializing beat: %s", err)

libbeat/cmd/instance/beat.go

@@ -61,6 +61,7 @@ import (
 	"github.com/elastic/beats/libbeat/version"
 	"github.com/elastic/go-sysinfo"
 	"github.com/elastic/go-sysinfo/types"
+	ucfg "github.com/elastic/go-ucfg"
 
 	// Register publisher pipeline modules
 	_ "github.com/elastic/beats/libbeat/publisher/includes"
@@ -140,12 +141,13 @@ func init() {
 // CryptGenRandom is used.
 func initRand() {
 	n, err := cryptRand.Int(cryptRand.Reader, big.NewInt(math.MaxInt64))
-	seed := n.Int64()
+	var seed int64
 	if err != nil {
 		// fallback to current timestamp
 		seed = time.Now().UnixNano()
+	} else {
+		seed = n.Int64()
 	}
-
 	rand.Seed(seed)
 }
 
@@ -520,8 +522,13 @@ func (b *Beat) configure(settings Settings) error {
 		return fmt.Errorf("could not initialize the keystore: %v", err)
 	}
 
-	// TODO: Allow the options to be more flexible for dynamic changes
-	common.OverwriteConfigOpts(keystore.ConfigOpts(store))
+	if settings.DisableConfigResolver {
+		common.OverwriteConfigOpts(obfuscateConfigOpts())
+	} else {
+		// TODO: Allow the options to be more flexible for dynamic changes
+		common.OverwriteConfigOpts(configOpts(store))
+	}
+
 	b.keystore = store
 	err = cloudid.OverwriteSettings(cfg)
 	if err != nil {
@@ -828,3 +835,23 @@ func logSystemInfo(info beat.Info) {
 		}
 	}
 }
+
+// configOpts returns ucfg config options with a resolver linked to the current keystore.
+// TODO: Refactor to allow insert into the config option array without having to redefine everything
+func configOpts(store keystore.Keystore) []ucfg.Option {
+	return []ucfg.Option{
+		ucfg.PathSep("."),
+		ucfg.Resolve(keystore.ResolverWrap(store)),
+		ucfg.ResolveEnv,
+		ucfg.VarExp,
+	}
+}
+
+// obfuscateConfigOpts disables any resolvers in the configuration, instead we return the field
+// reference string directly.
+func obfuscateConfigOpts() []ucfg.Option {
+	return []ucfg.Option{
+		ucfg.PathSep("."),
+		ucfg.ResolveNOOP,
+	}
+}

libbeat/cmd/instance/settings.go

@@ -26,10 +26,11 @@ import (
 
 // Settings contains basic settings for any beat to pass into GenRootCmd
 type Settings struct {
-	Name            string
-	IndexPrefix     string
-	Version         string
-	Monitoring      report.Settings
-	RunFlags        *pflag.FlagSet
-	ConfigOverrides *common.Config
+	Name                  string
+	IndexPrefix           string
+	Version               string
+	Monitoring            report.Settings
+	RunFlags              *pflag.FlagSet
+	ConfigOverrides       *common.Config
+	DisableConfigResolver bool
 }

libbeat/keystore/keystore.go

@@ -120,14 +120,3 @@ func ResolverWrap(keystore Keystore) func(string) (string, error) {
 		return string(v), nil
 	}
 }
-
-// ConfigOpts returns ucfg config options with a resolver linked to the current keystore.
-// TODO: Refactor to allow insert into the config option array without having to redefine everything
-func ConfigOpts(keystore Keystore) []ucfg.Option {
-	return []ucfg.Option{
-		ucfg.PathSep("."),
-		ucfg.Resolve(ResolverWrap(keystore)),
-		ucfg.ResolveEnv,
-		ucfg.VarExp,
-	}
-}

libbeat/tests/system/beat/beat.py

@@ -35,29 +35,37 @@ class Proc(object):
     the object gets collected.
     """
 
-    def __init__(self, args, outputfile):
+    def __init__(self, args, outputfile, env={}):
         self.args = args
         self.output = open(outputfile, "ab")
         self.stdin_read, self.stdin_write = os.pipe()
+        self.env = env
 
     def start(self):
 
         if sys.platform.startswith("win"):
+            # ensure that the environment is inherited to the subprocess.
+            variables = os.environ.copy()
+            variables = variables.update(self.env)
+
             self.proc = subprocess.Popen(
                 self.args,
                 stdin=self.stdin_read,
                 stdout=self.output,
                 stderr=subprocess.STDOUT,
                 bufsize=0,
-                creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
+                creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
+                env=variables)
         else:
             self.proc = subprocess.Popen(
                 self.args,
                 stdin=self.stdin_read,
                 stdout=self.output,
                 stderr=subprocess.STDOUT,
                 bufsize=0,
-            )
+                env=self.env)
+            # If a "No such file or directory" error points you here, run
+            # "make metricbeat.test" on metricbeat folder
         return self.proc
 
     def kill(self):
@@ -140,15 +148,16 @@ def run_beat(self,
                  output=None,
                  logging_args=["-e", "-v", "-d", "*"],
                  extra_args=[],
-                 exit_code=None):
+                 exit_code=None,
+                 env={}):
         """
         Executes beat.
         Waits for the process to finish before returning to
         the caller.
         """
         proc = self.start_beat(cmd=cmd, config=config, output=output,
                                logging_args=logging_args,
-                               extra_args=extra_args)
+                               extra_args=extra_args, env=env)
         if exit_code != None:
             return proc.check_wait(exit_code)
 
@@ -159,7 +168,8 @@ def start_beat(self,
                    config=None,
                    output=None,
                    logging_args=["-e", "-v", "-d", "*"],
-                   extra_args=[]):
+                   extra_args=[],
+                   env={}):
         """
         Starts beat and returns the process handle. The
         caller is responsible for stopping / waiting for the
@@ -190,7 +200,7 @@ def start_beat(self,
         if extra_args:
             args.extend(extra_args)
 
-        proc = Proc(args, os.path.join(self.working_dir, output))
+        proc = Proc(args, os.path.join(self.working_dir, output), env)
         proc.start()
         return proc
 

libbeat/tests/system/test_cmd.py

@@ -149,6 +149,24 @@ def test_export_config(self):
         assert self.log_contains("filename: mockbeat")
         assert self.log_contains("period: 1234")
 
+    def test_export_config_environment_variable(self):
+        """
+        Test export config works but doesn"t expose environment variable.
+        """
+        self.render_config_template("mockbeat",
+                                    os.path.join(self.working_dir,
+                                                 "libbeat.yml"),
+                                    metrics_period="${METRIC_PERIOD}")
+
+        exit_code = self.run_beat(
+            logging_args=[],
+            extra_args=["export", "config"],
+            config="libbeat.yml", env={'METRIC_PERIOD': '1234'})
+
+        assert exit_code == 0
+        assert self.log_contains("filename: mockbeat")
+        assert self.log_contains("period: ${METRIC_PERIOD}")
+
     def test_export_template(self):
         """
         Test export template works

libbeat/tests/system/test_keystore.py

@@ -70,3 +70,24 @@ def test_keystore_with_nested_key(self):
         self.wait_until(lambda: self.log_contains("no such host"))
         assert self.log_contains(secret)
         proc.check_kill_and_wait()
+
+    def test_export_config_with_keystore(self):
+        """
+        Test export config works and doesn't expose keystore value
+        """
+        key = "asecret"
+        secret = "asecretvalue"
+
+        self.render_config_template(keystore_path=self.keystore_path, elasticsearch={
+            'hosts': "${%s}" % key
+        })
+
+        exit_code = self.run_beat(extra_args=["keystore", "create"])
+        assert exit_code == 0
+
+        self.add_secret(key, value=secret)
+        exit_code = self.run_beat(extra_args=["export", "config"])
+
+        assert exit_code == 0
+        assert self.log_contains(secret) == False
+        assert self.log_contains("${%s}" % key)

vendor/vendor.json

@@ -672,12 +672,12 @@
 			"versionExact": "v0.0.3"
 		},
 		{
-			"checksumSHA1": "MK8/w0Idj7kRBUiBabARPdm9hOo=",
+			"checksumSHA1": "Yb61Nqnh+3igFci61hv9WYgk/hc=",
 			"path": "github.com/elastic/go-ucfg",
-			"revision": "581f7b1fe9d84f4c18ef0694d6e0eb944a925dae",
-			"revisionTime": "2018-07-13T14:04:29Z",
-			"version": "v0.6.1",
-			"versionExact": "v0.6.1"
+			"revision": "92d43887f91851c9936621665af7f796f4d03412",
+			"revisionTime": "2018-10-26T17:42:06Z",
+			"version": "v0.6.5",
+			"versionExact": "v0.6.5"
 		},
 		{
 			"checksumSHA1": "X+R/CD8SokJrmlxFTx2nSevRDhQ=",