Fix relative paths in prospectors definitions (#5443)

* Fix relative paths in prospectors definitions

If a relative path was used in a prospector definition, it could happen
that when loading the state from the registry, the path didn't match the
pattern, which made it ignore the saved state. Fixes #5442.

Additionally, this PR cleans up some code related to the `recursive_glob`
setting. The docs used to claim that the feature is disabled by default, but
that wasn't the case and it was impossible to disable it. This change leaves
it enabled by default, but makes it possible to disable it.

* addressed comments

CHANGELOG.asciidoc

@@ -52,6 +52,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Fix Filebeat not starting if command line and modules configs are used together. {issue}5376[5376]
 - Add support for adding string tags {pull}5395{5395}
 - Fix race condition when limiting the number of harvesters running in parallel {issue}5458[5458]
+- Fix relative paths in the prospector definitions. {pull}5443[5433]
+- Fix `recursive_globe.enabled` option. {pull}5443[5443]
 
 *Heartbeat*
 

filebeat/docs/filebeat-options.asciidoc

@@ -63,13 +63,15 @@ paths. You can specify one path per line. Each line begins with a dash (-).
 
 [float]
 [[recursive_glob]]
-==== `recursive_glob`
+==== `recursive_glob.enabled`
 
-*`enabled`*:: Enable expanding `**` into recursive glob patterns. With this feature enabled,
-the rightmost `**` in each path is expanded into a fixed
-number of glob patterns. For example: `/foo/**` expands to `/foo`, `/foo/*`,
-`/foo/*/*`, and so on. The feature is disabled by default, and if enabled it expands a single `**`
-into a 8-level deep `*` pattern.
+Enable expanding `**` into recursive glob patterns. With this feature enabled,
+the rightmost `**` in each path is expanded into a fixed number of glob
+patterns. For example: `/foo/**` expands to `/foo`, `/foo/*`, `/foo/*/*`, and so
+on. If enabled it expands a single `**` into a 8-level deep `*` pattern.
+
+This feature is enabled by default, set to `recursive_glob.enabled` to false to
+disable it.
 
 [float]
 ==== `encoding`

filebeat/prospector/log/config.go

@@ -2,6 +2,7 @@ package log
 
 import (
 	"fmt"
+	"path/filepath"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -33,6 +34,7 @@ var (
 		TailFiles:      false,
 		ScanSort:       "",
 		ScanOrder:      "asc",
+		RecursiveGlob:  true,
 
 		// Harvester
 		BufferSize: 16 * humanize.KiByte,
@@ -168,14 +170,15 @@ func (c *config) Validate() error {
 	return nil
 }
 
-func (c *config) resolvePaths() error {
-	var paths []string
+// resolveRecursiveGlobs expands `**` from the globs in multiple patterns
+func (c *config) resolveRecursiveGlobs() error {
 	if !c.RecursiveGlob {
 		logp.Debug("prospector", "recursive glob disabled")
-		paths = c.Paths
-	} else {
-		logp.Debug("prospector", "recursive glob enabled")
+		return nil
 	}
+
+	logp.Debug("prospector", "recursive glob enabled")
+	var paths []string
 	for _, path := range c.Paths {
 		patterns, err := file.GlobPatterns(path, recursiveGlobDepth)
 		if err != nil {
@@ -189,3 +192,17 @@ func (c *config) resolvePaths() error {
 	c.Paths = paths
 	return nil
 }
+
+// normalizeGlobPatterns calls `filepath.Abs` on all the globs from config
+func (c *config) normalizeGlobPatterns() error {
+	var paths []string
+	for _, path := range c.Paths {
+		pathAbs, err := filepath.Abs(path)
+		if err != nil {
+			return fmt.Errorf("Failed to get the absolute path for %s: %v", path, err)
+		}
+		paths = append(paths, pathAbs)
+	}
+	c.Paths = paths
+	return nil
+}

filebeat/prospector/log/prospector.go

@@ -85,9 +85,11 @@ func NewProspector(
 	if err := cfg.Unpack(&p.config); err != nil {
 		return nil, err
 	}
-	if err := p.config.resolvePaths(); err != nil {
-		logp.Err("Failed to resolve paths in config: %+v", err)
-		return nil, err
+	if err := p.config.resolveRecursiveGlobs(); err != nil {
+		return nil, fmt.Errorf("Failed to resolve recursive globs in config: %v", err)
+	}
+	if err := p.config.normalizeGlobPatterns(); err != nil {
+		return nil, fmt.Errorf("Failed to normalize globs patterns: %v", err)
 	}
 
 	// Create empty harvester to check if configs are fine
@@ -115,7 +117,7 @@ func NewProspector(
 // It goes through all states coming from the registry. Only the states which match the glob patterns of
 // the prospector will be loaded and updated. All other states will not be touched.
 func (p *Prospector) loadStates(states []file.State) error {
-	logp.Debug("prospector", "exclude_files: %s", p.config.ExcludeFiles)
+	logp.Debug("prospector", "exclude_files: %s. Number of stats: %d", p.config.ExcludeFiles, len(states))
 
 	for _, state := range states {
 		// Check if state source belongs to this prospector. If yes, update the state.

filebeat/tests/system/config/filebeat.yml.j2

@@ -10,7 +10,7 @@ filebeat.prospectors:
   # Paths that should be crawled and fetched
   {% if path %}paths:
     - {{ path }}{% endif %}
-  {% if recursive_glob %}recursive_glob.enabled: true
+  {% if disable_recursive_glob %}recursive_glob.enabled: false
   {% endif %}
   # Type of the files. Annotated in every documented
   scan_frequency: {{scan_frequency | default("0.1s") }}

filebeat/tests/system/test_prospector.py

@@ -683,8 +683,7 @@ def test_restart_recursive_glob(self):
         """
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/**",
-            scan_frequency="1s",
-            recursive_glob=True,
+            scan_frequency="1s"
         )
 
         testfile_dir = os.path.join(self.working_dir, "log", "some", "other", "subdir")
@@ -715,3 +714,23 @@ def test_restart_recursive_glob(self):
             name="output contains 'entry2'")
 
         filebeat.check_kill_and_wait()
+
+    def test_disable_recursive_glob(self):
+        """
+        Check that the recursive glob can be disabled from the config.
+        """
+        self.render_config_template(
+            path=os.path.abspath(self.working_dir) + "/log/**",
+            scan_frequency="1s",
+            disable_recursive_glob=True,
+        )
+
+        testfile_dir = os.path.join(self.working_dir, "log", "some", "other", "subdir")
+        os.makedirs(testfile_dir)
+        testfile_path = os.path.join(testfile_dir, "input")
+        filebeat = self.start_beat()
+        self.wait_until(
+            lambda: self.log_contains(
+                "recursive glob disabled"),
+            max_timeout=10)
+        filebeat.check_kill_and_wait()