Add double-star mutli-level directory glob matching to filelog receiv…

…er (#97)

docs/operators/file_input.md

@@ -25,6 +25,9 @@ The `file_input` operator reads logs from files. It will place the lines read in
 
 Note that by default, no logs will be read unless the monitored file is actively being written to because `start_at` defaults to `end`.
 
+`include` and `exclude` fields use `github.com/bmatcuk/doublestar` for expression language.
+For reference documentation see [here](https://github.com/bmatcuk/doublestar#patterns).
+
 #### `multiline` configuration
 
 If set, the `multiline` configuration block instructs the `file_input` operator to split log entries on a pattern other than newlines.

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	github.com/antonmedv/expr v1.8.9
+	github.com/bmatcuk/doublestar/v3 v3.0.0
 	github.com/json-iterator/go v1.1.10
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/observiq/ctimefmt v1.0.0

go.sum

@@ -119,6 +119,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkNo5WyHI=
+github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
@@ -879,8 +881,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5/go.mod h1:ppEjwdhyy7Y31EnHRDm1JkChoC7LXIJ7Ex0VYLWtZtQ=
-github.com/vektra/mockery v1.1.2 h1:uc0Yn67rJpjt8U/mAZimdCKn9AeA97BOkjpmtBSlfP4=
-github.com/vektra/mockery v1.1.2/go.mod h1:VcfZjKaFOPO+MpN4ZvwPjs4c48lkq1o3Ym8yHZJu0jU=
 github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM=
 github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/xdg-go/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:FV1RpvYFmF8wnKtr3ArzkC0b+tAySCbw8eP7QSIvLKM=
@@ -1223,7 +1223,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=

operator/builtin/input/file/config.go

@@ -17,11 +17,11 @@ package file
 import (
 	"bufio"
 	"fmt"
-	"path/filepath"
 	"regexp"
 	"strings"
 	"time"
 
+	"github.com/bmatcuk/doublestar/v3"
 	"golang.org/x/text/encoding"
 	"golang.org/x/text/encoding/ianaindex"
 	"golang.org/x/text/encoding/unicode"
@@ -91,15 +91,15 @@ func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator,
 
 	// Ensure includes can be parsed as globs
 	for _, include := range c.Include {
-		_, err := filepath.Match(include, "matchstring")
+		_, err := doublestar.PathMatch(include, "matchstring")
 		if err != nil {
 			return nil, fmt.Errorf("parse include glob: %s", err)
 		}
 	}
 
 	// Ensure excludes can be parsed as globs
 	for _, exclude := range c.Exclude {
-		_, err := filepath.Match(exclude, "matchstring")
+		_, err := doublestar.PathMatch(exclude, "matchstring")
 		if err != nil {
 			return nil, fmt.Errorf("parse exclude glob: %s", err)
 		}

operator/builtin/input/file/config_test.go

@@ -80,6 +80,33 @@ func TestConfig(t *testing.T) {
 				return cfg
 			}(),
 		},
+		{
+			"include_glob_double_asterisk",
+			false,
+			func() *InputConfig {
+				cfg := defaultCfg()
+				cfg.Include = append(cfg.Include, "**.log")
+				return cfg
+			}(),
+		},
+		{
+			"include_glob_double_asterisk_nested",
+			false,
+			func() *InputConfig {
+				cfg := defaultCfg()
+				cfg.Include = append(cfg.Include, "directory/**/*.log")
+				return cfg
+			}(),
+		},
+		{
+			"include_glob_double_asterisk_prefix",
+			false,
+			func() *InputConfig {
+				cfg := defaultCfg()
+				cfg.Include = append(cfg.Include, "**/directory/**/*.log")
+				return cfg
+			}(),
+		},
 		{
 			"include_inline",
 			false,
@@ -124,6 +151,36 @@ func TestConfig(t *testing.T) {
 				return cfg
 			}(),
 		},
+		{
+			"exclude_glob_double_asterisk",
+			false,
+			func() *InputConfig {
+				cfg := defaultCfg()
+				cfg.Include = append(cfg.Include, "*.log")
+				cfg.Exclude = append(cfg.Exclude, "not**.log")
+				return cfg
+			}(),
+		},
+		{
+			"exclude_glob_double_asterisk_nested",
+			false,
+			func() *InputConfig {
+				cfg := defaultCfg()
+				cfg.Include = append(cfg.Include, "*.log")
+				cfg.Exclude = append(cfg.Exclude, "directory/**/not*.log")
+				return cfg
+			}(),
+		},
+		{
+			"exclude_glob_double_asterisk_prefix",
+			false,
+			func() *InputConfig {
+				cfg := defaultCfg()
+				cfg.Include = append(cfg.Include, "*.log")
+				cfg.Exclude = append(cfg.Exclude, "**/directory/**/not*.log")
+				return cfg
+			}(),
+		},
 		{
 			"exclude_inline",
 			false,

operator/builtin/input/file/file.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/bmatcuk/doublestar/v3"
 	"go.uber.org/zap"
 	"golang.org/x/text/encoding"
 
@@ -187,7 +188,7 @@ func getMatches(includes, excludes []string) []string {
 	INCLUDE:
 		for _, match := range matches {
 			for _, exclude := range excludes {
-				if itMatches, _ := filepath.Match(exclude, match); itMatches {
+				if itMatches, _ := doublestar.PathMatch(exclude, match); itMatches {
 					continue INCLUDE
 				}
 			}

operator/builtin/input/file/testdata/exclude_glob_double_asterisk.yaml

@@ -0,0 +1,5 @@
+type: file_input
+include:
+  - "*.log"
+exclude:
+  - "not**.log"

operator/builtin/input/file/testdata/exclude_glob_double_asterisk_nested.yaml

@@ -0,0 +1,5 @@
+type: file_input
+include:
+  - "*.log"
+exclude:
+  - "directory/**/not*.log"

operator/builtin/input/file/testdata/exclude_glob_double_asterisk_prefix.yaml

@@ -0,0 +1,5 @@
+type: file_input
+include:
+  - "*.log"
+exclude:
+  - "**/directory/**/not*.log"

operator/builtin/input/file/testdata/include_glob_double_asterisk.yaml

@@ -0,0 +1,3 @@
+type: file_input
+include:
+  - "**.log"

operator/builtin/input/file/testdata/include_glob_double_asterisk_nested.yaml

@@ -0,0 +1,3 @@
+type: file_input
+include:
+  - "directory/**/*.log"

operator/builtin/input/file/testdata/include_glob_double_asterisk_prefix.yaml

@@ -0,0 +1,3 @@
+type: file_input
+include:
+  - "**/directory/**/*.log"