diff --git a/docs/operators/file_input.md b/docs/operators/file_input.md index 4819d36d..35da08bc 100644 --- a/docs/operators/file_input.md +++ b/docs/operators/file_input.md @@ -25,6 +25,9 @@ The `file_input` operator reads logs from files. It will place the lines read in Note that by default, no logs will be read unless the monitored file is actively being written to because `start_at` defaults to `end`. +`include` and `exclude` fields use `github.com/bmatcuk/doublestar` for expression language. +For reference documentation see [here](https://github.com/bmatcuk/doublestar#patterns). + #### `multiline` configuration If set, the `multiline` configuration block instructs the `file_input` operator to split log entries on a pattern other than newlines. diff --git a/go.mod b/go.mod index 9e03a206..a90f12bf 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/antonmedv/expr v1.8.9 + github.com/bmatcuk/doublestar/v3 v3.0.0 github.com/json-iterator/go v1.1.10 github.com/mitchellh/mapstructure v1.4.1 github.com/observiq/ctimefmt v1.0.0 diff --git a/go.sum b/go.sum index d0a238ab..0d84e468 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkNo5WyHI= +github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= @@ -879,8 +881,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5/go.mod h1:ppEjwdhyy7Y31EnHRDm1JkChoC7LXIJ7Ex0VYLWtZtQ= -github.com/vektra/mockery v1.1.2 h1:uc0Yn67rJpjt8U/mAZimdCKn9AeA97BOkjpmtBSlfP4= -github.com/vektra/mockery v1.1.2/go.mod h1:VcfZjKaFOPO+MpN4ZvwPjs4c48lkq1o3Ym8yHZJu0jU= github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xdg-go/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:FV1RpvYFmF8wnKtr3ArzkC0b+tAySCbw8eP7QSIvLKM= @@ -1223,7 +1223,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/operator/builtin/input/file/config.go b/operator/builtin/input/file/config.go index a3ec7dd7..fbac620d 100644 --- a/operator/builtin/input/file/config.go +++ b/operator/builtin/input/file/config.go @@ -17,11 +17,11 @@ package file import ( "bufio" "fmt" - "path/filepath" "regexp" "strings" "time" + "github.com/bmatcuk/doublestar/v3" "golang.org/x/text/encoding" "golang.org/x/text/encoding/ianaindex" "golang.org/x/text/encoding/unicode" @@ -91,7 +91,7 @@ func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator, // Ensure includes can be parsed as globs for _, include := range c.Include { - _, err := filepath.Match(include, "matchstring") + _, err := doublestar.PathMatch(include, "matchstring") if err != nil { return nil, fmt.Errorf("parse include glob: %s", err) } @@ -99,7 +99,7 @@ func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator, // Ensure excludes can be parsed as globs for _, exclude := range c.Exclude { - _, err := filepath.Match(exclude, "matchstring") + _, err := doublestar.PathMatch(exclude, "matchstring") if err != nil { return nil, fmt.Errorf("parse exclude glob: %s", err) } diff --git a/operator/builtin/input/file/config_test.go b/operator/builtin/input/file/config_test.go index bbc013f1..48ad32c6 100644 --- a/operator/builtin/input/file/config_test.go +++ b/operator/builtin/input/file/config_test.go @@ -80,6 +80,33 @@ func TestConfig(t *testing.T) { return cfg }(), }, + { + "include_glob_double_asterisk", + false, + func() *InputConfig { + cfg := defaultCfg() + cfg.Include = append(cfg.Include, "**.log") + return cfg + }(), + }, + { + "include_glob_double_asterisk_nested", + false, + func() *InputConfig { + cfg := defaultCfg() + cfg.Include = append(cfg.Include, "directory/**/*.log") + return cfg + }(), + }, + { + "include_glob_double_asterisk_prefix", + false, + func() *InputConfig { + cfg := defaultCfg() + cfg.Include = append(cfg.Include, "**/directory/**/*.log") + return cfg + }(), + }, { "include_inline", false, @@ -124,6 +151,36 @@ func TestConfig(t *testing.T) { return cfg }(), }, + { + "exclude_glob_double_asterisk", + false, + func() *InputConfig { + cfg := defaultCfg() + cfg.Include = append(cfg.Include, "*.log") + cfg.Exclude = append(cfg.Exclude, "not**.log") + return cfg + }(), + }, + { + "exclude_glob_double_asterisk_nested", + false, + func() *InputConfig { + cfg := defaultCfg() + cfg.Include = append(cfg.Include, "*.log") + cfg.Exclude = append(cfg.Exclude, "directory/**/not*.log") + return cfg + }(), + }, + { + "exclude_glob_double_asterisk_prefix", + false, + func() *InputConfig { + cfg := defaultCfg() + cfg.Include = append(cfg.Include, "*.log") + cfg.Exclude = append(cfg.Exclude, "**/directory/**/not*.log") + return cfg + }(), + }, { "exclude_inline", false, diff --git a/operator/builtin/input/file/file.go b/operator/builtin/input/file/file.go index 9400bf9f..60bf2fa8 100644 --- a/operator/builtin/input/file/file.go +++ b/operator/builtin/input/file/file.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/bmatcuk/doublestar/v3" "go.uber.org/zap" "golang.org/x/text/encoding" @@ -187,7 +188,7 @@ func getMatches(includes, excludes []string) []string { INCLUDE: for _, match := range matches { for _, exclude := range excludes { - if itMatches, _ := filepath.Match(exclude, match); itMatches { + if itMatches, _ := doublestar.PathMatch(exclude, match); itMatches { continue INCLUDE } } diff --git a/operator/builtin/input/file/testdata/exclude_glob_double_asterisk.yaml b/operator/builtin/input/file/testdata/exclude_glob_double_asterisk.yaml new file mode 100644 index 00000000..554ba10e --- /dev/null +++ b/operator/builtin/input/file/testdata/exclude_glob_double_asterisk.yaml @@ -0,0 +1,5 @@ +type: file_input +include: + - "*.log" +exclude: + - "not**.log" \ No newline at end of file diff --git a/operator/builtin/input/file/testdata/exclude_glob_double_asterisk_nested.yaml b/operator/builtin/input/file/testdata/exclude_glob_double_asterisk_nested.yaml new file mode 100644 index 00000000..534c5aba --- /dev/null +++ b/operator/builtin/input/file/testdata/exclude_glob_double_asterisk_nested.yaml @@ -0,0 +1,5 @@ +type: file_input +include: + - "*.log" +exclude: + - "directory/**/not*.log" \ No newline at end of file diff --git a/operator/builtin/input/file/testdata/exclude_glob_double_asterisk_prefix.yaml b/operator/builtin/input/file/testdata/exclude_glob_double_asterisk_prefix.yaml new file mode 100644 index 00000000..ef5b4ae8 --- /dev/null +++ b/operator/builtin/input/file/testdata/exclude_glob_double_asterisk_prefix.yaml @@ -0,0 +1,5 @@ +type: file_input +include: + - "*.log" +exclude: + - "**/directory/**/not*.log" \ No newline at end of file diff --git a/operator/builtin/input/file/testdata/include_glob_double_asterisk.yaml b/operator/builtin/input/file/testdata/include_glob_double_asterisk.yaml new file mode 100644 index 00000000..7686a592 --- /dev/null +++ b/operator/builtin/input/file/testdata/include_glob_double_asterisk.yaml @@ -0,0 +1,3 @@ +type: file_input +include: + - "**.log" \ No newline at end of file diff --git a/operator/builtin/input/file/testdata/include_glob_double_asterisk_nested.yaml b/operator/builtin/input/file/testdata/include_glob_double_asterisk_nested.yaml new file mode 100644 index 00000000..57b10f67 --- /dev/null +++ b/operator/builtin/input/file/testdata/include_glob_double_asterisk_nested.yaml @@ -0,0 +1,3 @@ +type: file_input +include: + - "directory/**/*.log" \ No newline at end of file diff --git a/operator/builtin/input/file/testdata/include_glob_double_asterisk_prefix.yaml b/operator/builtin/input/file/testdata/include_glob_double_asterisk_prefix.yaml new file mode 100644 index 00000000..675a7460 --- /dev/null +++ b/operator/builtin/input/file/testdata/include_glob_double_asterisk_prefix.yaml @@ -0,0 +1,3 @@ +type: file_input +include: + - "**/directory/**/*.log" \ No newline at end of file