diff --git a/go.mod b/go.mod index 9e03a206..6bf12e76 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/antonmedv/expr v1.8.9 + github.com/bmatcuk/doublestar v1.3.4 github.com/json-iterator/go v1.1.10 github.com/mitchellh/mapstructure v1.4.1 github.com/observiq/ctimefmt v1.0.0 diff --git a/go.sum b/go.sum index d0a238ab..1434ca81 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= +github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= @@ -879,8 +881,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5/go.mod h1:ppEjwdhyy7Y31EnHRDm1JkChoC7LXIJ7Ex0VYLWtZtQ= -github.com/vektra/mockery v1.1.2 h1:uc0Yn67rJpjt8U/mAZimdCKn9AeA97BOkjpmtBSlfP4= -github.com/vektra/mockery v1.1.2/go.mod h1:VcfZjKaFOPO+MpN4ZvwPjs4c48lkq1o3Ym8yHZJu0jU= github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xdg-go/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:FV1RpvYFmF8wnKtr3ArzkC0b+tAySCbw8eP7QSIvLKM= @@ -1223,7 +1223,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/operator/builtin/input/file/config.go b/operator/builtin/input/file/config.go index a3ec7dd7..b5eb312e 100644 --- a/operator/builtin/input/file/config.go +++ b/operator/builtin/input/file/config.go @@ -17,11 +17,11 @@ package file import ( "bufio" "fmt" - "path/filepath" "regexp" "strings" "time" + "github.com/bmatcuk/doublestar" "golang.org/x/text/encoding" "golang.org/x/text/encoding/ianaindex" "golang.org/x/text/encoding/unicode" @@ -91,7 +91,7 @@ func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator, // Ensure includes can be parsed as globs for _, include := range c.Include { - _, err := filepath.Match(include, "matchstring") + _, err := doublestar.PathMatch(include, "matchstring") if err != nil { return nil, fmt.Errorf("parse include glob: %s", err) } @@ -99,7 +99,7 @@ func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator, // Ensure excludes can be parsed as globs for _, exclude := range c.Exclude { - _, err := filepath.Match(exclude, "matchstring") + _, err := doublestar.PathMatch(exclude, "matchstring") if err != nil { return nil, fmt.Errorf("parse exclude glob: %s", err) } diff --git a/operator/builtin/input/file/config_test.go b/operator/builtin/input/file/config_test.go index bbc013f1..eb1f5b58 100644 --- a/operator/builtin/input/file/config_test.go +++ b/operator/builtin/input/file/config_test.go @@ -80,6 +80,24 @@ func TestConfig(t *testing.T) { return cfg }(), }, + { + "include_glob_double_asterisk", + false, + func() *InputConfig { + cfg := defaultCfg() + cfg.Include = append(cfg.Include, "**.log") + return cfg + }(), + }, + { + "include_glob_double_asterisk_nested", + false, + func() *InputConfig { + cfg := defaultCfg() + cfg.Include = append(cfg.Include, "directory/**/*.log") + return cfg + }(), + }, { "include_inline", false, @@ -124,6 +142,26 @@ func TestConfig(t *testing.T) { return cfg }(), }, + { + "exclude_glob_double_asterisk", + false, + func() *InputConfig { + cfg := defaultCfg() + cfg.Include = append(cfg.Include, "*.log") + cfg.Exclude = append(cfg.Exclude, "not**.log") + return cfg + }(), + }, + { + "exclude_glob_double_asterisk_nested", + false, + func() *InputConfig { + cfg := defaultCfg() + cfg.Include = append(cfg.Include, "*.log") + cfg.Exclude = append(cfg.Exclude, "directory/**/not*.log") + return cfg + }(), + }, { "exclude_inline", false, diff --git a/operator/builtin/input/file/file.go b/operator/builtin/input/file/file.go index 9400bf9f..d13af4c7 100644 --- a/operator/builtin/input/file/file.go +++ b/operator/builtin/input/file/file.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/bmatcuk/doublestar" "go.uber.org/zap" "golang.org/x/text/encoding" @@ -187,7 +188,7 @@ func getMatches(includes, excludes []string) []string { INCLUDE: for _, match := range matches { for _, exclude := range excludes { - if itMatches, _ := filepath.Match(exclude, match); itMatches { + if itMatches, _ := doublestar.PathMatch(exclude, match); itMatches { continue INCLUDE } } diff --git a/operator/builtin/input/file/testdata/exclude_glob_double_asterisk.yaml b/operator/builtin/input/file/testdata/exclude_glob_double_asterisk.yaml new file mode 100644 index 00000000..554ba10e --- /dev/null +++ b/operator/builtin/input/file/testdata/exclude_glob_double_asterisk.yaml @@ -0,0 +1,5 @@ +type: file_input +include: + - "*.log" +exclude: + - "not**.log" \ No newline at end of file diff --git a/operator/builtin/input/file/testdata/exclude_glob_double_asterisk_nested.yaml b/operator/builtin/input/file/testdata/exclude_glob_double_asterisk_nested.yaml new file mode 100644 index 00000000..534c5aba --- /dev/null +++ b/operator/builtin/input/file/testdata/exclude_glob_double_asterisk_nested.yaml @@ -0,0 +1,5 @@ +type: file_input +include: + - "*.log" +exclude: + - "directory/**/not*.log" \ No newline at end of file diff --git a/operator/builtin/input/file/testdata/include_glob_double_asterisk.yaml b/operator/builtin/input/file/testdata/include_glob_double_asterisk.yaml new file mode 100644 index 00000000..7686a592 --- /dev/null +++ b/operator/builtin/input/file/testdata/include_glob_double_asterisk.yaml @@ -0,0 +1,3 @@ +type: file_input +include: + - "**.log" \ No newline at end of file diff --git a/operator/builtin/input/file/testdata/include_glob_double_asterisk_nested.yaml b/operator/builtin/input/file/testdata/include_glob_double_asterisk_nested.yaml new file mode 100644 index 00000000..57b10f67 --- /dev/null +++ b/operator/builtin/input/file/testdata/include_glob_double_asterisk_nested.yaml @@ -0,0 +1,3 @@ +type: file_input +include: + - "directory/**/*.log" \ No newline at end of file