diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ef60ad1b..97e87ad26 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,9 +116,7 @@ jobs: command: make test test-windows: - executor: - name: win/default - size: large + executor: win/default steps: - checkout @@ -177,7 +175,7 @@ jobs: command: mkdir {bin,out,tmp} - run: name: Build Carbon Agent - command: go get -v -t -d ./... && go build -v -o ./bin/carbon ./ + command: GOPROXY=direct go build -v -o ./bin/carbon ./ - run: name: Build Log Bench command: GOPROXY=direct go get github.com/observiq/amazon-log-agent-benchmark-tool/cmd/logbench/ && diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35cefaf20..bf88d316e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,57 +68,57 @@ A PR is considered to be **ready to merge** when: ## Design Choices -Best practices for developing a builtin plugin are documented below, but for changes to +Best practices for developing a builtin operator are documented below, but for changes to the core agent, we are happy to discuss proposals in the issue tracker. -### Builtin Plugin Development +### Builtin Operator Development -In order to write a builtin plugin, follow these three steps: -1. Build a unique struct that satisfies the [`Plugin`](plugin/plugin.go) interface. This struct will define what your plugin does when executed in the pipeline. +In order to write a builtin operator, follow these three steps: +1. Build a unique struct that satisfies the [`Operator`](operator/operator.go) interface. This struct will define what your operator does when executed in the pipeline. ```go -type ExamplePlugin struct { +type ExampleOperator struct { FilePath string } -func (p *ExamplePlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (p *ExampleOperator) Process(ctx context.Context, entry *entry.Entry) error { // Processing logic } ``` -2. Build a unique config struct that satisfies the [`Config`](plugin/config.go) interface. This struct will define the parameters used to configure and build your plugin struct in step 1. +2. Build a unique config struct that satisfies the [`Config`](operator/config.go) interface. This struct will define the parameters used to configure and build your operator struct in step 1. ```go -type ExamplePluginConfig struct { +type ExampleOperatorConfig struct { filePath string } -func (c ExamplePluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - return &ExamplePlugin{ +func (c ExampleOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { + return &ExampleOperator{ filePath: c.FilePath, }, nil } ``` -3. Register your config struct in the plugin registry using an `init()` hook. This will ensure that the agent knows about your plugin at runtime and can build it from a YAML config. +3. Register your config struct in the operator registry using an `init()` hook. This will ensure that the agent knows about your operator at runtime and can build it from a YAML config. ```go func init() { - plugin.Register("example_plugin", &ExamplePluginConfig{}) + operator.Register("example_operator", &ExampleOperatorConfig{}) } ``` -## Any tips for building plugins? -We highly recommend that developers take advantage of [helpers](plugin/helper) when building their plugins. Helpers are structs that help satisfy common behavior shared across many plugins. By embedding these structs, you can skip having to satisfy certain aspects of the `plugin` and `config` interfaces. +## Any tips for building operators? +We highly recommend that developers take advantage of [helpers](operator/helper) when building their operators. Helpers are structs that help satisfy common behavior shared across many operators. By embedding these structs, you can skip having to satisfy certain aspects of the `operator` and `config` interfaces. -For example, almost all plugins should embed the [BasicPlugin](plugin/helper/basic_plugin.go) helper, as it provides simple functionality for returning a plugin id and plugin type. +For example, almost all operators should embed the [BasicOperator](operator/helper/basic_operator.go) helper, as it provides simple functionality for returning an operator id and operator type. ```go -// ExamplePlugin is a basic plugin, with a basic lifecycle, that consumes -// but doesn't send log entries. Rather than implementing every part of the plugin +// ExampleOperator is a basic operator, with a basic lifecycle, that consumes +// but doesn't send log entries. Rather than implementing every part of the operator // interface, we can embed the following helpers to achieve this effect. -type ExamplePlugin struct { - helper.BasicPlugin +type ExampleOperator struct { + helper.BasicOperator helper.BasicLifecycle helper.BasicOutput } diff --git a/README.md b/README.md index 88e927988..e908d9429 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ carbon # Supported flags: --config The location of the agent config file (default: ./config.yaml) ---plugin_dir The location of the custom plugins directory (default: ./plugins) +--plugin_dir The location of the plugins directory (default: ./plugins) --database The location of the offsets database file. If this is not specified, offsets will not be maintained across agent restarts --log_file The location of the agent log file. If not specified, carbon will log to `stderr` --debug Enables debug logging diff --git a/agent/agent.go b/agent/agent.go index 077086e1e..7da8a0eb2 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -7,9 +7,9 @@ import ( "time" "github.com/observiq/carbon/errors" + "github.com/observiq/carbon/operator" + _ "github.com/observiq/carbon/operator/builtin" // register operators "github.com/observiq/carbon/pipeline" - pg "github.com/observiq/carbon/plugin" - _ "github.com/observiq/carbon/plugin/builtin" // register plugins "go.etcd.io/bbolt" "go.uber.org/zap" ) @@ -21,7 +21,7 @@ type LogAgent struct { Database string *zap.SugaredLogger - database pg.Database + database operator.Database pipeline *pipeline.Pipeline running bool } @@ -39,13 +39,13 @@ func (a *LogAgent) Start() error { } a.database = database - registry, err := pg.NewCustomRegistry(a.PluginDir) + registry, err := operator.NewPluginRegistry(a.PluginDir) if err != nil { - a.Errorw("Failed to load custom plugin registry", zap.Any("error", err)) + a.Errorw("Failed to load plugin registry", zap.Any("error", err)) } - buildContext := pg.BuildContext{ - CustomRegistry: registry, + buildContext := operator.BuildContext{ + PluginRegistry: registry, Logger: a.SugaredLogger, Database: a.database, } @@ -83,9 +83,9 @@ func (a *LogAgent) Stop() { } // OpenDatabase will open and create a database. -func OpenDatabase(file string) (pg.Database, error) { +func OpenDatabase(file string) (operator.Database, error) { if file == "" { - return pg.NewStubDatabase(), nil + return operator.NewStubDatabase(), nil } if _, err := os.Stat(filepath.Dir(file)); err != nil { @@ -104,11 +104,11 @@ func OpenDatabase(file string) (pg.Database, error) { } // NewLogAgent creates a new carbon log agent. -func NewLogAgent(cfg *Config, logger *zap.SugaredLogger, pluginDir, databaseFile string) *LogAgent { +func NewLogAgent(cfg *Config, logger *zap.SugaredLogger, operatorDir, databaseFile string) *LogAgent { return &LogAgent{ Config: cfg, SugaredLogger: logger, - PluginDir: pluginDir, + PluginDir: operatorDir, Database: databaseFile, } } diff --git a/commands/example_test.go b/commands/example_test.go index 954709557..5e6206fb9 100644 --- a/commands/example_test.go +++ b/commands/example_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/observiq/carbon/plugin/builtin/output" + "github.com/observiq/carbon/operator/builtin/output" "github.com/stretchr/testify/require" ) diff --git a/commands/graph.go b/commands/graph.go index ee476296a..2cc68f952 100644 --- a/commands/graph.go +++ b/commands/graph.go @@ -4,8 +4,8 @@ import ( "os" "github.com/observiq/carbon/agent" - "github.com/observiq/carbon/plugin" - pg "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" + pg "github.com/observiq/carbon/operator" "github.com/spf13/cobra" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -21,7 +21,7 @@ func NewGraphCommand(rootFlags *RootFlags) *cobra.Command { return &cobra.Command{ Use: "graph", Args: cobra.NoArgs, - Short: "Export a dot-formatted representation of the plugin graph", + Short: "Export a dot-formatted representation of the operator graph", Run: func(command *cobra.Command, args []string) { runGraph(command, args, rootFlags) }, } } @@ -43,19 +43,19 @@ func runGraph(_ *cobra.Command, _ []string, flags *RootFlags) { os.Exit(1) } - customRegistry, err := plugin.NewCustomRegistry(flags.PluginDir) + pluginRegistry, err := operator.NewPluginRegistry(flags.PluginDir) if err != nil { - logger.Errorw("Failed to load custom plugin registry", zap.Any("error", err)) + logger.Errorw("Failed to load plugin registry", zap.Any("error", err)) } buildContext := pg.BuildContext{ - CustomRegistry: customRegistry, + PluginRegistry: pluginRegistry, Logger: logger, } pipeline, err := cfg.Pipeline.BuildPipeline(buildContext) if err != nil { - logger.Errorw("Failed to build plugin pipeline", zap.Any("error", err)) + logger.Errorw("Failed to build operator pipeline", zap.Any("error", err)) os.Exit(1) } diff --git a/commands/offsets.go b/commands/offsets.go index 11582519b..3dc9d24a7 100644 --- a/commands/offsets.go +++ b/commands/offsets.go @@ -6,7 +6,7 @@ import ( "os" agent "github.com/observiq/carbon/agent" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator/helper" "github.com/spf13/cobra" "go.etcd.io/bbolt" ) @@ -17,7 +17,7 @@ var stdout io.Writer = os.Stdout func NewOffsetsCmd(rootFlags *RootFlags) *cobra.Command { offsets := &cobra.Command{ Use: "offsets", - Short: "Manage input plugin offsets", + Short: "Manage input operator offsets", Args: cobra.NoArgs, Run: func(command *cobra.Command, args []string) { stdout.Write([]byte("No offsets subcommand specified. See `carbon offsets help` for details\n")) @@ -35,7 +35,7 @@ func NewOffsetsClearCmd(rootFlags *RootFlags) *cobra.Command { var all bool offsetsClear := &cobra.Command{ - Use: "clear [flags] [plugin_ids]", + Use: "clear [flags] [operator_ids]", Short: "Clear persisted offsets from the database", Args: cobra.ArbitraryArgs, Run: func(command *cobra.Command, args []string) { @@ -46,7 +46,7 @@ func NewOffsetsClearCmd(rootFlags *RootFlags) *cobra.Command { if all { if len(args) != 0 { - stdout.Write([]byte("Providing a list of plugin IDs does nothing with the --all flag\n")) + stdout.Write([]byte("Providing a list of operator IDs does nothing with the --all flag\n")) } err := db.Update(func(tx *bbolt.Tx) error { @@ -59,18 +59,18 @@ func NewOffsetsClearCmd(rootFlags *RootFlags) *cobra.Command { exitOnErr("Failed to delete offsets", err) } else { if len(args) == 0 { - stdout.Write([]byte("Must either specify a list of plugins or the --all flag\n")) + stdout.Write([]byte("Must either specify a list of operators or the --all flag\n")) os.Exit(1) } - for _, pluginID := range args { + for _, operatorID := range args { err = db.Update(func(tx *bbolt.Tx) error { offsetBucket := tx.Bucket(helper.OffsetsBucket) if offsetBucket == nil { return nil } - return offsetBucket.DeleteBucket([]byte(pluginID)) + return offsetBucket.DeleteBucket([]byte(operatorID)) }) exitOnErr("Failed to delete offsets", err) } @@ -87,7 +87,7 @@ func NewOffsetsClearCmd(rootFlags *RootFlags) *cobra.Command { func NewOffsetsListCmd(rootFlags *RootFlags) *cobra.Command { offsetsList := &cobra.Command{ Use: "list", - Short: "List plugins with persisted offsets", + Short: "List operators with persisted offsets", Args: cobra.NoArgs, Run: func(command *cobra.Command, args []string) { db, err := agent.OpenDatabase(rootFlags.DatabaseFile) diff --git a/commands/offsets_test.go b/commands/offsets_test.go index a84a12e09..d848abc09 100644 --- a/commands/offsets_test.go +++ b/commands/offsets_test.go @@ -8,7 +8,7 @@ import ( "testing" agent "github.com/observiq/carbon/agent" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" ) @@ -33,15 +33,15 @@ func TestOffsets(t *testing.T) { bucket, err := tx.CreateBucketIfNotExists(helper.OffsetsBucket) require.NoError(t, err) - _, err = bucket.CreateBucket([]byte("$.testpluginid1")) + _, err = bucket.CreateBucket([]byte("$.testoperatorid1")) require.NoError(t, err) - _, err = bucket.CreateBucket([]byte("$.testpluginid2")) + _, err = bucket.CreateBucket([]byte("$.testoperatorid2")) require.NoError(t, err) return nil }) db.Close() - // check that offsets list actually lists the plugin + // check that offsets list actually lists the operator offsetsList := NewRootCmd() offsetsList.SetArgs([]string{ "offsets", "list", @@ -51,7 +51,7 @@ func TestOffsets(t *testing.T) { err = offsetsList.Execute() require.NoError(t, err) - require.Equal(t, "$.testpluginid1\n$.testpluginid2\n", buf.String()) + require.Equal(t, "$.testoperatorid1\n$.testoperatorid2\n", buf.String()) // clear the offsets offsetsClear := NewRootCmd() @@ -59,16 +59,16 @@ func TestOffsets(t *testing.T) { "offsets", "clear", "--database", databasePath, "--config", configPath, - "$.testpluginid2", + "$.testoperatorid2", }) err = offsetsClear.Execute() require.NoError(t, err) - // Check that offsets list only shows uncleared plugin id + // Check that offsets list only shows uncleared operator id buf.Reset() err = offsetsList.Execute() require.NoError(t, err) - require.Equal(t, "$.testpluginid1\n", buf.String()) + require.Equal(t, "$.testoperatorid1\n", buf.String()) } diff --git a/go.mod b/go.mod index 13dd206bc..e50328dcb 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/observiq/ctimefmt v1.0.0 github.com/pkg/errors v0.9.1 // indirect github.com/spf13/cobra v1.0.0 - github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.5.1 go.etcd.io/bbolt v1.3.4 go.uber.org/zap v1.15.0 @@ -34,7 +33,7 @@ require ( google.golang.org/grpc v1.27.1 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.3.0 - honnef.co/go/tools v0.0.1-2020.1.3 + honnef.co/go/tools v0.0.1-2020.1.3 // indirect k8s.io/apimachinery v0.18.4 k8s.io/client-go v0.18.4 ) diff --git a/go.sum b/go.sum index 57893a328..e48f4a3a4 100644 --- a/go.sum +++ b/go.sum @@ -1,141 +1,66 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= cloud.google.com/go v0.43.0 h1:banaiRPAM8kUVYneOSkhgcDsLzEvL25FinuiSZaH/2w= cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go/logging v1.0.0 h1:kaunpnoEh9L4hu6JUsBa8Y20LBfKnCuDhKUgdZp7oK8= cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= -contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= -contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= -contrib.go.opencensus.io/exporter/stackdriver v0.11.0/go.mod h1:hA7rlmtavV03FGxzWXAPBUnZeZBhWN/QYQAuMtxc9Bk= -contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= -contrib.go.opencensus.io/resource v0.0.0-20190131005048-21591786a5e0/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= -github.com/Azure/azure-amqp-common-go v1.1.3/go.mod h1:FhZtXirFANw40UXI2ntweO+VOkfaw8s6vZxUiRhLYW8= -github.com/Azure/azure-amqp-common-go v1.1.4/go.mod h1:FhZtXirFANw40UXI2ntweO+VOkfaw8s6vZxUiRhLYW8= -github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-sdk-for-go v21.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v27.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-service-bus-go v0.4.1/go.mod h1:d9ho9e/06euiTwGpKxmlbpPhFUsfCsq6a4tZ68r51qI= -github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= -github.com/Azure/go-autorest v11.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v11.1.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69/go.mod h1:L1AbZdiDllfyYH5l5OkAaZtk7VkWe89bPJFmnDBNHxg= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= -github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= -github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190418212003-6ac0b49e7197/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo= github.com/Mottl/ctimefmt v0.0.0-20190803144728-fd2ac23a585a/go.mod h1:eyj2WSIdoPMPs2eNTLpSmM6Nzqo4V80/d6jHpnJ1SAI= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= -github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= -github.com/alecthomas/chroma v0.7.2-0.20200305040604-4f3623dce67a/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s= -github.com/alecthomas/chroma v0.7.3/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM= -github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= -github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= -github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= -github.com/alecthomas/kong v0.2.4/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= -github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA= -github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= -github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antonmedv/expr v1.8.2 h1:BfkVHGudYqq7jp3Ji33kTn+qZ9D19t/Mndg0ag/Ycq4= github.com/antonmedv/expr v1.8.2/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.18.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.19.16/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= -github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY= -github.com/bep/golibsass v0.6.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA= -github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= -github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= -github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= -github.com/disintegration/gift v1.2.1/go.mod h1:Jh2i7f7Q2BM7Ezno3PhfezbR1xpUg9dUg3/RlKGr4HI= -github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/elastic/go-elasticsearch v0.0.0 h1:Pd5fqOuBxKxv83b0+xOAJDAkziWYwFinWnBO0y+TZaA= -github.com/elastic/go-elasticsearch v0.0.0/go.mod h1:TkBSJBuTyFdBnrNqoPc54FN0vKf5c04IdM4zuStJ7xg= github.com/elastic/go-elasticsearch/v7 v7.7.0 h1:oQBx/S3RiaH0/kiP0scYSay9xgSmVAYJpuqEf+e9GZg= github.com/elastic/go-elasticsearch/v7 v7.7.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= -github.com/elastic/go-elasticsearch/v8 v8.0.0-20200429055951-e9eb76d55d12 h1:MzWVDDyyAsT9yXAkuCQovqdDiZ+cAQTiXqkHdX7wzwo= -github.com/elastic/go-elasticsearch/v8 v8.0.0-20200429055951-e9eb76d55d12/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanw/esbuild v0.6.2/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= -github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= -github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= -github.com/getkin/kin-openapi v0.14.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -144,18 +69,11 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gohugoio/hugo v0.74.2 h1:F1DGnxBo0hG8/JnODhCMNVnrkPn6HCbBtvveudzwiwU= -github.com/gohugoio/hugo v0.74.2/go.mod h1:2YCeGiVVSSdH/8lQ24W+G5mQTcdI0M10RlT7at1EACQ= -github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95/go.mod h1:bOlVlCa1/RajcHpXkrUXPSHB/Re1UnlXxD1Qp8SKOd8= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -172,30 +90,22 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93 h1:VvBteXw2zOXEgm0o3PgONTWf+bhUGsCaiNn3pbkU9LA= -github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.2.2/go.mod h1:7FHVg6mFpFQrjeUZrm+BaD50N5jnDKm50uVPTpyYOmU= github.com/googleapis/gax-go v1.0.3 h1:9dMLqhaibYONnDRcnHdUs9P8Mw64jLlZTYlDe3leBtQ= github.com/googleapis/gax-go v1.0.3/go.mod h1:QyXYajJFdARxGzjwUfbDFIse7Spkw81SJ4LrBJXtlQ8= -github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= -github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.2/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= @@ -203,19 +113,10 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= -github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -228,21 +129,14 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/go-syslog v1.0.1 h1:a/ARpnCDr/sX/hVH7dyQVi+COXlEzM4bNIoolOfw99Y= github.com/influxdata/go-syslog/v3 v3.0.0 h1:jichmjSZlYK0VMmlz+k4WeOQd7z745YLsvGMqwtYt4I= github.com/influxdata/go-syslog/v3 v3.0.0/go.mod h1:tulsOp+CecTAYC27u9miMgq21GqXRW6VdKbOG+QSP4Q= -github.com/jdkato/prose v1.1.1/go.mod h1:jkF0lkxaX5PFSlk9l4Gh9Y+T57TqUZziWT7uZbW5ADg= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kardianos/service v1.0.0 h1:HgQS3mFfOlyntWX8Oke98JcJLqt1DBcHR4kxShpYef0= @@ -258,31 +152,15 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/markbates/inflect v1.0.0/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88= -github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/mmark v1.3.6/go.mod h1:w7r9mkTvpS55jlfyn22qJ618itLryxXBhA7Jp3FIlkw= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= @@ -293,37 +171,19 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/muesli/smartcrop v0.3.0/go.mod h1:i2fCI/UorTfgEpPPLWiFBv4pye+YAG78RwcQLUkocpI= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= -github.com/niklasfasching/go-org v1.3.0 h1:X8Ob7WOF61iP4KnisZo1eVVei8/GzmNz3ymJrYLnIic= -github.com/niklasfasching/go-org v1.3.0/go.mod h1:AsLD6X7djzRIz4/RFZu8vwRL0VGjUvGZCCH1Nz0VdrU= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= -github.com/observiq/ctimefmt v0.0.0-20200611210846-e39bd83b5771 h1:vvoch82jTZ2EbgdjMQ437TBFKwzyeiDdwuGvMqG0XeQ= -github.com/observiq/ctimefmt v0.0.0-20200611210846-e39bd83b5771/go.mod h1:jfs3wpVShz6PBYh3nouK+OwrJNxXXZESIKm6xyKzzBY= -github.com/observiq/ctimefmt v0.0.0-20200612160717-3e07deba22cf h1:WTafs6lnKha7XqeguHvaboRwWSHdRwVBVrFQlIEYkvI= -github.com/observiq/ctimefmt v0.0.0-20200612160717-3e07deba22cf/go.mod h1:jfs3wpVShz6PBYh3nouK+OwrJNxXXZESIKm6xyKzzBY= github.com/observiq/ctimefmt v1.0.0 h1:r7vTJ+Slkrt9fZ67mkf+mA6zAdR5nGIJRMTzkUyvilk= github.com/observiq/ctimefmt v1.0.0/go.mod h1:mxi62//WbSpG/roCO1c6MqZ7zQTvjVtYheqHN3eOjvc= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= -github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -332,58 +192,37 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.5.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday v1.5.3-0.20200218234912-41c5fccfd6f6/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/fsync v0.9.0/go.mod h1:fNtJEfG3HiltN3y4cPOz6MLjos9+2pIEqLIgszqhp/0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= -github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -393,39 +232,17 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tdewolff/minify/v2 v2.6.2/go.mod h1:BkDSm8aMMT0ALGmpt7j3Ra7nLUgZL0qhyrAHXwxcy5w= -github.com/tdewolff/parse/v2 v2.4.2/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= -github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= -github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= -github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.31/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691/go.mod h1:YLF3kDffRfUH/bTxOxHhV6lxwIB3Vfj91rEwNMS9MXo= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -437,13 +254,9 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -gocloud.dev v0.15.0/go.mod h1:ShXCyJaGrJu9y/7a6+DSCyBb9MFGZ1P5wwPa0Wu6w34= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= @@ -459,7 +272,6 @@ golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522 h1:OeRHuibLsmZkFj773W4LcfAGs golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20191214001246-9130b4cfad52/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -478,33 +290,23 @@ golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190322120337-addf6b3196f6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190420063019-afa5a82059c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190523182746-aaccbc9213b0/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -518,36 +320,26 @@ golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200513112337-417ce2331b5c h1:kISX68E8gSkNYAFRFiDU8rl5RIn1sJYKYb/r2vMLDrU= golang.org/x/sys v0.0.0-20200513112337-417ce2331b5c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -565,8 +357,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -580,7 +370,6 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97 h1:DAuln/hGp+aJiHpID1Y1hYzMEPP5WLwtZHPb50mN0OE= golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -591,36 +380,27 @@ gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200304201815-d429ff31ee6c h1:Mm69MraVZ+yh1vw8pQOUW4uJkkSEQbbTr076A94lvqs= google.golang.org/genproto v0.0.0-20200304201815-d429ff31ee6c/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -630,7 +410,6 @@ google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -638,15 +417,11 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= -gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -662,27 +437,16 @@ k8s.io/api v0.18.4 h1:8x49nBRxuXGUlDlwlWd3RMY1SayZrzFfxea3UZSkFw4= k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4= k8s.io/apimachinery v0.18.4 h1:ST2beySjhqwJoIFk6p7Hp5v5O0hYY6Gngq/gUYXTPIA= k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apimachinery v0.18.5-rc.0 h1:V04jBUrFtuyH8bXfV/UJUfG7DvleLJsC6yZwAkLiiGE= -k8s.io/apimachinery v0.18.5-rc.0/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/client-go v0.18.4 h1:un55V1Q/B3JO3A76eS0kUSywgGK/WR3BQ8fHQjNa6Zc= k8s.io/client-go v0.18.4/go.mod h1:f5sXwL4yAZRkAtzOxRWUhA/N8XzGCb+nPZI8PfobZ9g= -k8s.io/client-go v1.5.1 h1:XaX/lo2/u3/pmFau8HN+sB5C/b4dc4Dmm2eXjBH4p1E= -k8s.io/client-go v1.5.1/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= -k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= -k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19 h1:7Nu2dTj82c6IaWvL7hImJzcXoTPz1MsSCH7r+0m6rfo= -k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -pack.ag/amqp v0.8.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= -pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index d3b5b836a..644375884 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -1,8 +1,8 @@ package testutil -// NewMockPlugin will return a basic plugin mock -func NewMockPlugin(id string) *Plugin { - mockOutput := &Plugin{} +// NewMockOperator will return a basic operator mock +func NewMockOperator(id string) *Operator { + mockOutput := &Operator{} mockOutput.On("ID").Return(id) mockOutput.On("CanProcess").Return(true) mockOutput.On("CanOutput").Return(true) diff --git a/internal/testutil/plugin.go b/internal/testutil/operator.go similarity index 73% rename from internal/testutil/plugin.go rename to internal/testutil/operator.go index 2e6d6f9bf..9b3c321b2 100644 --- a/internal/testutil/plugin.go +++ b/internal/testutil/operator.go @@ -8,18 +8,18 @@ import ( entry "github.com/observiq/carbon/entry" mock "github.com/stretchr/testify/mock" - plugin "github.com/observiq/carbon/plugin" + operator "github.com/observiq/carbon/operator" zap "go.uber.org/zap" ) -// Plugin is an autogenerated mock type for the Plugin type -type Plugin struct { +// Operator is an autogenerated mock type for the Operator type +type Operator struct { mock.Mock } // CanOutput provides a mock function with given fields: -func (_m *Plugin) CanOutput() bool { +func (_m *Operator) CanOutput() bool { ret := _m.Called() var r0 bool @@ -33,7 +33,7 @@ func (_m *Plugin) CanOutput() bool { } // CanProcess provides a mock function with given fields: -func (_m *Plugin) CanProcess() bool { +func (_m *Operator) CanProcess() bool { ret := _m.Called() var r0 bool @@ -47,7 +47,7 @@ func (_m *Plugin) CanProcess() bool { } // ID provides a mock function with given fields: -func (_m *Plugin) ID() string { +func (_m *Operator) ID() string { ret := _m.Called() var r0 string @@ -61,7 +61,7 @@ func (_m *Plugin) ID() string { } // Logger provides a mock function with given fields: -func (_m *Plugin) Logger() *zap.SugaredLogger { +func (_m *Operator) Logger() *zap.SugaredLogger { ret := _m.Called() var r0 *zap.SugaredLogger @@ -77,15 +77,15 @@ func (_m *Plugin) Logger() *zap.SugaredLogger { } // Outputs provides a mock function with given fields: -func (_m *Plugin) Outputs() []plugin.Plugin { +func (_m *Operator) Outputs() []operator.Operator { ret := _m.Called() - var r0 []plugin.Plugin - if rf, ok := ret.Get(0).(func() []plugin.Plugin); ok { + var r0 []operator.Operator + if rf, ok := ret.Get(0).(func() []operator.Operator); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]plugin.Plugin) + r0 = ret.Get(0).([]operator.Operator) } } @@ -93,7 +93,7 @@ func (_m *Plugin) Outputs() []plugin.Plugin { } // Process provides a mock function with given fields: _a0, _a1 -func (_m *Plugin) Process(_a0 context.Context, _a1 *entry.Entry) error { +func (_m *Operator) Process(_a0 context.Context, _a1 *entry.Entry) error { ret := _m.Called(_a0, _a1) var r0 error @@ -107,11 +107,11 @@ func (_m *Plugin) Process(_a0 context.Context, _a1 *entry.Entry) error { } // SetOutputs provides a mock function with given fields: _a0 -func (_m *Plugin) SetOutputs(_a0 []plugin.Plugin) error { +func (_m *Operator) SetOutputs(_a0 []operator.Operator) error { ret := _m.Called(_a0) var r0 error - if rf, ok := ret.Get(0).(func([]plugin.Plugin) error); ok { + if rf, ok := ret.Get(0).(func([]operator.Operator) error); ok { r0 = rf(_a0) } else { r0 = ret.Error(0) @@ -121,7 +121,7 @@ func (_m *Plugin) SetOutputs(_a0 []plugin.Plugin) error { } // Start provides a mock function with given fields: -func (_m *Plugin) Start() error { +func (_m *Operator) Start() error { ret := _m.Called() var r0 error @@ -135,7 +135,7 @@ func (_m *Plugin) Start() error { } // Stop provides a mock function with given fields: -func (_m *Plugin) Stop() error { +func (_m *Operator) Stop() error { ret := _m.Called() var r0 error @@ -149,7 +149,7 @@ func (_m *Plugin) Stop() error { } // Type provides a mock function with given fields: -func (_m *Plugin) Type() string { +func (_m *Operator) Type() string { ret := _m.Called() var r0 string diff --git a/internal/testutil/util.go b/internal/testutil/util.go index 5f91c07e6..f7dfb1386 100644 --- a/internal/testutil/util.go +++ b/internal/testutil/util.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "go.etcd.io/bbolt" "go.uber.org/zap/zaptest" ) @@ -50,8 +50,8 @@ func NewTestDatabase(t *testing.T) *bbolt.DB { } // NewBuildContext will return a new build context for testing -func NewBuildContext(t *testing.T) plugin.BuildContext { - return plugin.BuildContext{ +func NewBuildContext(t *testing.T) operator.BuildContext { + return operator.BuildContext{ Database: NewTestDatabase(t), Logger: zaptest.NewLogger(t).Sugar(), } diff --git a/operator/buffer/buffer.go b/operator/buffer/buffer.go new file mode 100644 index 000000000..cb7c6913d --- /dev/null +++ b/operator/buffer/buffer.go @@ -0,0 +1,76 @@ +package buffer + +import ( + "context" + "fmt" + "time" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/errors" + "github.com/observiq/carbon/operator" +) + +// Buffer is an entity that buffers log entries to an operator +type Buffer interface { + Flush(context.Context) error + Add(interface{}, int) error + AddWait(context.Context, interface{}, int) error + SetHandler(BundleHandler) + Process(context.Context, *entry.Entry) error +} + +func NewConfig() Config { + return Config{ + BufferType: "memory", + DelayThreshold: operator.Duration{Duration: time.Second}, + BundleCountThreshold: 10_000, + BundleByteThreshold: 4 * 1024 * 1024 * 1024, // 4MB + BundleByteLimit: 4 * 1024 * 1024 * 1024, // 4MB + BufferedByteLimit: 500 * 1024 * 1024 * 1024, // 500MB + HandlerLimit: 32, + Retry: NewRetryConfig(), + } +} + +// Config is the configuration of a buffer +type Config struct { + BufferType string `json:"type,omitempty" yaml:"type,omitempty"` + DelayThreshold operator.Duration `json:"delay_threshold,omitempty" yaml:"delay_threshold,omitempty"` + BundleCountThreshold int `json:"bundle_count_threshold,omitempty" yaml:"buffer_count_threshold,omitempty"` + BundleByteThreshold int `json:"bundle_byte_threshold,omitempty" yaml:"bundle_byte_threshold,omitempty"` + BundleByteLimit int `json:"bundle_byte_limit,omitempty" yaml:"bundle_byte_limit,omitempty"` + BufferedByteLimit int `json:"buffered_byte_limit,omitempty" yaml:"buffered_byte_limit,omitempty"` + HandlerLimit int `json:"handler_limit,omitempty" yaml:"handler_limit,omitempty"` + Retry RetryConfig `json:"retry,omitempty" yaml:"retry,omitempty"` +} + +// Build will build a buffer from the supplied configuration +func (config *Config) Build() (Buffer, error) { + switch config.BufferType { + case "memory", "": + return NewMemoryBuffer(config), nil + default: + return nil, errors.NewError( + fmt.Sprintf("Invalid buffer type %s", config.BufferType), + "The only supported buffer type is 'memory'", + ) + } +} + +func NewRetryConfig() RetryConfig { + return RetryConfig{ + InitialInterval: operator.Duration{Duration: 500 * time.Millisecond}, + RandomizationFactor: 0.5, + Multiplier: 1.5, + MaxInterval: operator.Duration{Duration: 15 * time.Minute}, + } +} + +// RetryConfig is the configuration of an entity that will retry processing after an error +type RetryConfig struct { + InitialInterval operator.Duration `json:"initial_interval,omitempty" yaml:"initial_interval,omitempty"` + RandomizationFactor float64 `json:"randomization_factor,omitempty" yaml:"randomization_factor,omitempty"` + Multiplier float64 `json:"multiplier,omitempty" yaml:"multiplier,omitempty"` + MaxInterval operator.Duration `json:"max_interval,omitempty" yaml:"max_interval,omitempty"` + MaxElapsedTime operator.Duration `json:"max_elapsed_time,omitempty" yaml:"max_elapsed_time,omitempty"` +} diff --git a/plugin/buffer/buffer_test.go b/operator/buffer/buffer_test.go similarity index 95% rename from plugin/buffer/buffer_test.go rename to operator/buffer/buffer_test.go index 9eb71062f..11cded7e1 100644 --- a/plugin/buffer/buffer_test.go +++ b/operator/buffer/buffer_test.go @@ -8,7 +8,7 @@ import ( "time" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/require" "go.uber.org/zap" yaml "gopkg.in/yaml.v2" @@ -34,7 +34,7 @@ func (b *bufferHandler) Logger() *zap.SugaredLogger { func TestBuffer(t *testing.T) { config := NewConfig() - config.DelayThreshold = plugin.Duration{ + config.DelayThreshold = operator.Duration{ Duration: 100 * time.Millisecond, } diff --git a/plugin/buffer/memory_buffer.go b/operator/buffer/memory_buffer.go similarity index 100% rename from plugin/buffer/memory_buffer.go rename to operator/buffer/memory_buffer.go diff --git a/plugin/buffer/memory_buffer_test.go b/operator/buffer/memory_buffer_test.go similarity index 90% rename from plugin/buffer/memory_buffer_test.go rename to operator/buffer/memory_buffer_test.go index 829f51138..20e66f549 100644 --- a/plugin/buffer/memory_buffer_test.go +++ b/operator/buffer/memory_buffer_test.go @@ -7,7 +7,7 @@ import ( "time" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -47,7 +47,7 @@ func newMockHandler(t *testing.T) *mockHandler { func TestMemoryBufferRetry(t *testing.T) { t.Run("FailOnce", func(t *testing.T) { cfg := NewConfig() - cfg.DelayThreshold = plugin.Duration{Duration: 10 * time.Millisecond} + cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Millisecond} buffer, err := cfg.Build() require.NoError(t, err) handler := newMockHandler(t) @@ -68,7 +68,7 @@ func TestMemoryBufferRetry(t *testing.T) { t.Run("ContextCancelled", func(t *testing.T) { cfg := NewConfig() - cfg.DelayThreshold = plugin.Duration{Duration: 10 * time.Millisecond} + cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Millisecond} buffer, err := cfg.Build() require.NoError(t, err) handler := newMockHandler(t) @@ -95,8 +95,8 @@ func TestMemoryBufferRetry(t *testing.T) { t.Run("ExceededLimit", func(t *testing.T) { cfg := NewConfig() - cfg.DelayThreshold = plugin.Duration{Duration: 10 * time.Millisecond} - cfg.Retry.MaxElapsedTime = plugin.Duration{Duration: time.Nanosecond} + cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Millisecond} + cfg.Retry.MaxElapsedTime = operator.Duration{Duration: time.Nanosecond} buffer, err := cfg.Build() require.NoError(t, err) handler := newMockHandler(t) @@ -124,7 +124,7 @@ func TestMemoryBufferRetry(t *testing.T) { func TestMemoryBufferFlush(t *testing.T) { t.Run("Simple", func(t *testing.T) { cfg := NewConfig() - cfg.DelayThreshold = plugin.Duration{Duration: 10 * time.Hour} + cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Hour} buffer, err := cfg.Build() require.NoError(t, err) handler := newMockHandler(t) @@ -156,7 +156,7 @@ func TestMemoryBufferFlush(t *testing.T) { t.Run("ContextCancelled", func(t *testing.T) { cfg := NewConfig() - cfg.DelayThreshold = plugin.Duration{Duration: 10 * time.Hour} + cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Hour} buffer, err := cfg.Build() require.NoError(t, err) handler := newMockHandler(t) diff --git a/operator/builtin/builtin.go b/operator/builtin/builtin.go new file mode 100644 index 000000000..c52ad1286 --- /dev/null +++ b/operator/builtin/builtin.go @@ -0,0 +1,9 @@ +package builtin + +import ( + // Load embedded packages when importing builtin operators + _ "github.com/observiq/carbon/operator/builtin/input" + _ "github.com/observiq/carbon/operator/builtin/output" + _ "github.com/observiq/carbon/operator/builtin/parser" + _ "github.com/observiq/carbon/operator/builtin/transformer" +) diff --git a/plugin/builtin/builtin_test.go b/operator/builtin/builtin_test.go similarity index 100% rename from plugin/builtin/builtin_test.go rename to operator/builtin/builtin_test.go diff --git a/plugin/builtin/input/file/file.go b/operator/builtin/input/file/file.go similarity index 86% rename from plugin/builtin/input/file/file.go rename to operator/builtin/input/file/file.go index c3c2bfa2b..d04c42780 100644 --- a/plugin/builtin/input/file/file.go +++ b/operator/builtin/input/file/file.go @@ -16,8 +16,8 @@ import ( "time" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" "golang.org/x/text/encoding" "golang.org/x/text/encoding/ianaindex" @@ -25,13 +25,13 @@ import ( ) func init() { - plugin.Register("file_input", func() plugin.Builder { return NewInputConfig("") }) + operator.Register("file_input", func() operator.Builder { return NewInputConfig("") }) } -func NewInputConfig(pluginID string) *InputConfig { +func NewInputConfig(operatorID string) *InputConfig { return &InputConfig{ - InputConfig: helper.NewInputConfig(pluginID, "file_input"), - PollInterval: plugin.Duration{Duration: 200 * time.Millisecond}, + InputConfig: helper.NewInputConfig(operatorID, "file_input"), + PollInterval: operator.Duration{Duration: 200 * time.Millisecond}, FilePathField: entry.NewNilField(), FileNameField: entry.NewNilField(), StartAt: "end", @@ -40,20 +40,20 @@ func NewInputConfig(pluginID string) *InputConfig { } } -// InputConfig is the configuration of a file input plugin +// InputConfig is the configuration of a file input operator type InputConfig struct { helper.InputConfig `yaml:",inline"` Include []string `json:"include,omitempty" yaml:"include,omitempty"` Exclude []string `json:"exclude,omitempty" yaml:"exclude,omitempty"` - PollInterval plugin.Duration `json:"poll_interval,omitempty" yaml:"poll_interval,omitempty"` - Multiline *MultilineConfig `json:"multiline,omitempty" yaml:"multiline,omitempty"` - FilePathField entry.Field `json:"file_path_field,omitempty" yaml:"file_path_field,omitempty"` - FileNameField entry.Field `json:"file_name_field,omitempty" yaml:"file_name_field,omitempty"` - StartAt string `json:"start_at,omitempty" yaml:"start_at,omitempty"` - MaxLogSize int `json:"max_log_size,omitempty" yaml:"max_log_size,omitempty"` - Encoding string `json:"encoding,omitempty" yaml:"encoding,omitempty"` + PollInterval operator.Duration `json:"poll_interval,omitempty" yaml:"poll_interval,omitempty"` + Multiline *MultilineConfig `json:"multiline,omitempty" yaml:"multiline,omitempty"` + FilePathField entry.Field `json:"file_path_field,omitempty" yaml:"file_path_field,omitempty"` + FileNameField entry.Field `json:"file_name_field,omitempty" yaml:"file_name_field,omitempty"` + StartAt string `json:"start_at,omitempty" yaml:"start_at,omitempty"` + MaxLogSize int `json:"max_log_size,omitempty" yaml:"max_log_size,omitempty"` + Encoding string `json:"encoding,omitempty" yaml:"encoding,omitempty"` } // MultilineConfig is the configuration a multiline operation @@ -62,9 +62,9 @@ type MultilineConfig struct { LineEndPattern string `json:"line_end_pattern" yaml:"line_end_pattern"` } -// Build will build a file input plugin from the supplied configuration -func (c InputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - inputPlugin, err := c.InputConfig.Build(context) +// Build will build a file input operator from the supplied configuration +func (c InputConfig) Build(context operator.BuildContext) (operator.Operator, error) { + inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err } @@ -109,8 +109,8 @@ func (c InputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { return nil, fmt.Errorf("invalid start_at location '%s'", c.StartAt) } - plugin := &InputPlugin{ - InputPlugin: inputPlugin, + operator := &InputOperator{ + InputOperator: inputOperator, Include: c.Include, Exclude: c.Exclude, SplitFunc: splitFunc, @@ -126,7 +126,7 @@ func (c InputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { MaxLogSize: c.MaxLogSize, } - return plugin, nil + return operator, nil } var encodingOverrides = map[string]encoding.Encoding{ @@ -186,9 +186,9 @@ func (c InputConfig) getSplitFunc(encoding encoding.Encoding) (bufio.SplitFunc, return splitFunc, nil } -// InputPlugin is a plugin that monitors files for entries -type InputPlugin struct { - helper.InputPlugin +// InputOperator is an operator that monitors files for entries +type InputOperator struct { + helper.InputOperator Include []string Exclude []string @@ -215,7 +215,7 @@ type InputPlugin struct { } // Start will start the file monitoring process -func (f *InputPlugin) Start() error { +func (f *InputOperator) Start() error { ctx, cancel := context.WithCancel(context.Background()) f.cancel = cancel f.wg = &sync.WaitGroup{} @@ -266,7 +266,7 @@ func (f *InputPlugin) Start() error { } // Stop will stop the file monitoring process -func (f *InputPlugin) Stop() error { +func (f *InputOperator) Stop() error { f.cancel() f.wg.Wait() f.syncKnownFiles() @@ -279,7 +279,7 @@ func (f *InputPlugin) Stop() error { // firstCheck indicates whether this is the first time checkFile has been called // after startup. This is important for the start_at parameter because, after initial // startup, we don't want to start at the end of newly-created files. -func (f *InputPlugin) checkFile(ctx context.Context, path string, firstCheck bool) { +func (f *InputOperator) checkFile(ctx context.Context, path string, firstCheck bool) { // Check if the file is currently being read if _, ok := f.runningFiles[path]; ok { @@ -314,14 +314,14 @@ func (f *InputPlugin) checkFile(ctx context.Context, path string, firstCheck boo go func(ctx context.Context, path string, offset, lastSeenSize int64) { defer f.readerWg.Done() messenger := f.newFileUpdateMessenger(path) - err := ReadToEnd(ctx, path, offset, lastSeenSize, messenger, f.SplitFunc, f.FilePathField, f.FileNameField, f.InputPlugin, f.MaxLogSize, f.encoding) + err := ReadToEnd(ctx, path, offset, lastSeenSize, messenger, f.SplitFunc, f.FilePathField, f.FileNameField, f.InputOperator, f.MaxLogSize, f.encoding) if err != nil { f.Warnw("Failed to read log file", zap.Error(err)) } }(ctx, path, knownFile.Offset, knownFile.LastSeenFileSize) } -func (f *InputPlugin) updateFile(message fileUpdateMessage) { +func (f *InputOperator) updateFile(message fileUpdateMessage) { if message.finished { delete(f.runningFiles, message.path) return @@ -385,7 +385,7 @@ func (f *InputPlugin) updateFile(message fileUpdateMessage) { knownFile.Offset = message.newOffset } -func (f *InputPlugin) drainMessages() { +func (f *InputOperator) drainMessages() { done := make(chan struct{}) go func() { f.readerWg.Wait() @@ -404,7 +404,7 @@ func (f *InputPlugin) drainMessages() { var knownFilesKey = "knownFiles" -func (f *InputPlugin) syncKnownFiles() { +func (f *InputOperator) syncKnownFiles() { var buf bytes.Buffer enc := gob.NewEncoder(&buf) err := enc.Encode(f.knownFiles) @@ -417,7 +417,7 @@ func (f *InputPlugin) syncKnownFiles() { f.persist.Sync() } -func (f *InputPlugin) readKnownFiles() (map[string]*knownFileInfo, error) { +func (f *InputOperator) readKnownFiles() (map[string]*knownFileInfo, error) { err := f.persist.Load() if err != nil { return nil, err @@ -439,7 +439,7 @@ func (f *InputPlugin) readKnownFiles() (map[string]*knownFileInfo, error) { return knownFiles, nil } -func (f *InputPlugin) newFileUpdateMessenger(path string) fileUpdateMessenger { +func (f *InputOperator) newFileUpdateMessenger(path string) fileUpdateMessenger { return fileUpdateMessenger{ path: path, c: f.fileUpdateChan, diff --git a/plugin/builtin/input/file/file_test.go b/operator/builtin/input/file/file_test.go similarity index 95% rename from plugin/builtin/input/file/file_test.go rename to operator/builtin/input/file/file_test.go index 32ff023e0..8a2b2e6dc 100644 --- a/plugin/builtin/input/file/file_test.go +++ b/operator/builtin/input/file/file_test.go @@ -14,43 +14,43 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -func newTestFileSource(t *testing.T) (*InputPlugin, chan *entry.Entry) { - mockOutput := testutil.NewMockPlugin("output") +func newTestFileSource(t *testing.T) (*InputOperator, chan *entry.Entry) { + mockOutput := testutil.NewMockOperator("output") receivedEntries := make(chan *entry.Entry, 1000) mockOutput.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { receivedEntries <- args.Get(1).(*entry.Entry) }) cfg := NewInputConfig("testfile") - cfg.PollInterval = plugin.Duration{Duration: 50 * time.Millisecond} + cfg.PollInterval = operator.Duration{Duration: 50 * time.Millisecond} cfg.StartAt = "beginning" cfg.Include = []string{"should-be-overwritten"} pg, err := cfg.Build(testutil.NewBuildContext(t)) if err != nil { - t.Fatalf("Error building plugin: %s", err) + t.Fatalf("Error building operator: %s", err) } - source := pg.(*InputPlugin) - source.OutputPlugins = []plugin.Plugin{mockOutput} + source := pg.(*InputOperator) + source.OutputOperators = []operator.Operator{mockOutput} return source, receivedEntries } func TestFileSource_Build(t *testing.T) { t.Parallel() - mockOutput := testutil.NewMockPlugin("mock") + mockOutput := testutil.NewMockOperator("mock") basicConfig := func() *InputConfig { cfg := NewInputConfig("testfile") cfg.OutputIDs = []string{"mock"} cfg.Include = []string{"/var/log/testpath.*"} cfg.Exclude = []string{"/var/log/testpath.ex*"} - cfg.PollInterval = plugin.Duration{Duration: 10 * time.Millisecond} + cfg.PollInterval = operator.Duration{Duration: 10 * time.Millisecond} cfg.FilePathField = entry.NewRecordField("testpath") return cfg } @@ -59,14 +59,14 @@ func TestFileSource_Build(t *testing.T) { name string modifyBaseConfig func(*InputConfig) errorRequirement require.ErrorAssertionFunc - validate func(*testing.T, *InputPlugin) + validate func(*testing.T, *InputOperator) }{ { "Basic", func(f *InputConfig) { return }, require.NoError, - func(t *testing.T, f *InputPlugin) { - require.Equal(t, f.OutputPlugins[0], mockOutput) + func(t *testing.T, f *InputOperator) { + require.Equal(t, f.OutputOperators[0], mockOutput) require.Equal(t, f.Include, []string{"/var/log/testpath.*"}) require.Equal(t, f.FilePathField, entry.NewRecordField("testpath")) require.Equal(t, f.PollInterval, 10*time.Millisecond) @@ -107,7 +107,7 @@ func TestFileSource_Build(t *testing.T) { } }, require.NoError, - func(t *testing.T, f *InputPlugin) {}, + func(t *testing.T, f *InputOperator) {}, }, { "MultilineConfiguredEndPattern", @@ -117,7 +117,7 @@ func TestFileSource_Build(t *testing.T) { } }, require.NoError, - func(t *testing.T, f *InputPlugin) {}, + func(t *testing.T, f *InputOperator) {}, }, } @@ -132,10 +132,10 @@ func TestFileSource_Build(t *testing.T) { return } - err = plg.SetOutputs([]plugin.Plugin{mockOutput}) + err = plg.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) - fileInput := plg.(*InputPlugin) + fileInput := plg.(*InputOperator) tc.validate(t, fileInput) }) } diff --git a/plugin/builtin/input/file/line_splitter.go b/operator/builtin/input/file/line_splitter.go similarity index 100% rename from plugin/builtin/input/file/line_splitter.go rename to operator/builtin/input/file/line_splitter.go diff --git a/plugin/builtin/input/file/line_splitter_test.go b/operator/builtin/input/file/line_splitter_test.go similarity index 100% rename from plugin/builtin/input/file/line_splitter_test.go rename to operator/builtin/input/file/line_splitter_test.go diff --git a/plugin/builtin/input/file/read_to_end.go b/operator/builtin/input/file/read_to_end.go similarity index 78% rename from plugin/builtin/input/file/read_to_end.go rename to operator/builtin/input/file/read_to_end.go index 8e5a67217..0f2bf76f9 100644 --- a/plugin/builtin/input/file/read_to_end.go +++ b/operator/builtin/input/file/read_to_end.go @@ -9,12 +9,12 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" "golang.org/x/text/encoding" ) -// ReadToEnd will read entries from a file and send them to the outputs of an input plugin +// ReadToEnd will read entries from a file and send them to the outputs of an input operator func ReadToEnd( ctx context.Context, path string, @@ -24,7 +24,7 @@ func ReadToEnd( splitFunc bufio.SplitFunc, filePathField entry.Field, fileNameField entry.Field, - inputPlugin helper.InputPlugin, + inputOperator helper.InputOperator, maxLogSize int, encoding encoding.Encoding, ) error { @@ -79,7 +79,7 @@ func ReadToEnd( // advanced since last cycle, read the rest of the file as an entry defer func() { if pos < stat.Size() && pos == startOffset && lastSeenFileSize == stat.Size() { - readRemaining(ctx, file, pos, stat.Size(), messenger, inputPlugin, filePathField, fileNameField, decoder, decodeBuffer) + readRemaining(ctx, file, pos, stat.Size(), messenger, inputOperator, filePathField, fileNameField, decoder, decodeBuffer) } }() @@ -104,37 +104,37 @@ func ReadToEnd( return err } - e := inputPlugin.NewEntry(string(decodeBuffer[:nDst])) + e := inputOperator.NewEntry(string(decodeBuffer[:nDst])) e.Set(filePathField, path) e.Set(fileNameField, filepath.Base(file.Name())) - inputPlugin.Write(ctx, e) + inputOperator.Write(ctx, e) messenger.SetOffset(pos) } } // readRemaining will read the remaining characters in a file as a log entry. -func readRemaining(ctx context.Context, file *os.File, filePos int64, fileSize int64, messenger fileUpdateMessenger, inputPlugin helper.InputPlugin, filePathField, fileNameField entry.Field, encoder *encoding.Decoder, decodeBuffer []byte) { +func readRemaining(ctx context.Context, file *os.File, filePos int64, fileSize int64, messenger fileUpdateMessenger, inputOperator helper.InputOperator, filePathField, fileNameField entry.Field, encoder *encoding.Decoder, decodeBuffer []byte) { _, err := file.Seek(filePos, 0) if err != nil { - inputPlugin.Errorf("failed to seek to read last log entry") + inputOperator.Errorf("failed to seek to read last log entry") return } msgBuf := make([]byte, fileSize-filePos) n, err := file.Read(msgBuf) if err != nil { - inputPlugin.Errorf("failed to read trailing log") + inputOperator.Errorf("failed to read trailing log") return } encoder.Reset() nDst, _, err := encoder.Transform(decodeBuffer, msgBuf, true) if err != nil { - inputPlugin.Errorw("failed to decode trailing log", zap.Error(err)) + inputOperator.Errorw("failed to decode trailing log", zap.Error(err)) } - e := inputPlugin.NewEntry(string(decodeBuffer[:nDst])) + e := inputOperator.NewEntry(string(decodeBuffer[:nDst])) e.Set(filePathField, file.Name()) e.Set(fileNameField, filepath.Base(file.Name())) - inputPlugin.Write(ctx, e) + inputOperator.Write(ctx, e) messenger.SetOffset(filePos + int64(n)) } diff --git a/plugin/builtin/input/generate.go b/operator/builtin/input/generate.go similarity index 73% rename from plugin/builtin/input/generate.go rename to operator/builtin/input/generate.go index ebaa0ca72..b85d36eac 100644 --- a/plugin/builtin/input/generate.go +++ b/operator/builtin/input/generate.go @@ -7,21 +7,21 @@ import ( "time" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("generate_input", func() plugin.Builder { return NewGenerateInputConfig("") }) + operator.Register("generate_input", func() operator.Builder { return NewGenerateInputConfig("") }) } -func NewGenerateInputConfig(pluginID string) *GenerateInputConfig { +func NewGenerateInputConfig(operatorID string) *GenerateInputConfig { return &GenerateInputConfig{ - InputConfig: helper.NewInputConfig(pluginID, "generate_input"), + InputConfig: helper.NewInputConfig(operatorID, "generate_input"), } } -// GenerateInputConfig is the configuration of a generate input plugin. +// GenerateInputConfig is the configuration of a generate input operator. type GenerateInputConfig struct { helper.InputConfig `yaml:",inline"` Entry entry.Entry `json:"entry" yaml:"entry"` @@ -29,9 +29,9 @@ type GenerateInputConfig struct { Static bool `json:"static" yaml:"static,omitempty"` } -// Build will build a generate input plugin. -func (c *GenerateInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - inputPlugin, err := c.InputConfig.Build(context) +// Build will build a generate input operator. +func (c *GenerateInputConfig) Build(context operator.BuildContext) (operator.Operator, error) { + inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err } @@ -39,17 +39,17 @@ func (c *GenerateInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, c.Entry.Record = recursiveMapInterfaceToMapString(c.Entry.Record) generateInput := &GenerateInput{ - InputPlugin: inputPlugin, - entry: c.Entry, - count: c.Count, - static: c.Static, + InputOperator: inputOperator, + entry: c.Entry, + count: c.Count, + static: c.Static, } return generateInput, nil } -// GenerateInput is a plugin that generates log entries. +// GenerateInput is an operator that generates log entries. type GenerateInput struct { - helper.InputPlugin + helper.InputOperator entry entry.Entry count int static bool diff --git a/plugin/builtin/input/generate_test.go b/operator/builtin/input/generate_test.go similarity index 76% rename from plugin/builtin/input/generate_test.go rename to operator/builtin/input/generate_test.go index e7d78ac10..668a7bc0f 100644 --- a/plugin/builtin/input/generate_test.go +++ b/operator/builtin/input/generate_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -16,7 +16,7 @@ import ( func TestInputGenerate(t *testing.T) { count := 5 basicConfig := func() *GenerateInputConfig { - cfg := NewGenerateInputConfig("test_plugin_id") + cfg := NewGenerateInputConfig("test_operator_id") cfg.OutputIDs = []string{"output1"} cfg.Entry = entry.Entry{ Record: "test message", @@ -26,17 +26,17 @@ func TestInputGenerate(t *testing.T) { } buildContext := testutil.NewBuildContext(t) - newPlugin, err := basicConfig().Build(buildContext) + newOperator, err := basicConfig().Build(buildContext) require.NoError(t, err) receivedEntries := make(chan *entry.Entry) - mockOutput := testutil.NewMockPlugin("output1") + mockOutput := testutil.NewMockOperator("output1") mockOutput.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { receivedEntries <- args.Get(1).(*entry.Entry) }) - generateInput := newPlugin.(*GenerateInput) - err = generateInput.SetOutputs([]plugin.Plugin{mockOutput}) + generateInput := newOperator.(*GenerateInput) + err = generateInput.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) err = generateInput.Start() @@ -53,7 +53,7 @@ func TestInputGenerate(t *testing.T) { } } -func TestRenderFromCustom(t *testing.T) { +func TestRenderFromPluginTemplate(t *testing.T) { templateText := ` pipeline: - id: my_generator @@ -66,7 +66,7 @@ pipeline: tmpl, err := template.New("my_generator").Parse(templateText) require.NoError(t, err) - registry := plugin.CustomRegistry{ + registry := operator.PluginRegistry{ "sample": tmpl, } @@ -76,16 +76,16 @@ pipeline: config, err := registry.Render("sample", params) require.NoError(t, err) - expectedConfig := plugin.CustomConfig{ - Pipeline: []plugin.Config{ + expectedConfig := operator.PluginConfig{ + Pipeline: []operator.Config{ { Builder: &GenerateInputConfig{ InputConfig: helper.InputConfig{ WriteTo: entry.NewRecordField(), WriterConfig: helper.WriterConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "my_generator", - PluginType: "generate_input", + OperatorID: "my_generator", + OperatorType: "generate_input", }, OutputIDs: []string{"sampleoutput"}, }, diff --git a/operator/builtin/input/input.go b/operator/builtin/input/input.go new file mode 100644 index 000000000..dc5ce2719 --- /dev/null +++ b/operator/builtin/input/input.go @@ -0,0 +1,6 @@ +package input + +import ( + // Load file package when importing input operators + _ "github.com/observiq/carbon/operator/builtin/input/file" +) diff --git a/plugin/builtin/input/journald.go b/operator/builtin/input/journald.go similarity index 65% rename from plugin/builtin/input/journald.go rename to operator/builtin/input/journald.go index 1eb3faecb..41f4d80c1 100644 --- a/plugin/builtin/input/journald.go +++ b/operator/builtin/input/journald.go @@ -15,22 +15,22 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" ) func init() { - plugin.Register("journald_input", func() plugin.Builder { return NewJournaldInputConfig("") }) + operator.Register("journald_input", func() operator.Builder { return NewJournaldInputConfig("") }) } -func NewJournaldInputConfig(pluginID string) *JournaldInputConfig { +func NewJournaldInputConfig(operatorID string) *JournaldInputConfig { return &JournaldInputConfig{ - InputConfig: helper.NewInputConfig(pluginID, "journald_input"), + InputConfig: helper.NewInputConfig(operatorID, "journald_input"), } } -// JournaldInputConfig is the configuration of a journald input plugin +// JournaldInputConfig is the configuration of a journald input operator type JournaldInputConfig struct { helper.InputConfig `yaml:",inline"` @@ -38,9 +38,9 @@ type JournaldInputConfig struct { Files []string `json:"files,omitempty" yaml:"files,omitempty"` } -// Build will build a journald input plugin from the supplied configuration -func (c JournaldInputConfig) Build(buildContext plugin.BuildContext) (plugin.Plugin, error) { - inputPlugin, err := c.InputConfig.Build(buildContext) +// Build will build a journald input operator from the supplied configuration +func (c JournaldInputConfig) Build(buildContext operator.BuildContext) (operator.Operator, error) { + inputOperator, err := c.InputConfig.Build(buildContext) if err != nil { return nil, err } @@ -66,8 +66,8 @@ func (c JournaldInputConfig) Build(buildContext plugin.BuildContext) (plugin.Plu } journaldInput := &JournaldInput{ - InputPlugin: inputPlugin, - persist: helper.NewScopedDBPersister(buildContext.Database, c.ID()), + InputOperator: inputOperator, + persist: helper.NewScopedDBPersister(buildContext.Database, c.ID()), newCmd: func(ctx context.Context, cursor []byte) cmd { if cursor != nil { args = append(args, "--after-cursor", string(cursor)) @@ -79,9 +79,9 @@ func (c JournaldInputConfig) Build(buildContext plugin.BuildContext) (plugin.Plu return journaldInput, nil } -// JournaldInput is a plugin that process logs using journald +// JournaldInput is an operator that process logs using journald type JournaldInput struct { - helper.InputPlugin + helper.InputOperator newCmd func(ctx context.Context, cursor []byte) cmd @@ -99,21 +99,21 @@ type cmd interface { var lastReadCursorKey = "lastReadCursor" // Start will start generating log entries. -func (plugin *JournaldInput) Start() error { +func (operator *JournaldInput) Start() error { ctx, cancel := context.WithCancel(context.Background()) - plugin.cancel = cancel - plugin.wg = &sync.WaitGroup{} + operator.cancel = cancel + operator.wg = &sync.WaitGroup{} - err := plugin.persist.Load() + err := operator.persist.Load() if err != nil { return err } // Start from a cursor if there is a saved offset - cursor := plugin.persist.Get(lastReadCursorKey) + cursor := operator.persist.Get(lastReadCursorKey) // Start journalctl - cmd := plugin.newCmd(ctx, cursor) + cmd := operator.newCmd(ctx, cursor) stdout, err := cmd.StdoutPipe() if err != nil { return fmt.Errorf("failed to get journalctl stdout: %s", err) @@ -124,24 +124,24 @@ func (plugin *JournaldInput) Start() error { } // Start a goroutine to periodically flush the offsets - plugin.wg.Add(1) + operator.wg.Add(1) go func() { - defer plugin.wg.Done() + defer operator.wg.Done() for { select { case <-ctx.Done(): return case <-time.After(time.Second): - plugin.syncOffsets() + operator.syncOffsets() } } }() // Start the reader goroutine - plugin.wg.Add(1) + operator.wg.Add(1) go func() { - defer plugin.wg.Done() - defer plugin.syncOffsets() + defer operator.wg.Done() + defer operator.syncOffsets() stdoutBuf := bufio.NewReader(stdout) @@ -149,27 +149,27 @@ func (plugin *JournaldInput) Start() error { line, err := stdoutBuf.ReadBytes('\n') if err != nil { if err != io.EOF { - plugin.Errorw("Received error reading from journalctl stdout", zap.Error(err)) + operator.Errorw("Received error reading from journalctl stdout", zap.Error(err)) } return } - entry, cursor, err := plugin.parseJournalEntry(line) + entry, cursor, err := operator.parseJournalEntry(line) if err != nil { - plugin.Warnw("Failed to parse journal entry", zap.Error(err)) + operator.Warnw("Failed to parse journal entry", zap.Error(err)) continue } - plugin.persist.Set(lastReadCursorKey, []byte(cursor)) - plugin.Write(ctx, entry) + operator.persist.Set(lastReadCursorKey, []byte(cursor)) + operator.Write(ctx, entry) } }() return nil } -func (plugin *JournaldInput) parseJournalEntry(line []byte) (*entry.Entry, string, error) { +func (operator *JournaldInput) parseJournalEntry(line []byte) (*entry.Entry, string, error) { var record map[string]interface{} - err := plugin.json.Unmarshal(line, &record) + err := operator.json.Unmarshal(line, &record) if err != nil { return nil, "", err } @@ -201,21 +201,21 @@ func (plugin *JournaldInput) parseJournalEntry(line []byte) (*entry.Entry, strin return nil, "", errors.New("journald field for cursor is not a string") } - entry := plugin.NewEntry(record) + entry := operator.NewEntry(record) entry.Timestamp = time.Unix(0, timestampInt*1000) // in microseconds return entry, cursorString, nil } -func (plugin *JournaldInput) syncOffsets() { - err := plugin.persist.Sync() +func (operator *JournaldInput) syncOffsets() { + err := operator.persist.Sync() if err != nil { - plugin.Errorw("Failed to sync offsets", zap.Error(err)) + operator.Errorw("Failed to sync offsets", zap.Error(err)) } } // Stop will stop generating logs. -func (plugin *JournaldInput) Stop() error { - plugin.cancel() - plugin.wg.Wait() +func (operator *JournaldInput) Stop() error { + operator.cancel() + operator.wg.Wait() return nil } diff --git a/plugin/builtin/input/journald_test.go b/operator/builtin/input/journald_test.go similarity index 96% rename from plugin/builtin/input/journald_test.go rename to operator/builtin/input/journald_test.go index b0d5cafb9..2c63e8ec1 100644 --- a/plugin/builtin/input/journald_test.go +++ b/operator/builtin/input/journald_test.go @@ -12,7 +12,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -37,13 +37,13 @@ func TestInputJournald(t *testing.T) { journaldInput, err := cfg.Build(testutil.NewBuildContext(t)) require.NoError(t, err) - mockOutput := testutil.NewMockPlugin("output") + mockOutput := testutil.NewMockOperator("output") received := make(chan *entry.Entry) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { received <- args.Get(1).(*entry.Entry) }).Return(nil) - err = journaldInput.SetOutputs([]plugin.Plugin{mockOutput}) + err = journaldInput.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) journaldInput.(*JournaldInput).newCmd = func(ctx context.Context, cursor []byte) cmd { diff --git a/plugin/builtin/input/tcp.go b/operator/builtin/input/tcp.go similarity index 81% rename from plugin/builtin/input/tcp.go rename to operator/builtin/input/tcp.go index 6a48fbd04..215ff1e93 100644 --- a/plugin/builtin/input/tcp.go +++ b/operator/builtin/input/tcp.go @@ -8,30 +8,30 @@ import ( "net" "sync" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("tcp_input", func() plugin.Builder { return NewTCPInputConfig("") }) + operator.Register("tcp_input", func() operator.Builder { return NewTCPInputConfig("") }) } -func NewTCPInputConfig(pluginID string) *TCPInputConfig { +func NewTCPInputConfig(operatorID string) *TCPInputConfig { return &TCPInputConfig{ - InputConfig: helper.NewInputConfig(pluginID, "tcp_input"), + InputConfig: helper.NewInputConfig(operatorID, "tcp_input"), } } -// TCPInputConfig is the configuration of a tcp input plugin. +// TCPInputConfig is the configuration of a tcp input operator. type TCPInputConfig struct { helper.InputConfig `yaml:",inline"` ListenAddress string `json:"listen_address,omitempty" yaml:"listen_address,omitempty"` } -// Build will build a tcp input plugin. -func (c TCPInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - inputPlugin, err := c.InputConfig.Build(context) +// Build will build a tcp input operator. +func (c TCPInputConfig) Build(context operator.BuildContext) (operator.Operator, error) { + inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err } @@ -46,15 +46,15 @@ func (c TCPInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error } tcpInput := &TCPInput{ - InputPlugin: inputPlugin, - address: address, + InputOperator: inputOperator, + address: address, } return tcpInput, nil } -// TCPInput is a plugin that listens for log entries over tcp. +// TCPInput is an operator that listens for log entries over tcp. type TCPInput struct { - helper.InputPlugin + helper.InputOperator address *net.TCPAddr listener *net.TCPListener diff --git a/plugin/builtin/input/tcp_test.go b/operator/builtin/input/tcp_test.go similarity index 80% rename from plugin/builtin/input/tcp_test.go rename to operator/builtin/input/tcp_test.go index 8fd43cc76..cd0cb0f08 100644 --- a/plugin/builtin/input/tcp_test.go +++ b/operator/builtin/input/tcp_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -20,8 +20,8 @@ func TestTCPInput(t *testing.T) { WriteTo: entry.NewRecordField(), WriterConfig: helper.WriterConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "test_id", - PluginType: "tcp_input", + OperatorID: "test_id", + OperatorType: "tcp_input", }, OutputIDs: []string{"test_output_id"}, }, @@ -34,12 +34,12 @@ func TestTCPInput(t *testing.T) { cfg.ListenAddress = "127.0.0.1:64001" buildContext := testutil.NewBuildContext(t) - newPlugin, err := cfg.Build(buildContext) + newOperator, err := cfg.Build(buildContext) require.NoError(t, err) - mockOutput := testutil.Plugin{} - tcpInput := newPlugin.(*TCPInput) - tcpInput.InputPlugin.OutputPlugins = []plugin.Plugin{&mockOutput} + mockOutput := testutil.Operator{} + tcpInput := newOperator.(*TCPInput) + tcpInput.InputOperator.OutputOperators = []operator.Operator{&mockOutput} entryChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { diff --git a/plugin/builtin/input/udp.go b/operator/builtin/input/udp.go similarity index 76% rename from plugin/builtin/input/udp.go rename to operator/builtin/input/udp.go index b49034d11..7080cd917 100644 --- a/plugin/builtin/input/udp.go +++ b/operator/builtin/input/udp.go @@ -7,30 +7,30 @@ import ( "strings" "sync" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("udp_input", func() plugin.Builder { return NewUDPInputConfig("") }) + operator.Register("udp_input", func() operator.Builder { return NewUDPInputConfig("") }) } -func NewUDPInputConfig(pluginID string) *UDPInputConfig { +func NewUDPInputConfig(operatorID string) *UDPInputConfig { return &UDPInputConfig{ - InputConfig: helper.NewInputConfig(pluginID, "udp_input"), + InputConfig: helper.NewInputConfig(operatorID, "udp_input"), } } -// UDPInputConfig is the configuration of a udp input plugin. +// UDPInputConfig is the configuration of a udp input operator. type UDPInputConfig struct { helper.InputConfig `yaml:",inline"` ListenAddress string `json:"listen_address,omitempty" yaml:"listen_address,omitempty"` } -// Build will build a udp input plugin. -func (c UDPInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - inputPlugin, err := c.InputConfig.Build(context) +// Build will build a udp input operator. +func (c UDPInputConfig) Build(context operator.BuildContext) (operator.Operator, error) { + inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err } @@ -45,15 +45,15 @@ func (c UDPInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error } udpInput := &UDPInput{ - InputPlugin: inputPlugin, - address: address, + InputOperator: inputOperator, + address: address, } return udpInput, nil } -// UDPInput is a plugin that listens to a socket for log entries. +// UDPInput is an operator that listens to a socket for log entries. type UDPInput struct { - helper.InputPlugin + helper.InputOperator address *net.UDPAddr connection net.PacketConn diff --git a/plugin/builtin/input/udp_test.go b/operator/builtin/input/udp_test.go similarity index 80% rename from plugin/builtin/input/udp_test.go rename to operator/builtin/input/udp_test.go index 0e160e1bf..cf2af503e 100644 --- a/plugin/builtin/input/udp_test.go +++ b/operator/builtin/input/udp_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -20,8 +20,8 @@ func TestUDPInput(t *testing.T) { WriteTo: entry.NewRecordField(), WriterConfig: helper.WriterConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "test_id", - PluginType: "udp_input", + OperatorID: "test_id", + OperatorType: "udp_input", }, OutputIDs: []string{"test_output_id"}, }, @@ -34,14 +34,14 @@ func TestUDPInput(t *testing.T) { cfg.ListenAddress = "127.0.0.1:63001" buildContext := testutil.NewBuildContext(t) - newPlugin, err := cfg.Build(buildContext) + newOperator, err := cfg.Build(buildContext) require.NoError(t, err) - mockOutput := testutil.Plugin{} - udpInput, ok := newPlugin.(*UDPInput) + mockOutput := testutil.Operator{} + udpInput, ok := newOperator.(*UDPInput) require.True(t, ok) - udpInput.InputPlugin.OutputPlugins = []plugin.Plugin{&mockOutput} + udpInput.InputOperator.OutputOperators = []operator.Operator{&mockOutput} entryChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { diff --git a/operator/builtin/output/drop.go b/operator/builtin/output/drop.go new file mode 100644 index 000000000..51ce22983 --- /dev/null +++ b/operator/builtin/output/drop.go @@ -0,0 +1,48 @@ +package output + +import ( + "context" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" +) + +func init() { + operator.Register("drop_output", func() operator.Builder { return NewDropOutputConfig("") }) +} + +func NewDropOutputConfig(operatorID string) *DropOutputConfig { + return &DropOutputConfig{ + OutputConfig: helper.NewOutputConfig(operatorID, "drop_output"), + } +} + +// DropOutputConfig is the configuration of a drop output operator. +type DropOutputConfig struct { + helper.OutputConfig `yaml:",inline"` +} + +// Build will build a drop output operator. +func (c DropOutputConfig) Build(context operator.BuildContext) (operator.Operator, error) { + outputOperator, err := c.OutputConfig.Build(context) + if err != nil { + return nil, err + } + + dropOutput := &DropOutput{ + OutputOperator: outputOperator, + } + + return dropOutput, nil +} + +// DropOutput is an operator that consumes and ignores incoming entries. +type DropOutput struct { + helper.OutputOperator +} + +// Process will drop the incoming entry. +func (p *DropOutput) Process(ctx context.Context, entry *entry.Entry) error { + return nil +} diff --git a/operator/builtin/output/drop_test.go b/operator/builtin/output/drop_test.go new file mode 100644 index 000000000..0c3a125ac --- /dev/null +++ b/operator/builtin/output/drop_test.go @@ -0,0 +1,16 @@ +package output + +import ( + "github.com/observiq/carbon/operator/helper" +) + +func newFakeNullOutput() *DropOutput { + return &DropOutput{ + OutputOperator: helper.OutputOperator{ + BasicOperator: helper.BasicOperator{ + OperatorID: "testnull", + OperatorType: "drop_output", + }, + }, + } +} diff --git a/plugin/builtin/output/elastic.go b/operator/builtin/output/elastic.go similarity index 84% rename from plugin/builtin/output/elastic.go rename to operator/builtin/output/elastic.go index 6115c32fd..91e8a21d1 100644 --- a/plugin/builtin/output/elastic.go +++ b/operator/builtin/output/elastic.go @@ -11,24 +11,24 @@ import ( uuid "github.com/hashicorp/go-uuid" "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/buffer" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/buffer" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" ) func init() { - plugin.Register("elastic_output", func() plugin.Builder { return NewElasticOutputConfig("") }) + operator.Register("elastic_output", func() operator.Builder { return NewElasticOutputConfig("") }) } -func NewElasticOutputConfig(pluginID string) *ElasticOutputConfig { +func NewElasticOutputConfig(operatorID string) *ElasticOutputConfig { return &ElasticOutputConfig{ - OutputConfig: helper.NewOutputConfig(pluginID, "elastic_output"), + OutputConfig: helper.NewOutputConfig(operatorID, "elastic_output"), BufferConfig: buffer.NewConfig(), } } -// ElasticOutputConfig is the configuration of an elasticsearch output plugin. +// ElasticOutputConfig is the configuration of an elasticsearch output operator. type ElasticOutputConfig struct { helper.OutputConfig `yaml:",inline"` BufferConfig buffer.Config `json:"buffer" yaml:"buffer"` @@ -42,9 +42,9 @@ type ElasticOutputConfig struct { IDField *entry.Field `json:"id_field,omitempty" yaml:"id_field,omitempty"` } -// Build will build an elasticsearch output plugin. -func (c ElasticOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - outputPlugin, err := c.OutputConfig.Build(context) +// Build will build an elasticsearch output operator. +func (c ElasticOutputConfig) Build(context operator.BuildContext) (operator.Operator, error) { + outputOperator, err := c.OutputConfig.Build(context) if err != nil { return nil, err } @@ -72,11 +72,11 @@ func (c ElasticOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, } elasticOutput := &ElasticOutput{ - OutputPlugin: outputPlugin, - Buffer: buffer, - client: client, - indexField: c.IndexField, - idField: c.IDField, + OutputOperator: outputOperator, + Buffer: buffer, + client: client, + indexField: c.IndexField, + idField: c.IDField, } buffer.SetHandler(elasticOutput) @@ -84,9 +84,9 @@ func (c ElasticOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, return elasticOutput, nil } -// ElasticOutput is a plugin that sends entries to elasticsearch. +// ElasticOutput is an operator that sends entries to elasticsearch. type ElasticOutput struct { - helper.OutputPlugin + helper.OutputOperator buffer.Buffer client *elasticsearch.Client diff --git a/plugin/builtin/output/elastic_test.go b/operator/builtin/output/elastic_test.go similarity index 100% rename from plugin/builtin/output/elastic_test.go rename to operator/builtin/output/elastic_test.go diff --git a/plugin/builtin/output/file.go b/operator/builtin/output/file.go similarity index 68% rename from plugin/builtin/output/file.go rename to operator/builtin/output/file.go index 1734362d7..13ad7ce2d 100644 --- a/plugin/builtin/output/file.go +++ b/operator/builtin/output/file.go @@ -9,21 +9,21 @@ import ( "sync" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("file_output", func() plugin.Builder { return NewFileOutputConfig("") }) + operator.Register("file_output", func() operator.Builder { return NewFileOutputConfig("") }) } -func NewFileOutputConfig(pluginID string) *FileOutputConfig { +func NewFileOutputConfig(operatorID string) *FileOutputConfig { return &FileOutputConfig{ - OutputConfig: helper.NewOutputConfig(pluginID, "file_output"), + OutputConfig: helper.NewOutputConfig(operatorID, "file_output"), } } -// FileOutputConfig is the configuration of a file output pluginn. +// FileOutputConfig is the configuration of a file output operatorn. type FileOutputConfig struct { helper.OutputConfig `yaml:",inline"` @@ -31,9 +31,9 @@ type FileOutputConfig struct { Format string `json:"format,omitempty" path:"format,omitempty"` } -// Build will build a file output plugin. -func (c FileOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - outputPlugin, err := c.OutputConfig.Build(context) +// Build will build a file output operator. +func (c FileOutputConfig) Build(context operator.BuildContext) (operator.Operator, error) { + outputOperator, err := c.OutputConfig.Build(context) if err != nil { return nil, err } @@ -51,17 +51,17 @@ func (c FileOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, err } fileOutput := &FileOutput{ - OutputPlugin: outputPlugin, - path: c.Path, - tmpl: tmpl, + OutputOperator: outputOperator, + path: c.Path, + tmpl: tmpl, } return fileOutput, nil } -// FileOutput is a plugin that writes logs to a file. +// FileOutput is an operator that writes logs to a file. type FileOutput struct { - helper.OutputPlugin + helper.OutputOperator path string tmpl *template.Template diff --git a/plugin/builtin/output/google_cloud.go b/operator/builtin/output/google_cloud.go similarity index 87% rename from plugin/builtin/output/google_cloud.go rename to operator/builtin/output/google_cloud.go index ae439f0e2..5c8530988 100644 --- a/plugin/builtin/output/google_cloud.go +++ b/operator/builtin/output/google_cloud.go @@ -13,9 +13,9 @@ import ( structpb "github.com/golang/protobuf/ptypes/struct" gax "github.com/googleapis/gax-go" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/buffer" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/buffer" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" "golang.org/x/oauth2/google" "google.golang.org/api/option" @@ -25,34 +25,34 @@ import ( ) func init() { - plugin.Register("google_cloud_output", func() plugin.Builder { return NewGoogleCloudOutputConfig("") }) + operator.Register("google_cloud_output", func() operator.Builder { return NewGoogleCloudOutputConfig("") }) } -func NewGoogleCloudOutputConfig(pluginID string) *GoogleCloudOutputConfig { +func NewGoogleCloudOutputConfig(operatorID string) *GoogleCloudOutputConfig { return &GoogleCloudOutputConfig{ - OutputConfig: helper.NewOutputConfig(pluginID, "google_cloud_output"), + OutputConfig: helper.NewOutputConfig(operatorID, "google_cloud_output"), BufferConfig: buffer.NewConfig(), - Timeout: plugin.Duration{Duration: 10 * time.Second}, + Timeout: operator.Duration{Duration: 10 * time.Second}, } } -// GoogleCloudOutputConfig is the configuration of a google cloud output plugin. +// GoogleCloudOutputConfig is the configuration of a google cloud output operator. type GoogleCloudOutputConfig struct { helper.OutputConfig `yaml:",inline"` BufferConfig buffer.Config `json:"buffer,omitempty" yaml:"buffer,omitempty"` - Credentials string `json:"credentials,omitempty" yaml:"credentials,omitempty"` - CredentialsFile string `json:"credentials_file,omitempty" yaml:"credentials_file,omitempty"` - ProjectID string `json:"project_id" yaml:"project_id"` - LogNameField *entry.Field `json:"log_name_field,omitempty" yaml:"log_name_field,omitempty"` - TraceField *entry.Field `json:"trace_field,omitempty" yaml:"trace_field,omitempty"` - SpanIDField *entry.Field `json:"span_id_field,omitempty" yaml:"span_id_field,omitempty"` - Timeout plugin.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"` + Credentials string `json:"credentials,omitempty" yaml:"credentials,omitempty"` + CredentialsFile string `json:"credentials_file,omitempty" yaml:"credentials_file,omitempty"` + ProjectID string `json:"project_id" yaml:"project_id"` + LogNameField *entry.Field `json:"log_name_field,omitempty" yaml:"log_name_field,omitempty"` + TraceField *entry.Field `json:"trace_field,omitempty" yaml:"trace_field,omitempty"` + SpanIDField *entry.Field `json:"span_id_field,omitempty" yaml:"span_id_field,omitempty"` + Timeout operator.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"` } -// Build will build a google cloud output plugin. -func (c GoogleCloudOutputConfig) Build(buildContext plugin.BuildContext) (plugin.Plugin, error) { - outputPlugin, err := c.OutputConfig.Build(buildContext) +// Build will build a google cloud output operator. +func (c GoogleCloudOutputConfig) Build(buildContext operator.BuildContext) (operator.Operator, error) { + outputOperator, err := c.OutputConfig.Build(buildContext) if err != nil { return nil, err } @@ -63,7 +63,7 @@ func (c GoogleCloudOutputConfig) Build(buildContext plugin.BuildContext) (plugin } googleCloudOutput := &GoogleCloudOutput{ - OutputPlugin: outputPlugin, + OutputOperator: outputOperator, credentials: c.Credentials, credentialsFile: c.CredentialsFile, projectID: c.ProjectID, @@ -79,9 +79,9 @@ func (c GoogleCloudOutputConfig) Build(buildContext plugin.BuildContext) (plugin return googleCloudOutput, nil } -// GoogleCloudOutput is a plugin that sends logs to google cloud logging. +// GoogleCloudOutput is an operator that sends logs to google cloud logging. type GoogleCloudOutput struct { - helper.OutputPlugin + helper.OutputOperator buffer.Buffer credentials string diff --git a/plugin/builtin/output/google_cloud_test.go b/operator/builtin/output/google_cloud_test.go similarity index 98% rename from plugin/builtin/output/google_cloud_test.go rename to operator/builtin/output/google_cloud_test.go index b002258af..61a13c2aa 100644 --- a/plugin/builtin/output/google_cloud_test.go +++ b/operator/builtin/output/google_cloud_test.go @@ -11,7 +11,7 @@ import ( gax "github.com/googleapis/gax-go" "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/require" "google.golang.org/genproto/googleapis/api/monitoredres" sev "google.golang.org/genproto/googleapis/logging/type" @@ -43,7 +43,7 @@ type googleCloudTestCase struct { func googleCloudBasicConfig() *GoogleCloudOutputConfig { cfg := NewGoogleCloudOutputConfig("test_id") cfg.ProjectID = "test_project_id" - cfg.BufferConfig.DelayThreshold = plugin.Duration{Duration: time.Millisecond} + cfg.BufferConfig.DelayThreshold = operator.Duration{Duration: time.Millisecond} return cfg } diff --git a/operator/builtin/output/stdout.go b/operator/builtin/output/stdout.go new file mode 100644 index 000000000..537e850e2 --- /dev/null +++ b/operator/builtin/output/stdout.go @@ -0,0 +1,64 @@ +package output + +import ( + "context" + "encoding/json" + "io" + "os" + "sync" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" +) + +// Stdout is a global handle to standard output +var Stdout io.Writer = os.Stdout + +func init() { + operator.Register("stdout", func() operator.Builder { return NewStdoutConfig("") }) +} + +func NewStdoutConfig(operatorID string) *StdoutConfig { + return &StdoutConfig{ + OutputConfig: helper.NewOutputConfig(operatorID, "stdout"), + } +} + +// StdoutConfig is the configuration of the Stdout operator +type StdoutConfig struct { + helper.OutputConfig `yaml:",inline"` +} + +// Build will build a stdout operator. +func (c StdoutConfig) Build(context operator.BuildContext) (operator.Operator, error) { + outputOperator, err := c.OutputConfig.Build(context) + if err != nil { + return nil, err + } + + return &StdoutOperator{ + OutputOperator: outputOperator, + encoder: json.NewEncoder(Stdout), + }, nil +} + +// StdoutOperator is an operator that logs entries using stdout. +type StdoutOperator struct { + helper.OutputOperator + encoder *json.Encoder + mux sync.Mutex +} + +// Process will log entries received. +func (o *StdoutOperator) Process(ctx context.Context, entry *entry.Entry) error { + o.mux.Lock() + err := o.encoder.Encode(entry) + if err != nil { + o.mux.Unlock() + o.Errorf("Failed to process entry: %s, $s", err, entry.Record) + return err + } + o.mux.Unlock() + return nil +} diff --git a/plugin/builtin/output/stdout_test.go b/operator/builtin/output/stdout_test.go similarity index 69% rename from plugin/builtin/output/stdout_test.go rename to operator/builtin/output/stdout_test.go index eda366f0f..aa0f890c5 100644 --- a/plugin/builtin/output/stdout_test.go +++ b/operator/builtin/output/stdout_test.go @@ -9,32 +9,32 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/require" ) -func TestStdoutPlugin(t *testing.T) { +func TestStdoutOperator(t *testing.T) { cfg := StdoutConfig{ OutputConfig: helper.OutputConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "test_plugin_id", - PluginType: "stdout", + OperatorID: "test_operator_id", + OperatorType: "stdout", }, }, } - plugin, err := cfg.Build(testutil.NewBuildContext(t)) + operator, err := cfg.Build(testutil.NewBuildContext(t)) require.NoError(t, err) var buf bytes.Buffer - plugin.(*StdoutPlugin).encoder = json.NewEncoder(&buf) + operator.(*StdoutOperator).encoder = json.NewEncoder(&buf) ts := time.Unix(1591042864, 0) e := &entry.Entry{ Timestamp: ts, Record: "test record", } - err = plugin.Process(context.Background(), e) + err = operator.Process(context.Background(), e) require.NoError(t, err) marshalledTimestamp, err := json.Marshal(ts) diff --git a/plugin/builtin/parser/json.go b/operator/builtin/parser/json.go similarity index 56% rename from plugin/builtin/parser/json.go rename to operator/builtin/parser/json.go index 497a69c90..5cc4a028e 100644 --- a/plugin/builtin/parser/json.go +++ b/operator/builtin/parser/json.go @@ -6,49 +6,49 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("json_parser", func() plugin.Builder { return NewJSONParserConfig("") }) + operator.Register("json_parser", func() operator.Builder { return NewJSONParserConfig("") }) } -func NewJSONParserConfig(pluginID string) *JSONParserConfig { +func NewJSONParserConfig(operatorID string) *JSONParserConfig { return &JSONParserConfig{ - ParserConfig: helper.NewParserConfig(pluginID, "json_parser"), + ParserConfig: helper.NewParserConfig(operatorID, "json_parser"), } } -// JSONParserConfig is the configuration of a JSON parser plugin. +// JSONParserConfig is the configuration of a JSON parser operator. type JSONParserConfig struct { helper.ParserConfig `yaml:",inline"` } -// Build will build a JSON parser plugin. -func (c JSONParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - parserPlugin, err := c.ParserConfig.Build(context) +// Build will build a JSON parser operator. +func (c JSONParserConfig) Build(context operator.BuildContext) (operator.Operator, error) { + parserOperator, err := c.ParserConfig.Build(context) if err != nil { return nil, err } jsonParser := &JSONParser{ - ParserPlugin: parserPlugin, - json: jsoniter.ConfigFastest, + ParserOperator: parserOperator, + json: jsoniter.ConfigFastest, } return jsonParser, nil } -// JSONParser is a plugin that parses JSON. +// JSONParser is an operator that parses JSON. type JSONParser struct { - helper.ParserPlugin + helper.ParserOperator json jsoniter.API } // Process will parse an entry for JSON. func (j *JSONParser) Process(ctx context.Context, entry *entry.Entry) error { - return j.ParserPlugin.ProcessWith(ctx, entry, j.parse) + return j.ParserOperator.ProcessWith(ctx, entry, j.parse) } // parse will parse a value as JSON. diff --git a/plugin/builtin/parser/json_test.go b/operator/builtin/parser/json_test.go similarity index 82% rename from plugin/builtin/parser/json_test.go rename to operator/builtin/parser/json_test.go index d143fbbc9..cf900ca8a 100644 --- a/plugin/builtin/parser/json_test.go +++ b/operator/builtin/parser/json_test.go @@ -8,27 +8,27 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.uber.org/zap" ) -func NewFakeJSONPlugin() (*JSONParser, *testutil.Plugin) { - mock := testutil.Plugin{} +func NewFakeJSONOperator() (*JSONParser, *testutil.Operator) { + mock := testutil.Operator{} logger, _ := zap.NewProduction() return &JSONParser{ - ParserPlugin: helper.ParserPlugin{ - TransformerPlugin: helper.TransformerPlugin{ - WriterPlugin: helper.WriterPlugin{ - BasicPlugin: helper.BasicPlugin{ - PluginID: "test", - PluginType: "json_parser", + ParserOperator: helper.ParserOperator{ + TransformerOperator: helper.TransformerOperator{ + WriterOperator: helper.WriterOperator{ + BasicOperator: helper.BasicOperator{ + OperatorID: "test", + OperatorType: "json_parser", SugaredLogger: logger.Sugar(), }, - OutputPlugins: []plugin.Plugin{&mock}, + OutputOperators: []operator.Operator{&mock}, }, }, ParseFrom: entry.NewRecordField("testfield"), @@ -39,7 +39,7 @@ func NewFakeJSONPlugin() (*JSONParser, *testutil.Plugin) { } func TestJSONImplementations(t *testing.T) { - require.Implements(t, (*plugin.Plugin)(nil), new(JSONParser)) + require.Implements(t, (*operator.Operator)(nil), new(JSONParser)) } func TestJSONParser(t *testing.T) { @@ -81,7 +81,7 @@ func TestJSONParser(t *testing.T) { output := entry.New() output.Record = tc.expectedRecord - parser, mockOutput := NewFakeJSONPlugin() + parser, mockOutput := NewFakeJSONOperator() mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { e := args[1].(*entry.Entry) require.Equal(t, tc.expectedRecord, e.Record) @@ -151,9 +151,9 @@ func TestJSONParserWithEmbeddedTimeParser(t *testing.T) { output := entry.New() output.Record = tc.expectedRecord - parser, mockOutput := NewFakeJSONPlugin() + parser, mockOutput := NewFakeJSONOperator() parseFrom := entry.NewRecordField("testparsed", "timestamp") - parser.ParserPlugin.TimeParser = &helper.TimeParser{ + parser.ParserOperator.TimeParser = &helper.TimeParser{ ParseFrom: &parseFrom, LayoutType: "epoch", Layout: "s", diff --git a/plugin/builtin/parser/regex.go b/operator/builtin/parser/regex.go similarity index 72% rename from plugin/builtin/parser/regex.go rename to operator/builtin/parser/regex.go index 85261ffc4..8e41a3e4d 100644 --- a/plugin/builtin/parser/regex.go +++ b/operator/builtin/parser/regex.go @@ -7,30 +7,30 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("regex_parser", func() plugin.Builder { return NewRegexParserConfig("") }) + operator.Register("regex_parser", func() operator.Builder { return NewRegexParserConfig("") }) } -func NewRegexParserConfig(pluginID string) *RegexParserConfig { +func NewRegexParserConfig(operatorID string) *RegexParserConfig { return &RegexParserConfig{ - ParserConfig: helper.NewParserConfig(pluginID, "regex_parser"), + ParserConfig: helper.NewParserConfig(operatorID, "regex_parser"), } } -// RegexParserConfig is the configuration of a regex parser plugin. +// RegexParserConfig is the configuration of a regex parser operator. type RegexParserConfig struct { helper.ParserConfig `yaml:",inline"` Regex string `json:"regex" yaml:"regex"` } -// Build will build a regex parser plugin. -func (c RegexParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - parserPlugin, err := c.ParserConfig.Build(context) +// Build will build a regex parser operator. +func (c RegexParserConfig) Build(context operator.BuildContext) (operator.Operator, error) { + parserOperator, err := c.ParserConfig.Build(context) if err != nil { return nil, err } @@ -58,22 +58,22 @@ func (c RegexParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, er } regexParser := &RegexParser{ - ParserPlugin: parserPlugin, - regexp: r, + ParserOperator: parserOperator, + regexp: r, } return regexParser, nil } -// RegexParser is a plugin that parses regex in an entry. +// RegexParser is an operator that parses regex in an entry. type RegexParser struct { - helper.ParserPlugin + helper.ParserOperator regexp *regexp.Regexp } // Process will parse an entry for regex. func (r *RegexParser) Process(ctx context.Context, entry *entry.Entry) error { - return r.ParserPlugin.ProcessWith(ctx, entry, r.parse) + return r.ParserOperator.ProcessWith(ctx, entry, r.parse) } // parse will parse a value using the supplied regex. diff --git a/plugin/builtin/parser/regex_test.go b/operator/builtin/parser/regex_test.go similarity index 82% rename from plugin/builtin/parser/regex_test.go rename to operator/builtin/parser/regex_test.go index 01229cb01..41acfa95d 100644 --- a/plugin/builtin/parser/regex_test.go +++ b/operator/builtin/parser/regex_test.go @@ -7,30 +7,30 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -func newFakeRegexParser() (*RegexParser, *testutil.Plugin) { - mockPlugin := testutil.Plugin{} +func newFakeRegexParser() (*RegexParser, *testutil.Operator) { + mockOperator := testutil.Operator{} return &RegexParser{ - ParserPlugin: helper.ParserPlugin{ - TransformerPlugin: helper.TransformerPlugin{ - WriterPlugin: helper.WriterPlugin{ - BasicPlugin: helper.BasicPlugin{ - PluginID: "regex_parser", - PluginType: "regex_parser", + ParserOperator: helper.ParserOperator{ + TransformerOperator: helper.TransformerOperator{ + WriterOperator: helper.WriterOperator{ + BasicOperator: helper.BasicOperator{ + OperatorID: "regex_parser", + OperatorType: "regex_parser", }, - OutputIDs: []string{"mock_output"}, - OutputPlugins: []plugin.Plugin{&mockPlugin}, + OutputIDs: []string{"mock_output"}, + OutputOperators: []operator.Operator{&mockOperator}, }, }, ParseFrom: entry.NewRecordField(), ParseTo: entry.NewRecordField(), }, - }, &mockPlugin + }, &mockOperator } func TestParserRegex(t *testing.T) { diff --git a/operator/builtin/parser/severity.go b/operator/builtin/parser/severity.go new file mode 100644 index 000000000..28c701a6a --- /dev/null +++ b/operator/builtin/parser/severity.go @@ -0,0 +1,63 @@ +package parser + +import ( + "context" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/errors" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" +) + +func init() { + operator.Register("severity_parser", func() operator.Builder { return NewSeverityParserConfig("") }) +} + +func NewSeverityParserConfig(operatorID string) *SeverityParserConfig { + return &SeverityParserConfig{ + TransformerConfig: helper.NewTransformerConfig(operatorID, "severity_parser"), + SeverityParserConfig: helper.NewSeverityParserConfig(), + } +} + +// SeverityParserConfig is the configuration of a severity parser operator. +type SeverityParserConfig struct { + helper.TransformerConfig `yaml:",inline"` + helper.SeverityParserConfig `yaml:",omitempty,inline"` +} + +// Build will build a time parser operator. +func (c SeverityParserConfig) Build(context operator.BuildContext) (operator.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) + if err != nil { + return nil, err + } + + severityParser, err := c.SeverityParserConfig.Build(context) + if err != nil { + return nil, err + } + + severityOperator := &SeverityParserOperator{ + TransformerOperator: transformerOperator, + SeverityParser: severityParser, + } + + return severityOperator, nil +} + +// SeverityParserOperator is an operator that parses time from a field to an entry. +type SeverityParserOperator struct { + helper.TransformerOperator + helper.SeverityParser +} + +// Process will parse time from an entry. +func (p *SeverityParserOperator) Process(ctx context.Context, entry *entry.Entry) error { + if err := p.Parse(ctx, entry); err != nil { + return errors.Wrap(err, "parse severity") + } + + p.Write(ctx, entry) + return nil +} diff --git a/plugin/builtin/parser/severity_test.go b/operator/builtin/parser/severity_test.go similarity index 94% rename from plugin/builtin/parser/severity_test.go rename to operator/builtin/parser/severity_test.go index 9720461ef..c90ff0aa7 100644 --- a/plugin/builtin/parser/severity_test.go +++ b/operator/builtin/parser/severity_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -304,21 +304,21 @@ func runSeverityParseTest(t *testing.T, cfg *SeverityParserConfig, ent *entry.En return func(t *testing.T) { buildContext := testutil.NewBuildContext(t) - severityPlugin, err := cfg.Build(buildContext) + severityOperator, err := cfg.Build(buildContext) if buildErr { - require.Error(t, err, "expected error when configuring plugin") + require.Error(t, err, "expected error when configuring operator") return } - require.NoError(t, err, "unexpected error when configuring plugin") + require.NoError(t, err, "unexpected error when configuring operator") - mockOutput := &testutil.Plugin{} + mockOutput := &testutil.Operator{} resultChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { resultChan <- args.Get(1).(*entry.Entry) }).Return(nil) - severityParser := severityPlugin.(*SeverityParserPlugin) - severityParser.OutputPlugins = []plugin.Plugin{mockOutput} + severityParser := severityOperator.(*SeverityParserOperator) + severityParser.OutputOperators = []operator.Operator{mockOutput} err = severityParser.Process(context.Background(), ent) if parseErr { @@ -337,7 +337,7 @@ func runSeverityParseTest(t *testing.T, cfg *SeverityParserConfig, ent *entry.En } func parseSeverityTestConfig(parseFrom entry.Field, preset string, mapping map[interface{}]interface{}) *SeverityParserConfig { - cfg := NewSeverityParserConfig("test_plugin_id") + cfg := NewSeverityParserConfig("test_operator_id") cfg.OutputIDs = []string{"output1"} cfg.SeverityParserConfig = helper.SeverityParserConfig{ ParseFrom: &parseFrom, diff --git a/plugin/builtin/parser/syslog.go b/operator/builtin/parser/syslog.go similarity index 84% rename from plugin/builtin/parser/syslog.go rename to operator/builtin/parser/syslog.go index a29fa1893..e6ac61e70 100644 --- a/plugin/builtin/parser/syslog.go +++ b/operator/builtin/parser/syslog.go @@ -9,29 +9,29 @@ import ( "github.com/influxdata/go-syslog/v3/rfc3164" "github.com/influxdata/go-syslog/v3/rfc5424" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("syslog_parser", func() plugin.Builder { return NewSyslogParserConfig("") }) + operator.Register("syslog_parser", func() operator.Builder { return NewSyslogParserConfig("") }) } -func NewSyslogParserConfig(pluginID string) *SyslogParserConfig { +func NewSyslogParserConfig(operatorID string) *SyslogParserConfig { return &SyslogParserConfig{ - ParserConfig: helper.NewParserConfig(pluginID, "syslog_parser"), + ParserConfig: helper.NewParserConfig(operatorID, "syslog_parser"), } } -// SyslogParserConfig is the configuration of a syslog parser plugin. +// SyslogParserConfig is the configuration of a syslog parser operator. type SyslogParserConfig struct { helper.ParserConfig `yaml:",inline"` Protocol string `json:"protocol,omitempty" yaml:"protocol,omitempty"` } -// Build will build a JSON parser plugin. -func (c SyslogParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { +// Build will build a JSON parser operator. +func (c SyslogParserConfig) Build(context operator.BuildContext) (operator.Operator, error) { if c.ParserConfig.TimeParser == nil { parseFromField := entry.NewRecordField("timestamp") c.ParserConfig.TimeParser = &helper.TimeParser{ @@ -40,7 +40,7 @@ func (c SyslogParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, e } } - parserPlugin, err := c.ParserConfig.Build(context) + parserOperator, err := c.ParserConfig.Build(context) if err != nil { return nil, err } @@ -55,8 +55,8 @@ func (c SyslogParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, e } syslogParser := &SyslogParser{ - ParserPlugin: parserPlugin, - machine: machine, + ParserOperator: parserOperator, + machine: machine, } return syslogParser, nil @@ -73,15 +73,15 @@ func buildMachine(protocol string) (syslog.Machine, error) { } } -// SyslogParser is a plugin that parses syslog. +// SyslogParser is an operator that parses syslog. type SyslogParser struct { - helper.ParserPlugin + helper.ParserOperator machine syslog.Machine } // Process will parse an entry field as syslog. func (s *SyslogParser) Process(ctx context.Context, entry *entry.Entry) error { - return s.ParserPlugin.ProcessWith(ctx, entry, s.parse) + return s.ParserOperator.ProcessWith(ctx, entry, s.parse) } // parse will parse a value as syslog. diff --git a/plugin/builtin/parser/syslog_test.go b/operator/builtin/parser/syslog_test.go similarity index 91% rename from plugin/builtin/parser/syslog_test.go rename to operator/builtin/parser/syslog_test.go index ebbd7c312..05f441090 100644 --- a/plugin/builtin/parser/syslog_test.go +++ b/operator/builtin/parser/syslog_test.go @@ -7,14 +7,14 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) func TestSyslogParser(t *testing.T) { basicConfig := func() *SyslogParserConfig { - cfg := NewSyslogParserConfig("test_plugin_id") + cfg := NewSyslogParserConfig("test_operator_id") cfg.OutputIDs = []string{"output1"} return cfg } @@ -96,17 +96,17 @@ func TestSyslogParser(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { buildContext := testutil.NewBuildContext(t) - newPlugin, err := tc.config.Build(buildContext) + newOperator, err := tc.config.Build(buildContext) require.NoError(t, err) - syslogParser := newPlugin.(*SyslogParser) + syslogParser := newOperator.(*SyslogParser) - mockOutput := testutil.NewMockPlugin("output1") + mockOutput := testutil.NewMockOperator("output1") entryChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = syslogParser.SetOutputs([]plugin.Plugin{mockOutput}) + err = syslogParser.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) newEntry := entry.New() diff --git a/operator/builtin/parser/time.go b/operator/builtin/parser/time.go new file mode 100644 index 000000000..9776f9af2 --- /dev/null +++ b/operator/builtin/parser/time.go @@ -0,0 +1,66 @@ +package parser + +import ( + "context" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/errors" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" +) + +func init() { + operator.Register("time_parser", func() operator.Builder { return NewTimeParserConfig("") }) +} + +func NewTimeParserConfig(operatorID string) *TimeParserConfig { + return &TimeParserConfig{ + TransformerConfig: helper.NewTransformerConfig(operatorID, "time_parser"), + TimeParser: helper.NewTimeParser(), + } +} + +// TimeParserConfig is the configuration of a time parser operator. +type TimeParserConfig struct { + helper.TransformerConfig `yaml:",inline"` + helper.TimeParser `yaml:",omitempty,inline"` +} + +// Build will build a time parser operator. +func (c TimeParserConfig) Build(context operator.BuildContext) (operator.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) + if err != nil { + return nil, err + } + + if err := c.TimeParser.Validate(context); err != nil { + return nil, err + } + + timeParser := &TimeParserOperator{ + TransformerOperator: transformerOperator, + TimeParser: c.TimeParser, + } + + return timeParser, nil +} + +// TimeParserOperator is an operator that parses time from a field to an entry. +type TimeParserOperator struct { + helper.TransformerOperator + helper.TimeParser +} + +// CanOutput will always return true for a parser operator. +func (t *TimeParserOperator) CanOutput() bool { + return true +} + +// Process will parse time from an entry. +func (t *TimeParserOperator) Process(ctx context.Context, entry *entry.Entry) error { + if err := t.Parse(ctx, entry); err != nil { + return errors.Wrap(err, "parse timestamp") + } + t.Write(ctx, entry) + return nil +} diff --git a/plugin/builtin/parser/time_test.go b/operator/builtin/parser/time_test.go similarity index 96% rename from plugin/builtin/parser/time_test.go rename to operator/builtin/parser/time_test.go index 37158a63b..deb3b107d 100644 --- a/plugin/builtin/parser/time_test.go +++ b/operator/builtin/parser/time_test.go @@ -8,8 +8,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -392,25 +392,25 @@ func runLossyTimeParseTest(t *testing.T, cfg *TimeParserConfig, ent *entry.Entry return func(t *testing.T) { buildContext := testutil.NewBuildContext(t) - gotimePlugin, err := cfg.Build(buildContext) + gotimeOperator, err := cfg.Build(buildContext) if buildErr { - require.Error(t, err, "expected error when configuring plugin") + require.Error(t, err, "expected error when configuring operator") return } require.NoError(t, err) - mockOutput := &testutil.Plugin{} + mockOutput := &testutil.Operator{} resultChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { resultChan <- args.Get(1).(*entry.Entry) }).Return(nil) - timeParser := gotimePlugin.(*TimeParserPlugin) - timeParser.OutputPlugins = []plugin.Plugin{mockOutput} + timeParser := gotimeOperator.(*TimeParserOperator) + timeParser.OutputOperators = []operator.Operator{mockOutput} err = timeParser.Process(context.Background(), ent) if parseErr { - require.Error(t, err, "expected error when configuring plugin") + require.Error(t, err, "expected error when configuring operator") return } require.NoError(t, err) @@ -426,7 +426,7 @@ func runLossyTimeParseTest(t *testing.T, cfg *TimeParserConfig, ent *entry.Entry } func parseTimeTestConfig(layoutType, layout string, parseFrom entry.Field) *TimeParserConfig { - cfg := NewTimeParserConfig("test_plugin_id") + cfg := NewTimeParserConfig("test_operator_id") cfg.OutputIDs = []string{"output1"} cfg.TimeParser = helper.TimeParser{ LayoutType: layoutType, diff --git a/plugin/builtin/transformer/k8s_metadata_decorator.go b/operator/builtin/transformer/k8s_metadata_decorator.go similarity index 82% rename from plugin/builtin/transformer/k8s_metadata_decorator.go rename to operator/builtin/transformer/k8s_metadata_decorator.go index 7a1df2170..5a7fdf9aa 100644 --- a/plugin/builtin/transformer/k8s_metadata_decorator.go +++ b/operator/builtin/transformer/k8s_metadata_decorator.go @@ -7,52 +7,52 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" ) func init() { - plugin.Register("k8s_metadata_decorator", func() plugin.Builder { return NewK8smetadataDecoratorConfig("") }) + operator.Register("k8s_metadata_decorator", func() operator.Builder { return NewK8smetadataDecoratorConfig("") }) } -func NewK8smetadataDecoratorConfig(pluginID string) *K8sMetadataDecoratorConfig { +func NewK8smetadataDecoratorConfig(operatorID string) *K8sMetadataDecoratorConfig { return &K8sMetadataDecoratorConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "k8s_metadata_decorator"), + TransformerConfig: helper.NewTransformerConfig(operatorID, "k8s_metadata_decorator"), PodNameField: entry.NewRecordField("pod_name"), NamespaceField: entry.NewRecordField("namespace"), - CacheTTL: plugin.Duration{Duration: 10 * time.Minute}, + CacheTTL: operator.Duration{Duration: 10 * time.Minute}, } } -// K8sMetadataDecoratorConfig is the configuration of k8s_metadata_decorator plugin +// K8sMetadataDecoratorConfig is the configuration of k8s_metadata_decorator operator type K8sMetadataDecoratorConfig struct { helper.TransformerConfig `yaml:",inline"` - PodNameField entry.Field `json:"pod_name_field,omitempty" yaml:"pod_name_field,omitempty"` - NamespaceField entry.Field `json:"namespace_field,omitempty" yaml:"namespace_field,omitempty"` - CacheTTL plugin.Duration `json:"cache_ttl,omitempty" yaml:"cache_ttl,omitempty"` + PodNameField entry.Field `json:"pod_name_field,omitempty" yaml:"pod_name_field,omitempty"` + NamespaceField entry.Field `json:"namespace_field,omitempty" yaml:"namespace_field,omitempty"` + CacheTTL operator.Duration `json:"cache_ttl,omitempty" yaml:"cache_ttl,omitempty"` } -// Build will build a k8s_metadata_decorator plugin from the supplied configuration -func (c K8sMetadataDecoratorConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { +// Build will build a k8s_metadata_decorator operator from the supplied configuration +func (c K8sMetadataDecoratorConfig) Build(context operator.BuildContext) (operator.Operator, error) { transformer, err := c.TransformerConfig.Build(context) if err != nil { return nil, errors.Wrap(err, "build transformer") } return &K8sMetadataDecorator{ - TransformerPlugin: transformer, - podNameField: c.PodNameField, - namespaceField: c.NamespaceField, - cacheTTL: c.CacheTTL.Raw(), + TransformerOperator: transformer, + podNameField: c.PodNameField, + namespaceField: c.NamespaceField, + cacheTTL: c.CacheTTL.Raw(), }, nil } -// K8sMetadataDecorator is a plugin for decorating entries with kubernetes metadata +// K8sMetadataDecorator is an operator for decorating entries with kubernetes metadata type K8sMetadataDecorator struct { - helper.TransformerPlugin + helper.TransformerOperator podNameField entry.Field namespaceField entry.Field @@ -89,13 +89,13 @@ func (m *MetadataCache) Store(key string, entry MetadataCacheEntry) { m.m.Store(key, entry) } -// Start will start the k8s_metadata_decorator plugin +// Start will start the k8s_metadata_decorator operator func (k *K8sMetadataDecorator) Start() error { config, err := rest.InClusterConfig() if err != nil { return errors.NewError( "agent not in kubernetes cluster", - "the k8s_metadata_decorator plugin only supports running in a pod inside a kubernetes cluster", + "the k8s_metadata_decorator operator only supports running in a pod inside a kubernetes cluster", ) } @@ -126,7 +126,7 @@ func (k *K8sMetadataDecorator) Start() error { return nil } -// Process will process an entry received by the k8s_metadata_decorator plugin +// Process will process an entry received by the k8s_metadata_decorator operator func (k *K8sMetadataDecorator) Process(ctx context.Context, entry *entry.Entry) error { var podName string err := entry.Read(k.podNameField, &podName) diff --git a/plugin/builtin/transformer/k8s_metadata_decorator_test.go b/operator/builtin/transformer/k8s_metadata_decorator_test.go similarity index 80% rename from plugin/builtin/transformer/k8s_metadata_decorator_test.go rename to operator/builtin/transformer/k8s_metadata_decorator_test.go index 4b76a69f6..076720ced 100644 --- a/plugin/builtin/transformer/k8s_metadata_decorator_test.go +++ b/operator/builtin/transformer/k8s_metadata_decorator_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -31,7 +31,7 @@ func TestMetadataCache(t *testing.T) { } func basicConfig() *K8sMetadataDecoratorConfig { - cfg := NewK8smetadataDecoratorConfig("testplugin") + cfg := NewK8smetadataDecoratorConfig("testoperator") cfg.OutputIDs = []string{"mock"} return cfg } @@ -40,11 +40,11 @@ func TestK8sMetadataDecoratorBuildDefault(t *testing.T) { cfg := basicConfig() expected := &K8sMetadataDecorator{ - TransformerPlugin: helper.TransformerPlugin{ - WriterPlugin: helper.WriterPlugin{ - BasicPlugin: helper.BasicPlugin{ - PluginID: "testplugin", - PluginType: "k8s_metadata_decorator", + TransformerOperator: helper.TransformerOperator{ + WriterOperator: helper.WriterOperator{ + BasicOperator: helper.BasicOperator{ + OperatorID: "testoperator", + OperatorType: "k8s_metadata_decorator", }, OutputIDs: []string{"mock"}, }, @@ -55,10 +55,10 @@ func TestK8sMetadataDecoratorBuildDefault(t *testing.T) { cacheTTL: 10 * time.Minute, } - plugin, err := cfg.Build(testutil.NewBuildContext(t)) - plugin.(*K8sMetadataDecorator).SugaredLogger = nil + operator, err := cfg.Build(testutil.NewBuildContext(t)) + operator.(*K8sMetadataDecorator).SugaredLogger = nil require.NoError(t, err) - require.Equal(t, expected, plugin) + require.Equal(t, expected, operator) } @@ -68,8 +68,8 @@ func TestK8sMetadataDecoratorCachedMetadata(t *testing.T) { pg, err := cfg.Build(testutil.NewBuildContext(t)) require.NoError(t, err) - mockOutput := testutil.NewMockPlugin("mock") - pg.SetOutputs([]plugin.Plugin{mockOutput}) + mockOutput := testutil.NewMockOperator("mock") + pg.SetOutputs([]operator.Operator{mockOutput}) // Preload cache so we don't hit the network k8s := pg.(*K8sMetadataDecorator) diff --git a/plugin/builtin/transformer/metadata.go b/operator/builtin/transformer/metadata.go similarity index 65% rename from plugin/builtin/transformer/metadata.go rename to operator/builtin/transformer/metadata.go index d6c82833c..0c8a06692 100644 --- a/plugin/builtin/transformer/metadata.go +++ b/operator/builtin/transformer/metadata.go @@ -5,31 +5,31 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("metadata", func() plugin.Builder { return NewMetadataPluginConfig("") }) + operator.Register("metadata", func() operator.Builder { return NewMetadataOperatorConfig("") }) } -func NewMetadataPluginConfig(pluginID string) *MetadataPluginConfig { - return &MetadataPluginConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "metadata"), +func NewMetadataOperatorConfig(operatorID string) *MetadataOperatorConfig { + return &MetadataOperatorConfig{ + TransformerConfig: helper.NewTransformerConfig(operatorID, "metadata"), } } -// MetadataPluginConfig is the configuration of a metadata plugin -type MetadataPluginConfig struct { +// MetadataOperatorConfig is the configuration of a metadatan operator +type MetadataOperatorConfig struct { helper.TransformerConfig `yaml:",inline"` Labels map[string]helper.ExprStringConfig `json:"labels" yaml:"labels"` Tags []helper.ExprStringConfig `json:"tags" yaml:"tags"` } -// Build will build a metadata plugin from the supplied configuration -func (c MetadataPluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) +// Build will build a metadatan operator from the supplied configuration +func (c MetadataOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err } @@ -44,29 +44,29 @@ func (c MetadataPluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, return nil, errors.Wrap(err, "validate labels") } - restructurePlugin := &MetadataPlugin{ - TransformerPlugin: transformerPlugin, - labeler: labeler, - tagger: tagger, + restructureOperator := &MetadataOperator{ + TransformerOperator: transformerOperator, + labeler: labeler, + tagger: tagger, } - return restructurePlugin, nil + return restructureOperator, nil } -// MetadataPlugin is a plugin that can add metadata to incoming entries -type MetadataPlugin struct { - helper.TransformerPlugin +// MetadataOperator is an operator that can add metadata to incoming entries +type MetadataOperator struct { + helper.TransformerOperator labeler *labeler tagger *tagger } // Process will process an incoming entry using the metadata transform. -func (p *MetadataPlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (p *MetadataOperator) Process(ctx context.Context, entry *entry.Entry) error { return p.ProcessWith(ctx, entry, p.Transform) } // Transform will transform an entry using the labeler and tagger. -func (p *MetadataPlugin) Transform(entry *entry.Entry) (*entry.Entry, error) { +func (p *MetadataOperator) Transform(entry *entry.Entry) (*entry.Entry, error) { err := p.labeler.Label(entry) if err != nil { return entry, err diff --git a/plugin/builtin/transformer/metadata_test.go b/operator/builtin/transformer/metadata_test.go similarity index 80% rename from plugin/builtin/transformer/metadata_test.go rename to operator/builtin/transformer/metadata_test.go index f17d163de..2bb5c5280 100644 --- a/plugin/builtin/transformer/metadata_test.go +++ b/operator/builtin/transformer/metadata_test.go @@ -8,8 +8,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -18,21 +18,21 @@ func TestMetadata(t *testing.T) { os.Setenv("TEST_METADATA_PLUGIN_ENV", "foo") defer os.Unsetenv("TEST_METADATA_PLUGIN_ENV") - baseConfig := func() *MetadataPluginConfig { - cfg := NewMetadataPluginConfig("test_plugin_id") + baseConfig := func() *MetadataOperatorConfig { + cfg := NewMetadataOperatorConfig("test_operator_id") cfg.OutputIDs = []string{"output1"} return cfg } cases := []struct { name string - config *MetadataPluginConfig + config *MetadataOperatorConfig input *entry.Entry expected *entry.Entry }{ { "AddTagLiteral", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Tags = []helper.ExprStringConfig{"tag1"} return cfg @@ -46,7 +46,7 @@ func TestMetadata(t *testing.T) { }, { "AddTagExpr", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Tags = []helper.ExprStringConfig{`prefix-EXPR( 'test1' )`} return cfg @@ -60,7 +60,7 @@ func TestMetadata(t *testing.T) { }, { "AddLabelLiteral", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Labels = map[string]helper.ExprStringConfig{ "label1": "value1", @@ -78,7 +78,7 @@ func TestMetadata(t *testing.T) { }, { "AddLabelExpr", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Labels = map[string]helper.ExprStringConfig{ "label1": `EXPR("start" + "end")`, @@ -96,7 +96,7 @@ func TestMetadata(t *testing.T) { }, { "AddLabelEnv", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Labels = map[string]helper.ExprStringConfig{ "label1": `EXPR(env("TEST_METADATA_PLUGIN_ENV"))`, @@ -114,7 +114,7 @@ func TestMetadata(t *testing.T) { }, { "AddTagEnv", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Tags = []helper.ExprStringConfig{`EXPR(env("TEST_METADATA_PLUGIN_ENV"))`} return cfg @@ -130,19 +130,19 @@ func TestMetadata(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - metadataPlugin, err := tc.config.Build(testutil.NewBuildContext(t)) + metadataOperator, err := tc.config.Build(testutil.NewBuildContext(t)) require.NoError(t, err) - mockOutput := testutil.NewMockPlugin("output1") + mockOutput := testutil.NewMockOperator("output1") entryChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = metadataPlugin.SetOutputs([]plugin.Plugin{mockOutput}) + err = metadataOperator.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) - err = metadataPlugin.Process(context.Background(), tc.input) + err = metadataOperator.Process(context.Background(), tc.input) require.NoError(t, err) select { diff --git a/operator/builtin/transformer/noop.go b/operator/builtin/transformer/noop.go new file mode 100644 index 000000000..77e75322a --- /dev/null +++ b/operator/builtin/transformer/noop.go @@ -0,0 +1,49 @@ +package transformer + +import ( + "context" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" +) + +func init() { + operator.Register("noop", func() operator.Builder { return NewNoopOperatorConfig("") }) +} + +func NewNoopOperatorConfig(operatorID string) *NoopOperatorConfig { + return &NoopOperatorConfig{ + TransformerConfig: helper.NewTransformerConfig(operatorID, "noop"), + } +} + +// NoopOperatorConfig is the configuration of a noop operator. +type NoopOperatorConfig struct { + helper.TransformerConfig `yaml:",inline"` +} + +// Build will build a noop operator. +func (c NoopOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) + if err != nil { + return nil, err + } + + noopOperator := &NoopOperator{ + TransformerOperator: transformerOperator, + } + + return noopOperator, nil +} + +// NoopOperator is an operator that performs no operations on an entry. +type NoopOperator struct { + helper.TransformerOperator +} + +// Process will forward the entry to the next output without any alterations. +func (p *NoopOperator) Process(ctx context.Context, entry *entry.Entry) error { + p.Write(ctx, entry) + return nil +} diff --git a/plugin/builtin/transformer/noop_test.go b/operator/builtin/transformer/noop_test.go similarity index 72% rename from plugin/builtin/transformer/noop_test.go rename to operator/builtin/transformer/noop_test.go index 8c2fc024a..ba61bff60 100644 --- a/plugin/builtin/transformer/noop_test.go +++ b/operator/builtin/transformer/noop_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/require" ) -func TestNoopPluginBuild(t *testing.T) { - cfg := NewNoopPluginConfig("test_plugin_id") +func TestNoopOperatorBuild(t *testing.T) { + cfg := NewNoopOperatorConfig("test_operator_id") cfg.OutputIDs = []string{"output"} _, err := cfg.Build(testutil.NewBuildContext(t)) diff --git a/operator/builtin/transformer/rate_limit.go b/operator/builtin/transformer/rate_limit.go new file mode 100644 index 000000000..2e97dbde1 --- /dev/null +++ b/operator/builtin/transformer/rate_limit.go @@ -0,0 +1,106 @@ +package transformer + +import ( + "context" + "fmt" + "time" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" +) + +func init() { + operator.Register("rate_limit", func() operator.Builder { return NewRateLimitConfig("") }) +} + +func NewRateLimitConfig(operatorID string) *RateLimitConfig { + return &RateLimitConfig{ + TransformerConfig: helper.NewTransformerConfig(operatorID, "rate_limit"), + } +} + +// RateLimitConfig is the configuration of a rate filter operator. +type RateLimitConfig struct { + helper.TransformerConfig `yaml:",inline"` + + Rate float64 `json:"rate,omitempty" yaml:"rate,omitempty"` + Interval operator.Duration `json:"interval,omitempty" yaml:"interval,omitempty"` + Burst uint `json:"burst,omitempty" yaml:"burst,omitempty"` +} + +// Build will build a rate limit operator. +func (c RateLimitConfig) Build(context operator.BuildContext) (operator.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) + if err != nil { + return nil, err + } + + var interval time.Duration + switch { + case c.Rate != 0 && c.Interval.Raw() != 0: + return nil, fmt.Errorf("only one of 'rate' or 'interval' can be defined") + case c.Rate < 0 || c.Interval.Raw() < 0: + return nil, fmt.Errorf("rate and interval must be greater than zero") + case c.Rate > 0: + interval = time.Second / time.Duration(c.Rate) + default: + interval = c.Interval.Raw() + } + + rateLimitOperator := &RateLimitOperator{ + TransformerOperator: transformerOperator, + interval: interval, + burst: c.Burst, + } + + return rateLimitOperator, nil +} + +// RateLimitOperator is an operator that limits the rate of log consumption between operators. +type RateLimitOperator struct { + helper.TransformerOperator + + interval time.Duration + burst uint + isReady chan struct{} + cancel context.CancelFunc +} + +// Process will wait until a rate is met before sending an entry to the output. +func (p *RateLimitOperator) Process(ctx context.Context, entry *entry.Entry) error { + <-p.isReady + p.Write(ctx, entry) + return nil +} + +// Start will start the rate limit operator. +func (p *RateLimitOperator) Start() error { + p.isReady = make(chan struct{}, p.burst) + ticker := time.NewTicker(p.interval) + + ctx, cancel := context.WithCancel(context.Background()) + p.cancel = cancel + + // Buffer the ticker ticks in isReady to allow bursts + go func() { + defer ticker.Stop() + defer close(p.isReady) + for { + select { + case <-ticker.C: + p.isReady <- struct{}{} + case <-ctx.Done(): + return + } + } + }() + + return nil +} + +// Stop will stop the rate limit operator. +func (p *RateLimitOperator) Stop() error { + p.cancel() + return nil +} diff --git a/plugin/builtin/transformer/rate_limit_test.go b/operator/builtin/transformer/rate_limit_test.go similarity index 88% rename from plugin/builtin/transformer/rate_limit_test.go rename to operator/builtin/transformer/rate_limit_test.go index 70c6e1c6b..859dd75d6 100644 --- a/plugin/builtin/transformer/rate_limit_test.go +++ b/operator/builtin/transformer/rate_limit_test.go @@ -8,7 +8,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -25,12 +25,12 @@ func TestRateLimit(t *testing.T) { require.NoError(t, err) receivedLog := make(chan struct{}, 100) - mockOutput := testutil.NewMockPlugin("output1") + mockOutput := testutil.NewMockOperator("output1") mockOutput.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { receivedLog <- struct{}{} }) - err = rateLimit.SetOutputs([]plugin.Plugin{mockOutput}) + err = rateLimit.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) err = rateLimit.Start() diff --git a/plugin/builtin/transformer/restructure.go b/operator/builtin/transformer/restructure.go similarity index 89% rename from plugin/builtin/transformer/restructure.go rename to operator/builtin/transformer/restructure.go index eb56d11d4..2f1e847ca 100644 --- a/plugin/builtin/transformer/restructure.go +++ b/operator/builtin/transformer/restructure.go @@ -9,55 +9,55 @@ import ( "github.com/antonmedv/expr/vm" "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("restructure", func() plugin.Builder { return NewRestructurePluginConfig("") }) + operator.Register("restructure", func() operator.Builder { return NewRestructureOperatorConfig("") }) } -func NewRestructurePluginConfig(pluginID string) *RestructurePluginConfig { - return &RestructurePluginConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "restructure"), +func NewRestructureOperatorConfig(operatorID string) *RestructureOperatorConfig { + return &RestructureOperatorConfig{ + TransformerConfig: helper.NewTransformerConfig(operatorID, "restructure"), } } -// RestructurePluginConfig is the configuration of a restructure plugin -type RestructurePluginConfig struct { +// RestructureOperatorConfig is the configuration of a restructure operator +type RestructureOperatorConfig struct { helper.TransformerConfig `yaml:",inline"` Ops []Op `json:"ops" yaml:"ops"` } -// Build will build a restructure plugin from the supplied configuration -func (c RestructurePluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) +// Build will build a restructure operator from the supplied configuration +func (c RestructureOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err } - restructurePlugin := &RestructurePlugin{ - TransformerPlugin: transformerPlugin, - ops: c.Ops, + restructureOperator := &RestructureOperator{ + TransformerOperator: transformerOperator, + ops: c.Ops, } - return restructurePlugin, nil + return restructureOperator, nil } -// RestructurePlugin is a plugin that can restructure incoming entries using operations -type RestructurePlugin struct { - helper.TransformerPlugin +// RestructureOperator is an operator that can restructure incoming entries using operations +type RestructureOperator struct { + helper.TransformerOperator ops []Op } // Process will process an entry with a restructure transformation. -func (p *RestructurePlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (p *RestructureOperator) Process(ctx context.Context, entry *entry.Entry) error { return p.ProcessWith(ctx, entry, p.Transform) } // Transform will apply the restructure operations to an entry -func (p *RestructurePlugin) Transform(entry *entry.Entry) (*entry.Entry, error) { +func (p *RestructureOperator) Transform(entry *entry.Entry) (*entry.Entry, error) { for _, op := range p.ops { err := op.Apply(entry) if err != nil { diff --git a/plugin/builtin/transformer/restructure_test.go b/operator/builtin/transformer/restructure_test.go similarity index 90% rename from plugin/builtin/transformer/restructure_test.go rename to operator/builtin/transformer/restructure_test.go index e519a0db1..78656b80d 100644 --- a/plugin/builtin/transformer/restructure_test.go +++ b/operator/builtin/transformer/restructure_test.go @@ -11,32 +11,32 @@ import ( "github.com/antonmedv/expr/vm" "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.uber.org/zap" yaml "gopkg.in/yaml.v2" ) -func NewFakeRestructurePlugin() (*RestructurePlugin, *testutil.Plugin) { - mock := testutil.Plugin{} +func NewFakeRestructureOperator() (*RestructureOperator, *testutil.Operator) { + mock := testutil.Operator{} logger, _ := zap.NewProduction() - return &RestructurePlugin{ - TransformerPlugin: helper.TransformerPlugin{ - WriterPlugin: helper.WriterPlugin{ - BasicPlugin: helper.BasicPlugin{ - PluginID: "test", - PluginType: "restructure", + return &RestructureOperator{ + TransformerOperator: helper.TransformerOperator{ + WriterOperator: helper.WriterOperator{ + BasicOperator: helper.BasicOperator{ + OperatorID: "test", + OperatorType: "restructure", SugaredLogger: logger.Sugar(), }, - OutputPlugins: []plugin.Plugin{&mock}, + OutputOperators: []operator.Operator{&mock}, }, }, }, &mock } -func TestRestructurePlugin(t *testing.T) { +func TestRestructureOperator(t *testing.T) { os.Setenv("TEST_RESTRUCTURE_PLUGIN_ENV", "foo") defer os.Unsetenv("TEST_RESTRUCTURE_PLUGIN_ENV") @@ -202,14 +202,14 @@ func TestRestructurePlugin(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - plugin, mockOutput := NewFakeRestructurePlugin() - plugin.ops = tc.ops + operator, mockOutput := NewFakeRestructureOperator() + operator.ops = tc.ops mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { require.Equal(t, tc.output, args[1].(*entry.Entry)) }).Return(nil) - err := plugin.Process(context.Background(), tc.input) + err := operator.Process(context.Background(), tc.input) require.NoError(t, err) }) } @@ -342,13 +342,13 @@ ops: }] }` - expected := plugin.Config(plugin.Config{ - Builder: &RestructurePluginConfig{ + expected := operator.Config(operator.Config{ + Builder: &RestructureOperatorConfig{ TransformerConfig: helper.TransformerConfig{ WriterConfig: helper.WriterConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "my_restructure", - PluginType: "restructure", + OperatorID: "my_restructure", + OperatorType: "restructure", }, OutputIDs: []string{"test_output"}, }, @@ -392,12 +392,12 @@ ops: }, }) - var unmarshalledYAML plugin.Config + var unmarshalledYAML operator.Config err := yaml.UnmarshalStrict([]byte(configYAML), &unmarshalledYAML) require.NoError(t, err) require.Equal(t, expected, unmarshalledYAML) - var unmarshalledJSON plugin.Config + var unmarshalledJSON operator.Config err = json.Unmarshal([]byte(configJSON), &unmarshalledJSON) require.NoError(t, err) require.Equal(t, expected, unmarshalledJSON) diff --git a/operator/builtin/transformer/router.go b/operator/builtin/transformer/router.go new file mode 100644 index 000000000..93730887b --- /dev/null +++ b/operator/builtin/transformer/router.go @@ -0,0 +1,166 @@ +package transformer + +import ( + "context" + "fmt" + + "github.com/antonmedv/expr" + "github.com/antonmedv/expr/vm" + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" + "go.uber.org/zap" +) + +func init() { + operator.Register("router", func() operator.Builder { return NewRouterOperatorConfig("") }) +} + +func NewRouterOperatorConfig(operatorID string) *RouterOperatorConfig { + return &RouterOperatorConfig{ + BasicConfig: helper.NewBasicConfig(operatorID, "router"), + } +} + +// RouterOperatorConfig is the configuration of a router operator +type RouterOperatorConfig struct { + helper.BasicConfig `yaml:",inline"` + Routes []*RouterOperatorRouteConfig `json:"routes" yaml:"routes"` +} + +// RouterOperatorRouteConfig is the configuration of a route on a router operator +type RouterOperatorRouteConfig struct { + Expression string `json:"expr" yaml:"expr"` + OutputIDs helper.OutputIDs `json:"output" yaml:"output"` +} + +// Build will build a router operator from the supplied configuration +func (c RouterOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { + basicOperator, err := c.BasicConfig.Build(context) + if err != nil { + return nil, err + } + + routes := make([]*RouterOperatorRoute, 0, len(c.Routes)) + for _, routeConfig := range c.Routes { + compiled, err := expr.Compile(routeConfig.Expression, expr.AsBool(), expr.AllowUndefinedVariables()) + if err != nil { + return nil, fmt.Errorf("failed to compile expression '%s': %w", routeConfig.Expression, err) + } + route := RouterOperatorRoute{ + Expression: compiled, + OutputIDs: routeConfig.OutputIDs, + } + routes = append(routes, &route) + } + + routerOperator := &RouterOperator{ + BasicOperator: basicOperator, + routes: routes, + } + + return routerOperator, nil +} + +// SetNamespace will namespace the router operator and the outputs contained in its routes +func (c *RouterOperatorConfig) SetNamespace(namespace string, exclusions ...string) { + c.BasicConfig.SetNamespace(namespace, exclusions...) + for _, route := range c.Routes { + for i, outputID := range route.OutputIDs { + if helper.CanNamespace(outputID, exclusions) { + route.OutputIDs[i] = helper.AddNamespace(outputID, namespace) + } + } + } +} + +// RouterOperator is an operator that routes entries based on matching expressions +type RouterOperator struct { + helper.BasicOperator + routes []*RouterOperatorRoute +} + +// RouterOperatorRoute is a route on a router operator +type RouterOperatorRoute struct { + Expression *vm.Program + OutputIDs helper.OutputIDs + OutputOperators []operator.Operator +} + +// CanProcess will always return true for a router operator +func (p *RouterOperator) CanProcess() bool { + return true +} + +// Process will route incoming entries based on matching expressions +func (p *RouterOperator) Process(ctx context.Context, entry *entry.Entry) error { + env := helper.GetExprEnv(entry) + defer helper.PutExprEnv(env) + + for _, route := range p.routes { + matches, err := vm.Run(route.Expression, env) + if err != nil { + p.Warnw("Running expression returned an error", zap.Error(err)) + continue + } + + // we compile the expression with "AsBool", so this should be safe + if matches.(bool) { + for _, output := range route.OutputOperators { + _ = output.Process(ctx, entry) + } + break + } + } + + return nil +} + +// CanOutput will always return true for a router operator +func (p *RouterOperator) CanOutput() bool { + return true +} + +// Outputs will return all connected operators. +func (p *RouterOperator) Outputs() []operator.Operator { + outputs := make([]operator.Operator, 0, len(p.routes)) + for _, route := range p.routes { + outputs = append(outputs, route.OutputOperators...) + } + return outputs +} + +// SetOutputs will set the outputs of the router operator. +func (p *RouterOperator) SetOutputs(operators []operator.Operator) error { + for _, route := range p.routes { + outputOperators, err := p.findOperators(operators, route.OutputIDs) + if err != nil { + return fmt.Errorf("failed to set outputs on route: %s", err) + } + route.OutputOperators = outputOperators + } + return nil +} + +// findOperators will find a subset of operators from a collection. +func (p *RouterOperator) findOperators(operators []operator.Operator, operatorIDs []string) ([]operator.Operator, error) { + result := make([]operator.Operator, 0) + for _, operatorID := range operatorIDs { + operator, err := p.findOperator(operators, operatorID) + if err != nil { + return nil, err + } + result = append(result, operator) + } + return result, nil +} + +// findOperator will find an operator from a collection. +func (p *RouterOperator) findOperator(operators []operator.Operator, operatorID string) (operator.Operator, error) { + for _, operator := range operators { + if operator.ID() == operatorID { + return operator, nil + } + } + return nil, fmt.Errorf("operator %s does not exist", operatorID) +} diff --git a/plugin/builtin/transformer/router_test.go b/operator/builtin/transformer/router_test.go similarity index 71% rename from plugin/builtin/transformer/router_test.go rename to operator/builtin/transformer/router_test.go index d8c415499..a0f0f4abb 100644 --- a/plugin/builtin/transformer/router_test.go +++ b/operator/builtin/transformer/router_test.go @@ -7,21 +7,21 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -func TestRouterPlugin(t *testing.T) { +func TestRouterOperator(t *testing.T) { os.Setenv("TEST_ROUTER_PLUGIN_ENV", "foo") defer os.Unsetenv("TEST_ROUTER_PLUGIN_ENV") - basicConfig := func() *RouterPluginConfig { - return &RouterPluginConfig{ + basicConfig := func() *RouterOperatorConfig { + return &RouterOperatorConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "test_plugin_id", - PluginType: "router", + OperatorID: "test_operator_id", + OperatorType: "router", }, } } @@ -29,13 +29,13 @@ func TestRouterPlugin(t *testing.T) { cases := []struct { name string input *entry.Entry - routes []*RouterPluginRouteConfig + routes []*RouterOperatorRouteConfig expectedCounts map[string]int }{ { "DefaultRoute", entry.New(), - []*RouterPluginRouteConfig{ + []*RouterOperatorRouteConfig{ { "true", []string{"output1"}, @@ -46,7 +46,7 @@ func TestRouterPlugin(t *testing.T) { { "NoMatch", entry.New(), - []*RouterPluginRouteConfig{ + []*RouterOperatorRouteConfig{ { `false`, []string{"output1"}, @@ -61,7 +61,7 @@ func TestRouterPlugin(t *testing.T) { "message": "test_message", }, }, - []*RouterPluginRouteConfig{ + []*RouterOperatorRouteConfig{ { `$.message == "non_match"`, []string{"output1"}, @@ -80,7 +80,7 @@ func TestRouterPlugin(t *testing.T) { "message": "test_message", }, }, - []*RouterPluginRouteConfig{ + []*RouterOperatorRouteConfig{ { `env("TEST_ROUTER_PLUGIN_ENV") == "foo"`, []string{"output1"}, @@ -100,25 +100,25 @@ func TestRouterPlugin(t *testing.T) { cfg.Routes = tc.routes buildContext := testutil.NewBuildContext(t) - newPlugin, err := cfg.Build(buildContext) + newOperator, err := cfg.Build(buildContext) require.NoError(t, err) results := map[string]int{} - mock1 := testutil.NewMockPlugin("output1") + mock1 := testutil.NewMockOperator("output1") mock1.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { results["output1"] = results["output1"] + 1 }) - mock2 := testutil.NewMockPlugin("output2") + mock2 := testutil.NewMockOperator("output2") mock2.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { results["output2"] = results["output2"] + 1 }) - routerPlugin := newPlugin.(*RouterPlugin) - err = routerPlugin.SetOutputs([]plugin.Plugin{mock1, mock2}) + routerOperator := newOperator.(*RouterOperator) + err = routerOperator.SetOutputs([]operator.Operator{mock1, mock2}) require.NoError(t, err) - err = routerPlugin.Process(context.Background(), tc.input) + err = routerOperator.Process(context.Background(), tc.input) require.NoError(t, err) require.Equal(t, tc.expectedCounts, results) diff --git a/plugin/config.go b/operator/config.go similarity index 82% rename from plugin/config.go rename to operator/config.go index 7ad879c70..31d2cc399 100644 --- a/plugin/config.go +++ b/operator/config.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "encoding/json" @@ -8,22 +8,22 @@ import ( "go.uber.org/zap" ) -// Config is the configuration of a plugin +// Config is the configuration of an operator type Config struct { Builder } -// Builder is an entity that can build plugins +// Builder is an entity that can build operators type Builder interface { ID() string Type() string - Build(BuildContext) (Plugin, error) + Build(BuildContext) (Operator, error) SetNamespace(namespace string, exclude ...string) } -// BuildContext supplies contextual resources when building a plugin. +// BuildContext supplies contextual resources when building an operator. type BuildContext struct { - CustomRegistry CustomRegistry + PluginRegistry PluginRegistry Database Database Logger *zap.SugaredLogger } @@ -58,18 +58,18 @@ func NewStubDatabase() *StubDatabase { return &StubDatabase{} } -// registry is a global registry of plugin types to plugin builders. +// registry is a global registry of operator types to operator builders. var registry = make(map[string]func() Builder) -// Register will register a function to a plugin type. +// Register will register a function to an operator type. // This function will return a builder for the supplied type. -func Register(pluginType string, newBuilder func() Builder) { - registry[pluginType] = newBuilder +func Register(operatorType string, newBuilder func() Builder) { + registry[operatorType] = newBuilder } -// IsDefined will return a boolean indicating if a plugin type is registered and defined. -func IsDefined(pluginType string) bool { - _, ok := registry[pluginType] +// IsDefined will return a boolean indicating if an operator type is registered and defined. +func IsDefined(operatorType string) bool { + _, ok := registry[operatorType] return ok } diff --git a/plugin/config_test.go b/operator/config_test.go similarity index 75% rename from plugin/config_test.go rename to operator/config_test.go index c9294ac3d..d8930b3d0 100644 --- a/plugin/config_test.go +++ b/operator/config_test.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "encoding/json" @@ -25,15 +25,15 @@ func TestStubDatabase(t *testing.T) { } type FakeBuilder struct { - PluginID string `json:"id" yaml:"id"` - PluginType string `json:"type" yaml:"type"` - Array []string `json:"array" yaml:"array"` + OperatorID string `json:"id" yaml:"id"` + OperatorType string `json:"type" yaml:"type"` + Array []string `json:"array" yaml:"array"` } -func (f *FakeBuilder) SetNamespace(s string, e ...string) {} -func (f *FakeBuilder) Build(context BuildContext) (Plugin, error) { return nil, nil } -func (f *FakeBuilder) ID() string { return "custom" } -func (f *FakeBuilder) Type() string { return "custom" } +func (f *FakeBuilder) SetNamespace(s string, e ...string) {} +func (f *FakeBuilder) Build(context BuildContext) (Operator, error) { return nil, nil } +func (f *FakeBuilder) ID() string { return "plugin" } +func (f *FakeBuilder) Type() string { return "plugin" } func TestUnmarshalJSONErrors(t *testing.T) { t.Run("InvalidJSON", func(t *testing.T) { @@ -61,8 +61,8 @@ func TestUnmarshalJSONErrors(t *testing.T) { }) t.Run("TypeSpecificUnmarshal", func(t *testing.T) { - raw := `{"id":"custom","type":"custom","array":"non-array-value"}` - Register("custom", func() Builder { return &FakeBuilder{} }) + raw := `{"id":"plugin","type":"plugin","array":"non-array-value"}` + Register("plugin", func() Builder { return &FakeBuilder{} }) var cfg Config err := json.Unmarshal([]byte(raw), &cfg) require.Error(t, err) @@ -73,14 +73,14 @@ func TestUnmarshalJSONErrors(t *testing.T) { func TestMarshalJSON(t *testing.T) { cfg := Config{ Builder: &FakeBuilder{ - PluginID: "custom", - PluginType: "custom", - Array: []string{"test"}, + OperatorID: "plugin", + OperatorType: "plugin", + Array: []string{"test"}, }, } out, err := json.Marshal(cfg) require.NoError(t, err) - expected := `{"id":"custom","type":"custom","array":["test"]}` + expected := `{"id":"plugin","type":"plugin","array":["test"]}` require.Equal(t, expected, string(out)) } @@ -94,7 +94,7 @@ func TestUnmarshalYAMLErrors(t *testing.T) { }) t.Run("MissingType", func(t *testing.T) { - raw := "id: custom\n" + raw := "id: plugin\n" var cfg Config err := yaml.Unmarshal([]byte(raw), &cfg) require.Error(t, err) @@ -102,7 +102,7 @@ func TestUnmarshalYAMLErrors(t *testing.T) { }) t.Run("NonStringType", func(t *testing.T) { - raw := "id: custom\ntype: 123" + raw := "id: plugin\ntype: 123" var cfg Config err := yaml.Unmarshal([]byte(raw), &cfg) require.Error(t, err) @@ -110,7 +110,7 @@ func TestUnmarshalYAMLErrors(t *testing.T) { }) t.Run("UnknownType", func(t *testing.T) { - raw := "id: custom\ntype: unknown\n" + raw := "id: plugin\ntype: unknown\n" var cfg Config err := yaml.Unmarshal([]byte(raw), &cfg) require.Error(t, err) @@ -118,8 +118,8 @@ func TestUnmarshalYAMLErrors(t *testing.T) { }) t.Run("TypeSpecificUnmarshal", func(t *testing.T) { - raw := "id: custom\ntype: custom\narray: nonarray" - Register("custom", func() Builder { return &FakeBuilder{} }) + raw := "id: plugin\ntype: plugin\narray: nonarray" + Register("plugin", func() Builder { return &FakeBuilder{} }) var cfg Config err := yaml.Unmarshal([]byte(raw), &cfg) require.Error(t, err) @@ -130,13 +130,13 @@ func TestUnmarshalYAMLErrors(t *testing.T) { func TestMarshalYAML(t *testing.T) { cfg := Config{ Builder: &FakeBuilder{ - PluginID: "custom", - PluginType: "custom", - Array: []string{"test"}, + OperatorID: "plugin", + OperatorType: "plugin", + Array: []string{"test"}, }, } out, err := yaml.Marshal(cfg) require.NoError(t, err) - expected := "id: custom\ntype: custom\narray:\n- test\n" + expected := "id: plugin\ntype: plugin\narray:\n- test\n" require.Equal(t, expected, string(out)) } diff --git a/plugin/duration.go b/operator/duration.go similarity index 98% rename from plugin/duration.go rename to operator/duration.go index c22f34625..e53d8dbbc 100644 --- a/plugin/duration.go +++ b/operator/duration.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "encoding/json" diff --git a/plugin/duration_test.go b/operator/duration_test.go similarity index 98% rename from plugin/duration_test.go rename to operator/duration_test.go index d4711691e..8c5c85d3d 100644 --- a/plugin/duration_test.go +++ b/operator/duration_test.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "encoding/json" diff --git a/plugin/helper/expr_string.go b/operator/helper/expr_string.go similarity index 100% rename from plugin/helper/expr_string.go rename to operator/helper/expr_string.go diff --git a/plugin/helper/expr_string_test.go b/operator/helper/expr_string_test.go similarity index 100% rename from plugin/helper/expr_string_test.go rename to operator/helper/expr_string_test.go diff --git a/operator/helper/input.go b/operator/helper/input.go new file mode 100644 index 000000000..70ed69eee --- /dev/null +++ b/operator/helper/input.go @@ -0,0 +1,65 @@ +package helper + +import ( + "context" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/errors" + "github.com/observiq/carbon/operator" + "go.uber.org/zap" +) + +func NewInputConfig(operatorID, operatorType string) InputConfig { + return InputConfig{ + WriterConfig: NewWriterConfig(operatorID, operatorType), + WriteTo: entry.NewRecordField(), + } +} + +// InputConfig provides a basic implementation of an input operator config. +type InputConfig struct { + WriterConfig `yaml:",inline"` + WriteTo entry.Field `json:"write_to" yaml:"write_to"` +} + +// Build will build a base producer. +func (c InputConfig) Build(context operator.BuildContext) (InputOperator, error) { + writerOperator, err := c.WriterConfig.Build(context) + if err != nil { + return InputOperator{}, errors.WithDetails(err, "operator_id", c.ID()) + } + + inputOperator := InputOperator{ + WriterOperator: writerOperator, + WriteTo: c.WriteTo, + } + + return inputOperator, nil +} + +// InputOperator provides a basic implementation of an input operator. +type InputOperator struct { + WriterOperator + WriteTo entry.Field +} + +// NewEntry will create a new entry using the write_to field. +func (i *InputOperator) NewEntry(value interface{}) *entry.Entry { + entry := entry.New() + entry.Set(i.WriteTo, value) + return entry +} + +// CanProcess will always return false for an input operator. +func (i *InputOperator) CanProcess() bool { + return false +} + +// Process will always return an error if called. +func (i *InputOperator) Process(ctx context.Context, entry *entry.Entry) error { + i.Errorw("Operator received an entry, but can not process", zap.Any("entry", entry)) + return errors.NewError( + "Operator can not process logs.", + "Ensure that operator is not configured to receive logs from other operators", + ) +} diff --git a/plugin/helper/input_test.go b/operator/helper/input_test.go similarity index 69% rename from plugin/helper/input_test.go rename to operator/helper/input_test.go index 7480b5ef3..8a35fae2a 100644 --- a/plugin/helper/input_test.go +++ b/operator/helper/input_test.go @@ -26,8 +26,8 @@ func TestInputConfigMissingOutput(t *testing.T) { config := InputConfig{ WriterConfig: WriterConfig{ BasicConfig: BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", }, }, WriteTo: entry.Field{}, @@ -42,8 +42,8 @@ func TestInputConfigValid(t *testing.T) { WriteTo: entry.Field{}, WriterConfig: WriterConfig{ BasicConfig: BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", }, OutputIDs: []string{"test-output"}, }, @@ -58,24 +58,24 @@ func TestInputConfigSetNamespace(t *testing.T) { WriteTo: entry.Field{}, WriterConfig: WriterConfig{ BasicConfig: BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", }, OutputIDs: []string{"test-output"}, }, } config.SetNamespace("test-namespace") - require.Equal(t, "test-namespace.test-id", config.PluginID) + require.Equal(t, "test-namespace.test-id", config.OperatorID) require.Equal(t, "test-namespace.test-output", config.OutputIDs[0]) } -func TestInputPluginCanProcess(t *testing.T) { +func TestInputOperatorCanProcess(t *testing.T) { buildContext := testutil.NewBuildContext(t) - input := InputPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + input := InputOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -83,13 +83,13 @@ func TestInputPluginCanProcess(t *testing.T) { require.False(t, input.CanProcess()) } -func TestInputPluginProcess(t *testing.T) { +func TestInputOperatorProcess(t *testing.T) { buildContext := testutil.NewBuildContext(t) - input := InputPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + input := InputOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -98,17 +98,17 @@ func TestInputPluginProcess(t *testing.T) { ctx := context.Background() err := input.Process(ctx, entry) require.Error(t, err) - require.Equal(t, err.Error(), "Plugin can not process logs.") + require.Equal(t, err.Error(), "Operator can not process logs.") } -func TestInputPluginNewEntry(t *testing.T) { +func TestInputOperatorNewEntry(t *testing.T) { buildContext := testutil.NewBuildContext(t) writeTo := entry.NewRecordField("test-field") - input := InputPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + input := InputOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, diff --git a/plugin/helper/namespace.go b/operator/helper/namespace.go similarity index 100% rename from plugin/helper/namespace.go rename to operator/helper/namespace.go diff --git a/operator/helper/operator.go b/operator/helper/operator.go new file mode 100644 index 000000000..bb1838719 --- /dev/null +++ b/operator/helper/operator.go @@ -0,0 +1,103 @@ +package helper + +import ( + "github.com/observiq/carbon/errors" + "github.com/observiq/carbon/operator" + "go.uber.org/zap" +) + +func NewBasicConfig(operatorID, operatorType string) BasicConfig { + return BasicConfig{ + OperatorID: operatorID, + OperatorType: operatorType, + } +} + +// BasicConfig provides a basic implemention for an operator config. +type BasicConfig struct { + OperatorID string `json:"id" yaml:"id"` + OperatorType string `json:"type" yaml:"type"` +} + +// ID will return the operator id. +func (c BasicConfig) ID() string { + if c.OperatorID == "" { + return c.OperatorType + } + return c.OperatorID +} + +// Type will return the operator type. +func (c BasicConfig) Type() string { + return c.OperatorType +} + +// Build will build a basic operator. +func (c BasicConfig) Build(context operator.BuildContext) (BasicOperator, error) { + if c.OperatorType == "" { + return BasicOperator{}, errors.NewError( + "missing required `type` field.", + "ensure that all operators have a uniquely defined `type` field.", + "operator_id", c.ID(), + ) + } + + if context.Logger == nil { + return BasicOperator{}, errors.NewError( + "operator build context is missing a logger.", + "this is an unexpected internal error", + "operator_id", c.ID(), + "operator_type", c.Type(), + ) + } + + operator := BasicOperator{ + OperatorID: c.ID(), + OperatorType: c.Type(), + SugaredLogger: context.Logger.With("operator_id", c.ID(), "operator_type", c.Type()), + } + + return operator, nil +} + +// SetNamespace will namespace the operator id. +func (c *BasicConfig) SetNamespace(namespace string, exclusions ...string) { + if CanNamespace(c.ID(), exclusions) { + c.OperatorID = AddNamespace(c.ID(), namespace) + } +} + +// BasicOperator provides a basic implementation of an operator. +type BasicOperator struct { + OperatorID string + OperatorType string + *zap.SugaredLogger +} + +// ID will return the operator id. +func (p *BasicOperator) ID() string { + if p.OperatorID == "" { + return p.OperatorType + } + return p.OperatorID +} + +// Type will return the operator type. +func (p *BasicOperator) Type() string { + return p.OperatorType +} + +// Logger returns the operator's scoped logger. +func (p *BasicOperator) Logger() *zap.SugaredLogger { + return p.SugaredLogger +} + +// Start will start the operator. +func (p *BasicOperator) Start() error { + return nil +} + +// Stop will stop the operator. +func (p *BasicOperator) Stop() error { + return nil +} diff --git a/operator/helper/operator_test.go b/operator/helper/operator_test.go new file mode 100644 index 000000000..715384122 --- /dev/null +++ b/operator/helper/operator_test.go @@ -0,0 +1,112 @@ +package helper + +import ( + "testing" + + "github.com/observiq/carbon/internal/testutil" + "github.com/observiq/carbon/operator" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestBasicConfigID(t *testing.T) { + config := BasicConfig{ + OperatorID: "test-id", + OperatorType: "test-type", + } + require.Equal(t, "test-id", config.ID()) +} + +func TestBasicConfigType(t *testing.T) { + config := BasicConfig{ + OperatorID: "test-id", + OperatorType: "test-type", + } + require.Equal(t, "test-type", config.Type()) +} + +func TestBasicConfigBuildWithoutID(t *testing.T) { + config := BasicConfig{ + OperatorType: "test-type", + } + context := testutil.NewBuildContext(t) + _, err := config.Build(context) + require.NoError(t, err) +} + +func TestBasicConfigBuildWithoutType(t *testing.T) { + config := BasicConfig{ + OperatorID: "test-id", + } + context := operator.BuildContext{} + _, err := config.Build(context) + require.Error(t, err) + require.Contains(t, err.Error(), "missing required `type` field.") +} + +func TestBasicConfigBuildMissingLogger(t *testing.T) { + config := BasicConfig{ + OperatorID: "test-id", + OperatorType: "test-type", + } + context := operator.BuildContext{} + _, err := config.Build(context) + require.Error(t, err) + require.Contains(t, err.Error(), "operator build context is missing a logger.") +} + +func TestBasicConfigBuildValid(t *testing.T) { + config := BasicConfig{ + OperatorID: "test-id", + OperatorType: "test-type", + } + context := testutil.NewBuildContext(t) + operator, err := config.Build(context) + require.NoError(t, err) + require.Equal(t, "test-id", operator.OperatorID) + require.Equal(t, "test-type", operator.OperatorType) +} + +func TestBasicOperatorID(t *testing.T) { + operator := BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", + } + require.Equal(t, "test-id", operator.ID()) +} + +func TestBasicOperatorType(t *testing.T) { + operator := BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", + } + require.Equal(t, "test-type", operator.Type()) +} + +func TestBasicOperatorLogger(t *testing.T) { + logger := &zap.SugaredLogger{} + operator := BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", + SugaredLogger: logger, + } + require.Equal(t, logger, operator.Logger()) +} + +func TestBasicOperatorStart(t *testing.T) { + operator := BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", + } + err := operator.Start() + require.NoError(t, err) +} + +func TestBasicOperatorStop(t *testing.T) { + operator := BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", + } + err := operator.Stop() + require.NoError(t, err) +} diff --git a/operator/helper/output.go b/operator/helper/output.go new file mode 100644 index 000000000..b76558f02 --- /dev/null +++ b/operator/helper/output.go @@ -0,0 +1,66 @@ +package helper + +import ( + "github.com/observiq/carbon/errors" + "github.com/observiq/carbon/operator" +) + +func NewOutputConfig(operatorID, operatorType string) OutputConfig { + return OutputConfig{ + BasicConfig: NewBasicConfig(operatorID, operatorType), + } +} + +// OutputConfig provides a basic implementation of an output operator config. +type OutputConfig struct { + BasicConfig `mapstructure:",squash" yaml:",inline"` +} + +// Build will build an output operator. +func (c OutputConfig) Build(context operator.BuildContext) (OutputOperator, error) { + basicOperator, err := c.BasicConfig.Build(context) + if err != nil { + return OutputOperator{}, err + } + + outputOperator := OutputOperator{ + BasicOperator: basicOperator, + } + + return outputOperator, nil +} + +// SetNamespace will namespace the id and output of the operator config. +func (c *OutputConfig) SetNamespace(namespace string, exclusions ...string) { + if CanNamespace(c.ID(), exclusions) { + c.OperatorID = AddNamespace(c.ID(), namespace) + } +} + +// OutputOperator provides a basic implementation of an output operator. +type OutputOperator struct { + BasicOperator +} + +// CanProcess will always return true for an output operator. +func (o *OutputOperator) CanProcess() bool { + return true +} + +// CanOutput will always return false for an output operator. +func (o *OutputOperator) CanOutput() bool { + return false +} + +// Outputs will always return an empty array for an output operator. +func (o *OutputOperator) Outputs() []operator.Operator { + return []operator.Operator{} +} + +// SetOutputs will return an error if called. +func (o *OutputOperator) SetOutputs(operators []operator.Operator) error { + return errors.NewError( + "Operator can not output, but is attempting to set an output.", + "This is an unexpected internal error. Please submit a bug/issue.", + ) +} diff --git a/plugin/helper/output_test.go b/operator/helper/output_test.go similarity index 55% rename from plugin/helper/output_test.go rename to operator/helper/output_test.go index 643d3b81b..96b9c871e 100644 --- a/plugin/helper/output_test.go +++ b/operator/helper/output_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/require" ) @@ -19,8 +19,8 @@ func TestOutputConfigMissingBase(t *testing.T) { func TestOutputConfigBuildValid(t *testing.T) { config := OutputConfig{ BasicConfig: BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", }, } context := testutil.NewBuildContext(t) @@ -31,61 +31,61 @@ func TestOutputConfigBuildValid(t *testing.T) { func TestOutputConfigNamespace(t *testing.T) { config := OutputConfig{ BasicConfig: BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", }, } config.SetNamespace("test-namespace") require.Equal(t, "test-namespace.test-id", config.ID()) } -func TestOutputPluginCanProcess(t *testing.T) { +func TestOutputOperatorCanProcess(t *testing.T) { buildContext := testutil.NewBuildContext(t) - output := OutputPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + output := OutputOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, } require.True(t, output.CanProcess()) } -func TestOutputPluginCanOutput(t *testing.T) { +func TestOutputOperatorCanOutput(t *testing.T) { buildContext := testutil.NewBuildContext(t) - output := OutputPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + output := OutputOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, } require.False(t, output.CanOutput()) } -func TestOutputPluginOutputs(t *testing.T) { +func TestOutputOperatorOutputs(t *testing.T) { buildContext := testutil.NewBuildContext(t) - output := OutputPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + output := OutputOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, } - require.Equal(t, []plugin.Plugin{}, output.Outputs()) + require.Equal(t, []operator.Operator{}, output.Outputs()) } -func TestOutputPluginSetOutputs(t *testing.T) { +func TestOutputOperatorSetOutputs(t *testing.T) { buildContext := testutil.NewBuildContext(t) - output := OutputPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + output := OutputOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, } - err := output.SetOutputs([]plugin.Plugin{}) + err := output.SetOutputs([]operator.Operator{}) require.Error(t, err) - require.Contains(t, err.Error(), "Plugin can not output") + require.Contains(t, err.Error(), "Operator can not output") } diff --git a/plugin/helper/parser.go b/operator/helper/parser.go similarity index 71% rename from plugin/helper/parser.go rename to operator/helper/parser.go index bb0bada03..98dd1bc94 100644 --- a/plugin/helper/parser.go +++ b/operator/helper/parser.go @@ -5,12 +5,12 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" ) -func NewParserConfig(pluginID, pluginType string) ParserConfig { +func NewParserConfig(operatorID, operatorType string) ParserConfig { return ParserConfig{ - TransformerConfig: NewTransformerConfig(pluginID, pluginType), + TransformerConfig: NewTransformerConfig(operatorID, operatorType), ParseFrom: entry.NewRecordField(), ParseTo: entry.NewRecordField(), Preserve: false, @@ -28,49 +28,49 @@ type ParserConfig struct { SeverityParserConfig *SeverityParserConfig `json:"severity,omitempty" yaml:"severity,omitempty"` } -// Build will build a parser plugin. -func (c ParserConfig) Build(context plugin.BuildContext) (ParserPlugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) +// Build will build a parser operator. +func (c ParserConfig) Build(context operator.BuildContext) (ParserOperator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { - return ParserPlugin{}, err + return ParserOperator{}, err } if c.ParseFrom.String() == c.ParseTo.String() && c.Preserve { - transformerPlugin.Warnw( + transformerOperator.Warnw( "preserve is true, but parse_to is set to the same field as parse_from, "+ "which will cause the original value to be overwritten", - "plugin_id", c.ID(), + "operator_id", c.ID(), ) } - parserPlugin := ParserPlugin{ - TransformerPlugin: transformerPlugin, - ParseFrom: c.ParseFrom, - ParseTo: c.ParseTo, - Preserve: c.Preserve, + parserOperator := ParserOperator{ + TransformerOperator: transformerOperator, + ParseFrom: c.ParseFrom, + ParseTo: c.ParseTo, + Preserve: c.Preserve, } if c.TimeParser != nil { if err := c.TimeParser.Validate(context); err != nil { - return ParserPlugin{}, err + return ParserOperator{}, err } - parserPlugin.TimeParser = c.TimeParser + parserOperator.TimeParser = c.TimeParser } if c.SeverityParserConfig != nil { severityParser, err := c.SeverityParserConfig.Build(context) if err != nil { - return ParserPlugin{}, err + return ParserOperator{}, err } - parserPlugin.SeverityParser = &severityParser + parserOperator.SeverityParser = &severityParser } - return parserPlugin, nil + return parserOperator, nil } -// ParserPlugin provides a basic implementation of a parser plugin. -type ParserPlugin struct { - TransformerPlugin +// ParserOperator provides a basic implementation of a parser operator. +type ParserOperator struct { + TransformerOperator ParseFrom entry.Field ParseTo entry.Field Preserve bool @@ -79,7 +79,7 @@ type ParserPlugin struct { } // ProcessWith will process an entry with a parser function. -func (p *ParserPlugin) ProcessWith(ctx context.Context, entry *entry.Entry, parse ParseFunction) error { +func (p *ParserOperator) ProcessWith(ctx context.Context, entry *entry.Entry, parse ParseFunction) error { value, ok := entry.Get(p.ParseFrom) if !ok { err := errors.NewError( diff --git a/plugin/helper/parser_test.go b/operator/helper/parser_test.go similarity index 79% rename from plugin/helper/parser_test.go rename to operator/helper/parser_test.go index b035f8283..9dbc4fd30 100644 --- a/plugin/helper/parser_test.go +++ b/operator/helper/parser_test.go @@ -8,7 +8,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" @@ -49,12 +49,12 @@ func TestParserConfigBuildValid(t *testing.T) { } func TestParserMissingField(t *testing.T) { - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: zaptest.NewLogger(t).Sugar(), }, }, @@ -74,12 +74,12 @@ func TestParserMissingField(t *testing.T) { func TestParserInvalidParse(t *testing.T) { buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -99,12 +99,12 @@ func TestParserInvalidParse(t *testing.T) { func TestParserInvalidTimeParse(t *testing.T) { buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -131,12 +131,12 @@ func TestParserInvalidTimeParse(t *testing.T) { func TestParserInvalidSeverityParse(t *testing.T) { buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -160,12 +160,12 @@ func TestParserInvalidSeverityParse(t *testing.T) { func TestParserInvalidTimeValidSeverityParse(t *testing.T) { buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -203,12 +203,12 @@ func TestParserInvalidTimeValidSeverityParse(t *testing.T) { func TestParserValidTimeInvalidSeverityParse(t *testing.T) { buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -246,20 +246,20 @@ func TestParserValidTimeInvalidSeverityParse(t *testing.T) { } func TestParserOutput(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ OnError: DropOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, + OutputOperators: []operator.Operator{output}, }, }, ParseFrom: entry.NewRecordField(), @@ -276,20 +276,20 @@ func TestParserOutput(t *testing.T) { } func TestParserWithPreserve(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ OnError: DropOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, + OutputOperators: []operator.Operator{output}, }, }, ParseFrom: entry.NewRecordField("parse_from"), @@ -315,20 +315,20 @@ func TestParserWithPreserve(t *testing.T) { } func TestParserWithoutPreserve(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ OnError: DropOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, + OutputOperators: []operator.Operator{output}, }, }, ParseFrom: entry.NewRecordField("parse_from"), diff --git a/plugin/helper/persister.go b/operator/helper/persister.go similarity index 93% rename from plugin/helper/persister.go rename to operator/helper/persister.go index 4c19d9071..fea266f72 100644 --- a/plugin/helper/persister.go +++ b/operator/helper/persister.go @@ -3,7 +3,7 @@ package helper import ( "sync" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "go.etcd.io/bbolt" ) @@ -18,13 +18,13 @@ type Persister interface { // ScopedBBoltPersister is a persister that uses a database for the backend type ScopedBBoltPersister struct { scope []byte - db plugin.Database + db operator.Database cache map[string][]byte cacheMux sync.Mutex } // NewScopedDBPersister returns a new ScopedBBoltPersister -func NewScopedDBPersister(db plugin.Database, scope string) *ScopedBBoltPersister { +func NewScopedDBPersister(db operator.Database, scope string) *ScopedBBoltPersister { return &ScopedBBoltPersister{ scope: []byte(scope), db: db, diff --git a/plugin/helper/severity.go b/operator/helper/severity.go similarity index 100% rename from plugin/helper/severity.go rename to operator/helper/severity.go diff --git a/plugin/helper/severity_builder.go b/operator/helper/severity_builder.go similarity index 94% rename from plugin/helper/severity_builder.go rename to operator/helper/severity_builder.go index 228b5e4ba..295700183 100644 --- a/plugin/helper/severity_builder.go +++ b/operator/helper/severity_builder.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" ) const minSeverity = 0 @@ -73,8 +73,8 @@ type SeverityParserConfig struct { } // Build builds a SeverityParser from a SeverityParserConfig -func (c *SeverityParserConfig) Build(context plugin.BuildContext) (SeverityParser, error) { - pluginMapping := getBuiltinMapping(c.Preset) +func (c *SeverityParserConfig) Build(context operator.BuildContext) (SeverityParser, error) { + operatorMapping := getBuiltinMapping(c.Preset) for severity, unknown := range c.Mapping { sev, err := validateSeverity(severity) @@ -89,14 +89,14 @@ func (c *SeverityParserConfig) Build(context plugin.BuildContext) (SeverityParse if err != nil { return SeverityParser{}, err } - pluginMapping.add(sev, v...) + operatorMapping.add(sev, v...) } case interface{}: v, err := parseableValues(u) if err != nil { return SeverityParser{}, err } - pluginMapping.add(sev, v...) + operatorMapping.add(sev, v...) } } @@ -107,7 +107,7 @@ func (c *SeverityParserConfig) Build(context plugin.BuildContext) (SeverityParse p := SeverityParser{ ParseFrom: *c.ParseFrom, Preserve: c.Preserve, - Mapping: pluginMapping, + Mapping: operatorMapping, } return p, nil diff --git a/plugin/helper/time.go b/operator/helper/time.go similarity index 97% rename from plugin/helper/time.go rename to operator/helper/time.go index a6c662878..66f525745 100644 --- a/plugin/helper/time.go +++ b/operator/helper/time.go @@ -11,7 +11,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" ) // StrptimeKey is literally "strptime", and is the default layout type @@ -24,7 +24,7 @@ const GotimeKey = "gotime" const EpochKey = "epoch" // NativeKey is literally "native" and refers to Golang's native time.Time -const NativeKey = "native" // provided for plugin development +const NativeKey = "native" // provided for operator development func NewTimeParser() TimeParser { return TimeParser{ @@ -46,7 +46,7 @@ func (t *TimeParser) IsZero() bool { } // Validate validates a TimeParser, and reconfigures it if necessary -func (t *TimeParser) Validate(context plugin.BuildContext) error { +func (t *TimeParser) Validate(context operator.BuildContext) error { if t.ParseFrom == nil { return fmt.Errorf("missing required parameter 'parse_from'") } diff --git a/plugin/helper/time_test.go b/operator/helper/time_test.go similarity index 100% rename from plugin/helper/time_test.go rename to operator/helper/time_test.go diff --git a/plugin/helper/transformer.go b/operator/helper/transformer.go similarity index 55% rename from plugin/helper/transformer.go rename to operator/helper/transformer.go index b17ae5f8e..df2b96fdf 100644 --- a/plugin/helper/transformer.go +++ b/operator/helper/transformer.go @@ -5,13 +5,13 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "go.uber.org/zap" ) -func NewTransformerConfig(pluginID, pluginType string) TransformerConfig { +func NewTransformerConfig(operatorID, operatorType string) TransformerConfig { return TransformerConfig{ - WriterConfig: NewWriterConfig(pluginID, pluginType), + WriterConfig: NewWriterConfig(operatorID, operatorType), OnError: SendOnError, } } @@ -22,44 +22,44 @@ type TransformerConfig struct { OnError string `json:"on_error" yaml:"on_error"` } -// Build will build a transformer plugin. -func (c TransformerConfig) Build(context plugin.BuildContext) (TransformerPlugin, error) { - writerPlugin, err := c.WriterConfig.Build(context) +// Build will build a transformer operator. +func (c TransformerConfig) Build(context operator.BuildContext) (TransformerOperator, error) { + writerOperator, err := c.WriterConfig.Build(context) if err != nil { - return TransformerPlugin{}, errors.WithDetails(err, "plugin_id", c.ID()) + return TransformerOperator{}, errors.WithDetails(err, "operator_id", c.ID()) } switch c.OnError { case SendOnError, DropOnError: default: - return TransformerPlugin{}, errors.NewError( - "plugin config has an invalid `on_error` field.", + return TransformerOperator{}, errors.NewError( + "operator config has an invalid `on_error` field.", "ensure that the `on_error` field is set to either `send` or `drop`.", "on_error", c.OnError, ) } - transformerPlugin := TransformerPlugin{ - WriterPlugin: writerPlugin, - OnError: c.OnError, + transformerOperator := TransformerOperator{ + WriterOperator: writerOperator, + OnError: c.OnError, } - return transformerPlugin, nil + return transformerOperator, nil } -// TransformerPlugin provides a basic implementation of a transformer plugin. -type TransformerPlugin struct { - WriterPlugin +// TransformerOperator provides a basic implementation of a transformer operator. +type TransformerOperator struct { + WriterOperator OnError string } -// CanProcess will always return true for a transformer plugin. -func (t *TransformerPlugin) CanProcess() bool { +// CanProcess will always return true for a transformer operator. +func (t *TransformerOperator) CanProcess() bool { return true } // ProcessWith will process an entry with a transform function. -func (t *TransformerPlugin) ProcessWith(ctx context.Context, entry *entry.Entry, transform TransformFunction) error { +func (t *TransformerOperator) ProcessWith(ctx context.Context, entry *entry.Entry, transform TransformFunction) error { newEntry, err := transform(entry) if err != nil { return t.HandleEntryError(ctx, entry, err) @@ -69,7 +69,7 @@ func (t *TransformerPlugin) ProcessWith(ctx context.Context, entry *entry.Entry, } // HandleEntryError will handle an entry error using the on_error strategy. -func (t *TransformerPlugin) HandleEntryError(ctx context.Context, entry *entry.Entry, err error) error { +func (t *TransformerOperator) HandleEntryError(ctx context.Context, entry *entry.Entry, err error) error { t.Errorw("Failed to process entry", zap.Any("error", err), zap.Any("action", t.OnError), zap.Any("entry", entry)) if t.OnError == SendOnError { t.Write(ctx, entry) diff --git a/plugin/helper/transformer_test.go b/operator/helper/transformer_test.go similarity index 75% rename from plugin/helper/transformer_test.go rename to operator/helper/transformer_test.go index 447d0463a..f9cbcb5b0 100644 --- a/plugin/helper/transformer_test.go +++ b/operator/helper/transformer_test.go @@ -7,7 +7,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -45,18 +45,18 @@ func TestTransformerOnErrorInvalid(t *testing.T) { cfg.OnError = "invalid" _, err := cfg.Build(testutil.NewBuildContext(t)) require.Error(t, err) - require.Contains(t, err.Error(), "plugin config has an invalid `on_error` field.") + require.Contains(t, err.Error(), "operator config has an invalid `on_error` field.") } func TestTransformerConfigSetNamespace(t *testing.T) { cfg := NewTransformerConfig("test-id", "test-type") cfg.OutputIDs = []string{"test-output"} cfg.SetNamespace("test-namespace") - require.Equal(t, "test-namespace.test-id", cfg.PluginID) + require.Equal(t, "test-namespace.test-id", cfg.OperatorID) require.Equal(t, "test-namespace.test-output", cfg.OutputIDs[0]) } -func TestTransformerPluginCanProcess(t *testing.T) { +func TestTransformerOperatorCanProcess(t *testing.T) { cfg := NewTransformerConfig("test", "test") transformer, err := cfg.Build(testutil.NewBuildContext(t)) require.NoError(t, err) @@ -64,20 +64,20 @@ func TestTransformerPluginCanProcess(t *testing.T) { } func TestTransformerDropOnError(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - transformer := TransformerPlugin{ + transformer := TransformerOperator{ OnError: DropOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, - OutputIDs: []string{"test-output"}, + OutputOperators: []operator.Operator{output}, + OutputIDs: []string{"test-output"}, }, } ctx := context.Background() @@ -92,20 +92,20 @@ func TestTransformerDropOnError(t *testing.T) { } func TestTransformerSendOnError(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - transformer := TransformerPlugin{ + transformer := TransformerOperator{ OnError: SendOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, - OutputIDs: []string{"test-output"}, + OutputOperators: []operator.Operator{output}, + OutputIDs: []string{"test-output"}, }, } ctx := context.Background() @@ -120,20 +120,20 @@ func TestTransformerSendOnError(t *testing.T) { } func TestTransformerProcessWithValid(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - transformer := TransformerPlugin{ + transformer := TransformerOperator{ OnError: SendOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, - OutputIDs: []string{"test-output"}, + OutputOperators: []operator.Operator{output}, + OutputIDs: []string{"test-output"}, }, } ctx := context.Background() diff --git a/operator/helper/writer.go b/operator/helper/writer.go new file mode 100644 index 000000000..2c3b8b5f6 --- /dev/null +++ b/operator/helper/writer.go @@ -0,0 +1,191 @@ +package helper + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/operator" +) + +func NewWriterConfig(operatorID, operatorType string) WriterConfig { + return WriterConfig{ + BasicConfig: NewBasicConfig(operatorID, operatorType), + } +} + +// WriterConfig is the configuration of a writer operator. +type WriterConfig struct { + BasicConfig `yaml:",inline"` + OutputIDs OutputIDs `json:"output" yaml:"output"` +} + +// Build will build a writer operator from the config. +func (c WriterConfig) Build(context operator.BuildContext) (WriterOperator, error) { + basicOperator, err := c.BasicConfig.Build(context) + if err != nil { + return WriterOperator{}, err + } + + writer := WriterOperator{ + OutputIDs: c.OutputIDs, + BasicOperator: basicOperator, + } + return writer, nil +} + +// SetNamespace will namespace the output ids of the writer. +func (c *WriterConfig) SetNamespace(namespace string, exclusions ...string) { + c.BasicConfig.SetNamespace(namespace, exclusions...) + for i, outputID := range c.OutputIDs { + if CanNamespace(outputID, exclusions) { + c.OutputIDs[i] = AddNamespace(outputID, namespace) + } + } +} + +// WriterOperator is an operator that can write to other operators. +type WriterOperator struct { + BasicOperator + OutputIDs OutputIDs + OutputOperators []operator.Operator +} + +// Write will write an entry to the outputs of the operator. +func (w *WriterOperator) Write(ctx context.Context, e *entry.Entry) { + for i, operator := range w.OutputOperators { + if i == len(w.OutputOperators)-1 { + _ = operator.Process(ctx, e) + return + } + operator.Process(ctx, e.Copy()) + } +} + +// CanOutput always returns true for a writer operator. +func (w *WriterOperator) CanOutput() bool { + return true +} + +// Outputs returns the outputs of the writer operator. +func (w *WriterOperator) Outputs() []operator.Operator { + return w.OutputOperators +} + +// SetOutputs will set the outputs of the operator. +func (w *WriterOperator) SetOutputs(operators []operator.Operator) error { + outputOperators := make([]operator.Operator, 0) + + for _, operatorID := range w.OutputIDs { + operator, ok := w.findOperator(operators, operatorID) + if !ok { + return fmt.Errorf("operator '%s' does not exist", operatorID) + } + + if !operator.CanProcess() { + return fmt.Errorf("operator '%s' can not process entries", operatorID) + } + + outputOperators = append(outputOperators, operator) + } + + // No outputs have been set, so use the next configured operator + if len(w.OutputIDs) == 0 { + currentOperatorIndex := -1 + for i, operator := range operators { + if operator.ID() == w.ID() { + currentOperatorIndex = i + break + } + } + if currentOperatorIndex == -1 { + return fmt.Errorf("unexpectedly could not find self in array of operators") + } + nextOperatorIndex := currentOperatorIndex + 1 + if nextOperatorIndex == len(operators) { + return fmt.Errorf("cannot omit output for the last operator in the pipeline") + } + nextOperator := operators[nextOperatorIndex] + if !nextOperator.CanProcess() { + return fmt.Errorf("operator '%s' cannot process entries, but it was selected as a receiver because 'output' was omitted", nextOperator.ID()) + } + outputOperators = append(outputOperators, nextOperator) + } + + w.OutputOperators = outputOperators + return nil +} + +// FindOperator will find an operator matching the supplied id. +func (w *WriterOperator) findOperator(operators []operator.Operator, operatorID string) (operator.Operator, bool) { + for _, operator := range operators { + if operator.ID() == operatorID { + return operator, true + } + } + return nil, false +} + +// OutputIDs is a collection of operator IDs used as outputs. +type OutputIDs []string + +// UnmarshalJSON will unmarshal a string or array of strings to OutputIDs. +func (o *OutputIDs) UnmarshalJSON(bytes []byte) error { + var value interface{} + err := json.Unmarshal(bytes, &value) + if err != nil { + return err + } + + ids, err := o.fromInterface(value) + if err != nil { + return err + } + + *o = ids + return nil +} + +// UnmarshalYAML will unmarshal a string or array of strings to OutputIDs. +func (o *OutputIDs) UnmarshalYAML(unmarshal func(interface{}) error) error { + var value interface{} + err := unmarshal(&value) + if err != nil { + return err + } + + ids, err := o.fromInterface(value) + if err != nil { + return err + } + + *o = ids + return nil +} + +// fromInterface will parse OutputIDs from a raw interface. +func (o *OutputIDs) fromInterface(value interface{}) (OutputIDs, error) { + if str, ok := value.(string); ok { + return OutputIDs{str}, nil + } + + if array, ok := value.([]interface{}); ok { + return o.fromArray(array) + } + + return nil, fmt.Errorf("value is not of type string or string array") +} + +// fromArray will parse OutputIDs from a raw array. +func (o *OutputIDs) fromArray(array []interface{}) (OutputIDs, error) { + ids := OutputIDs{} + for _, rawValue := range array { + strValue, ok := rawValue.(string) + if !ok { + return nil, fmt.Errorf("value in array is not of type string") + } + ids = append(ids, strValue) + } + return ids, nil +} diff --git a/plugin/helper/writer_test.go b/operator/helper/writer_test.go similarity index 82% rename from plugin/helper/writer_test.go rename to operator/helper/writer_test.go index 8d5387292..1f5476198 100644 --- a/plugin/helper/writer_test.go +++ b/operator/helper/writer_test.go @@ -7,7 +7,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" yaml "gopkg.in/yaml.v2" @@ -16,7 +16,7 @@ import ( func TestWriterConfigMissingOutput(t *testing.T) { config := WriterConfig{ BasicConfig: BasicConfig{ - PluginType: "testtype", + OperatorType: "testtype", }, } context := testutil.NewBuildContext(t) @@ -28,7 +28,7 @@ func TestWriterConfigValidBuild(t *testing.T) { config := WriterConfig{ OutputIDs: OutputIDs{"output"}, BasicConfig: BasicConfig{ - PluginType: "testtype", + OperatorType: "testtype", }, } context := testutil.NewBuildContext(t) @@ -44,13 +44,13 @@ func TestWriterConfigSetNamespace(t *testing.T) { require.Equal(t, OutputIDs{"namespace.output1", "namespace.output2"}, config.OutputIDs) } -func TestWriterPluginWrite(t *testing.T) { - output1 := &testutil.Plugin{} +func TestWriterOperatorWrite(t *testing.T) { + output1 := &testutil.Operator{} output1.On("Process", mock.Anything, mock.Anything).Return(nil) - output2 := &testutil.Plugin{} + output2 := &testutil.Operator{} output2.On("Process", mock.Anything, mock.Anything).Return(nil) - writer := WriterPlugin{ - OutputPlugins: []plugin.Plugin{output1, output2}, + writer := WriterOperator{ + OutputOperators: []operator.Operator{output1, output2}, } ctx := context.Background() @@ -61,18 +61,18 @@ func TestWriterPluginWrite(t *testing.T) { output2.AssertCalled(t, "Process", ctx, mock.Anything) } -func TestWriterPluginCanOutput(t *testing.T) { - writer := WriterPlugin{} +func TestWriterOperatorCanOutput(t *testing.T) { + writer := WriterOperator{} require.True(t, writer.CanOutput()) } -func TestWriterPluginOutputs(t *testing.T) { - output1 := &testutil.Plugin{} +func TestWriterOperatorOutputs(t *testing.T) { + output1 := &testutil.Operator{} output1.On("Process", mock.Anything, mock.Anything).Return(nil) - output2 := &testutil.Plugin{} + output2 := &testutil.Operator{} output2.On("Process", mock.Anything, mock.Anything).Return(nil) - writer := WriterPlugin{ - OutputPlugins: []plugin.Plugin{output1, output2}, + writer := WriterOperator{ + OutputOperators: []operator.Operator{output1, output2}, } ctx := context.Background() @@ -84,44 +84,44 @@ func TestWriterPluginOutputs(t *testing.T) { } func TestWriterSetOutputsMissing(t *testing.T) { - output1 := &testutil.Plugin{} + output1 := &testutil.Operator{} output1.On("ID").Return("output1") - writer := WriterPlugin{ + writer := WriterOperator{ OutputIDs: OutputIDs{"output2"}, } - err := writer.SetOutputs([]plugin.Plugin{output1}) + err := writer.SetOutputs([]operator.Operator{output1}) require.Error(t, err) require.Contains(t, err.Error(), "does not exist") } func TestWriterSetOutputsInvalid(t *testing.T) { - output1 := &testutil.Plugin{} + output1 := &testutil.Operator{} output1.On("ID").Return("output1") output1.On("CanProcess").Return(false) - writer := WriterPlugin{ + writer := WriterOperator{ OutputIDs: OutputIDs{"output1"}, } - err := writer.SetOutputs([]plugin.Plugin{output1}) + err := writer.SetOutputs([]operator.Operator{output1}) require.Error(t, err) require.Contains(t, err.Error(), "can not process entries") } func TestWriterSetOutputsValid(t *testing.T) { - output1 := &testutil.Plugin{} + output1 := &testutil.Operator{} output1.On("ID").Return("output1") output1.On("CanProcess").Return(true) - output2 := &testutil.Plugin{} + output2 := &testutil.Operator{} output2.On("ID").Return("output2") output2.On("CanProcess").Return(true) - writer := WriterPlugin{ + writer := WriterOperator{ OutputIDs: OutputIDs{"output1", "output2"}, } - err := writer.SetOutputs([]plugin.Plugin{output1, output2}) + err := writer.SetOutputs([]operator.Operator{output1, output2}) require.NoError(t, err) - require.Equal(t, []plugin.Plugin{output1, output2}, writer.Outputs()) + require.Equal(t, []operator.Operator{output1, output2}, writer.Outputs()) } func TestUnmarshalJSONString(t *testing.T) { diff --git a/operator/operator.go b/operator/operator.go new file mode 100644 index 000000000..7c1215054 --- /dev/null +++ b/operator/operator.go @@ -0,0 +1,37 @@ +//go:generate mockery -name=^(Operator)$ -output=../internal/testutil -outpkg=testutil -case=snake + +package operator + +import ( + "context" + + "github.com/observiq/carbon/entry" + "go.uber.org/zap" +) + +// Operator is a log monitoring component. +type Operator interface { + // ID returns the id of the operator. + ID() string + // Type returns the type of the operator. + Type() string + + // Start will start the operator. + Start() error + // Stop will stop the operator. + Stop() error + + // CanOutput indicates if the operator will output entries to other operators. + CanOutput() bool + // Outputs returns the list of connected outputs. + Outputs() []Operator + // SetOutputs will set the connected outputs. + SetOutputs([]Operator) error + + // CanProcess indicates if the operator will process entries from other operators. + CanProcess() bool + // Process will process an entry from an operator. + Process(context.Context, *entry.Entry) error + // Logger returns the operator's logger + Logger() *zap.SugaredLogger +} diff --git a/plugin/custom.go b/operator/plugin.go similarity index 53% rename from plugin/custom.go rename to operator/plugin.go index bd336dec1..6e54f27d6 100644 --- a/plugin/custom.go +++ b/operator/plugin.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "bytes" @@ -12,31 +12,31 @@ import ( yaml "gopkg.in/yaml.v2" ) -// CustomConfig is the rendered config of a custom plugin. -type CustomConfig struct { +// PluginConfig is the rendered config of a plugin. +type PluginConfig struct { Version string Title string Description string - Parameters map[string]CustomParameter + Parameters map[string]PluginParameter Pipeline []Config } -// CustomParameter is a basic description of a custom plugin's parameter. -type CustomParameter struct { +// PluginParameter is a basic description of a plugin's parameter. +type PluginParameter struct { Label string Description string Type string } -// CustomRegistry is a registry of custom plugin templates. -type CustomRegistry map[string]*template.Template +// PluginRegistry is a registry of plugin templates. +type PluginRegistry map[string]*template.Template -// Render will render a custom config using the params and plugin type. -func (r CustomRegistry) Render(pluginType string, params map[string]interface{}) (CustomConfig, error) { +// Render will render a plugin config using the params and plugin type. +func (r PluginRegistry) Render(pluginType string, params map[string]interface{}) (PluginConfig, error) { template, ok := r[pluginType] if !ok { - return CustomConfig{}, errors.NewError( - "custom plugin type does not exist", + return PluginConfig{}, errors.NewError( + "plugin type does not exist", "ensure that all plugins are defined with a registered type", "plugin_type", pluginType, ) @@ -44,19 +44,19 @@ func (r CustomRegistry) Render(pluginType string, params map[string]interface{}) var writer bytes.Buffer if err := template.Execute(&writer, params); err != nil { - return CustomConfig{}, errors.NewError( - "failed to render template for custom plugin", - "ensure that all parameters are valid for the custom plugin", + return PluginConfig{}, errors.NewError( + "failed to render template for plugin", + "ensure that all parameters are valid for the plugin", "plugin_type", pluginType, "error_message", err.Error(), ) } - var config CustomConfig + var config PluginConfig if err := yaml.UnmarshalStrict(writer.Bytes(), &config); err != nil { - return CustomConfig{}, errors.NewError( - "failed to unmarshal custom template to custom config", - "ensure that the custom template renders a valid pipeline", + return PluginConfig{}, errors.NewError( + "failed to unmarshal plugin template to plugin config", + "ensure that the plugin template renders a valid pipeline", "plugin_type", pluginType, "rendered_config", writer.String(), "error_message", err.Error(), @@ -66,19 +66,19 @@ func (r CustomRegistry) Render(pluginType string, params map[string]interface{}) return config, nil } -// IsDefined returns a boolean indicating if a custom plugin is defined and registered. -func (r CustomRegistry) IsDefined(pluginType string) bool { +// IsDefined returns a boolean indicating if a plugin is defined and registered. +func (r PluginRegistry) IsDefined(pluginType string) bool { _, ok := r[pluginType] return ok } -// LoadAll will load all custom plugin templates contained in a directory. -func (r CustomRegistry) LoadAll(dir string, pattern string) error { +// LoadAll will load all plugin templates contained in a directory. +func (r PluginRegistry) LoadAll(dir string, pattern string) error { glob := filepath.Join(dir, pattern) filePaths, err := filepath.Glob(glob) if err != nil { return errors.NewError( - "failed to find custom plugins with glob pattern", + "failed to find plugins with glob pattern", "ensure that the plugin directory and file pattern are valid", "glob_pattern", glob, ) @@ -93,7 +93,7 @@ func (r CustomRegistry) LoadAll(dir string, pattern string) error { if len(failures) > 0 { return errors.NewError( - "failed to load all custom plugins", + "failed to load all plugins", "review the list of failures for more information", "failures", strings.Join(failures, ", "), ) @@ -102,8 +102,8 @@ func (r CustomRegistry) LoadAll(dir string, pattern string) error { return nil } -// Load will load a custom plugin template from a file path. -func (r CustomRegistry) Load(path string) error { +// Load will load a plugin template from a file path. +func (r PluginRegistry) Load(path string) error { fileName := filepath.Base(path) pluginType := strings.TrimSuffix(fileName, filepath.Ext(fileName)) @@ -115,24 +115,24 @@ func (r CustomRegistry) Load(path string) error { return r.Add(pluginType, string(fileContents)) } -// Add will add a custom plugin to the registry. -func (r CustomRegistry) Add(pluginType string, contents string) error { +// Add will add a plugin to the registry. +func (r PluginRegistry) Add(pluginType string, contents string) error { if IsDefined(pluginType) { return fmt.Errorf("plugin type %s already exists as a builtin plugin", pluginType) } pluginTemplate, err := template.New(pluginType).Parse(contents) if err != nil { - return fmt.Errorf("failed to parse %s as a custom template: %s", pluginType, err) + return fmt.Errorf("failed to parse %s as a plugin template: %s", pluginType, err) } r[pluginType] = pluginTemplate return nil } -// NewCustomRegistry creates a new custom plugin registry from a plugin directory. -func NewCustomRegistry(dir string) (CustomRegistry, error) { - registry := CustomRegistry{} +// NewPluginRegistry creates a new plugin registry from a plugin directory. +func NewPluginRegistry(dir string) (PluginRegistry, error) { + registry := PluginRegistry{} if err := registry.LoadAll(dir, "*.yaml"); err != nil { return registry, err } diff --git a/plugin/custom_test.go b/operator/plugin_test.go similarity index 88% rename from plugin/custom_test.go rename to operator/plugin_test.go index 6472553ae..43d9b421e 100644 --- a/plugin/custom_test.go +++ b/operator/plugin_test.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "io/ioutil" @@ -23,7 +23,7 @@ func NewTempDir(t *testing.T) string { return tempDir } -func TestCustomRegistry_LoadAll(t *testing.T) { +func TestPluginRegistry_LoadAll(t *testing.T) { tempDir, err := ioutil.TempDir("", "") require.NoError(t, err) t.Cleanup(func() { @@ -51,48 +51,48 @@ record: err = ioutil.WriteFile(filepath.Join(tempDir, "test2.yaml"), test2, 0666) require.NoError(t, err) - customRegistry := CustomRegistry{} - err = customRegistry.LoadAll(tempDir, "*.yaml") + pluginRegistry := PluginRegistry{} + err = pluginRegistry.LoadAll(tempDir, "*.yaml") require.NoError(t, err) - require.Equal(t, 2, len(customRegistry)) + require.Equal(t, 2, len(pluginRegistry)) } -func TestCustomRegistryRender(t *testing.T) { +func TestPluginRegistryRender(t *testing.T) { t.Run("ErrorTypeDoesNotExist", func(t *testing.T) { - reg := CustomRegistry{} + reg := PluginRegistry{} _, err := reg.Render("unknown", map[string]interface{}{}) require.Error(t, err) require.Contains(t, err.Error(), "does not exist") }) t.Run("ErrorExecFailure", func(t *testing.T) { - tmpl, err := template.New("customtype").Parse(`{{ .panicker }}`) + tmpl, err := template.New("plugintype").Parse(`{{ .panicker }}`) require.NoError(t, err) - reg := CustomRegistry{ - "customtype": tmpl, + reg := PluginRegistry{ + "plugintype": tmpl, } params := map[string]interface{}{ "panicker": func() { panic("testpanic") }, } - _, err = reg.Render("customtype", params) + _, err = reg.Render("plugintype", params) require.Contains(t, err.Error(), "failed to render") }) } -func TestCustomRegistryLoad(t *testing.T) { +func TestPluginRegistryLoad(t *testing.T) { t.Run("LoadAllBadGlob", func(t *testing.T) { - reg := CustomRegistry{} + reg := PluginRegistry{} err := reg.LoadAll("", `[]`) require.Error(t, err) require.Contains(t, err.Error(), "with glob pattern") }) t.Run("AddDuplicate", func(t *testing.T) { - reg := CustomRegistry{} + reg := PluginRegistry{} Register("copy", func() Builder { return nil }) err := reg.Add("copy", "pipeline:\n") require.Error(t, err) @@ -100,10 +100,10 @@ func TestCustomRegistryLoad(t *testing.T) { }) t.Run("AddBadTemplate", func(t *testing.T) { - reg := CustomRegistry{} + reg := PluginRegistry{} err := reg.Add("new", "{{ nofunc }") require.Error(t, err) - require.Contains(t, err.Error(), "as a custom template") + require.Contains(t, err.Error(), "as a plugin template") }) t.Run("LoadAllWithFailures", func(t *testing.T) { @@ -112,13 +112,13 @@ func TestCustomRegistryLoad(t *testing.T) { err := ioutil.WriteFile(pluginPath, []byte("pipeline:\n"), 0755) require.NoError(t, err) - reg := CustomRegistry{} + reg := PluginRegistry{} err = reg.LoadAll(tempDir, "*.yaml") require.Error(t, err) }) } -func TestCustomPluginMetadata(t *testing.T) { +func TestPluginMetadata(t *testing.T) { testCases := []struct { name string @@ -280,7 +280,7 @@ pipeline: for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - reg := CustomRegistry{} + reg := PluginRegistry{} err := reg.Add(tc.name, tc.template) require.NoError(t, err) _, err = reg.Render(tc.name, map[string]interface{}{}) diff --git a/pipeline/config.go b/pipeline/config.go index 65ee521e7..35867887e 100644 --- a/pipeline/config.go +++ b/pipeline/config.go @@ -5,8 +5,8 @@ import ( "strings" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" yaml "gopkg.in/yaml.v2" ) @@ -14,18 +14,18 @@ import ( type Config []Params // BuildPipeline will build a pipeline from the config. -func (c Config) BuildPipeline(context plugin.BuildContext) (*Pipeline, error) { - pluginConfigs, err := c.buildPluginConfigs(context.CustomRegistry) +func (c Config) BuildPipeline(context operator.BuildContext) (*Pipeline, error) { + operatorConfigs, err := c.buildOperatorConfigs(context.PluginRegistry) if err != nil { return nil, err } - plugins, err := c.buildPlugins(pluginConfigs, context) + operators, err := c.buildOperators(operatorConfigs, context) if err != nil { return nil, err } - pipeline, err := NewPipeline(plugins) + pipeline, err := NewPipeline(operators) if err != nil { return nil, err } @@ -33,43 +33,43 @@ func (c Config) BuildPipeline(context plugin.BuildContext) (*Pipeline, error) { return pipeline, nil } -func (c Config) buildPlugins(pluginConfigs []plugin.Config, context plugin.BuildContext) ([]plugin.Plugin, error) { - plugins := make([]plugin.Plugin, 0, len(pluginConfigs)) - for _, pluginConfig := range pluginConfigs { - plugin, err := pluginConfig.Build(context) +func (c Config) buildOperators(operatorConfigs []operator.Config, context operator.BuildContext) ([]operator.Operator, error) { + operators := make([]operator.Operator, 0, len(operatorConfigs)) + for _, operatorConfig := range operatorConfigs { + operator, err := operatorConfig.Build(context) if err != nil { return nil, errors.WithDetails(err, - "plugin_id", pluginConfig.ID(), - "plugin_type", pluginConfig.Type(), + "operator_id", operatorConfig.ID(), + "operator_type", operatorConfig.Type(), ) } - plugins = append(plugins, plugin) + operators = append(operators, operator) } - return plugins, nil + return operators, nil } -func (c Config) buildPluginConfigs(customRegistry plugin.CustomRegistry) ([]plugin.Config, error) { - pluginConfigs := make([]plugin.Config, 0, len(c)) +func (c Config) buildOperatorConfigs(pluginRegistry operator.PluginRegistry) ([]operator.Config, error) { + operatorConfigs := make([]operator.Config, 0, len(c)) for _, params := range c { if err := params.Validate(); err != nil { return nil, errors.Wrap(err, "validate config params") } - configs, err := params.BuildConfigs(customRegistry, "$") + configs, err := params.BuildConfigs(pluginRegistry, "$") if err != nil { - return nil, errors.Wrap(err, "build plugin configs") + return nil, errors.Wrap(err, "build operator configs") } - pluginConfigs = append(pluginConfigs, configs...) + operatorConfigs = append(operatorConfigs, configs...) } - return pluginConfigs, nil + return operatorConfigs, nil } -// Params is a raw params map that can be converted into a plugin config. +// Params is a raw params map that can be converted into an operator config. type Params map[string]interface{} // ID returns the id field in the params map. @@ -124,12 +124,12 @@ func (p Params) NamespaceExclusions(namespace string) []string { return exclusions } -// Validate will validate the basic fields required to make a plugin config. +// Validate will validate the basic fields required to make an operator config. func (p Params) Validate() error { if p.Type() == "" { return errors.NewError( - "missing required `type` field for plugin config", - "ensure that all plugin configs have a defined type field", + "missing required `type` field for operator config", + "ensure that all operator configs have a defined type field", "id", p.ID(), ) } @@ -177,26 +177,26 @@ func (p Params) getStringArray(key string) []string { } } -// BuildConfigs will build plugin configs from a params map. -func (p Params) BuildConfigs(customRegistry plugin.CustomRegistry, namespace string) ([]plugin.Config, error) { - if plugin.IsDefined(p.Type()) { +// BuildConfigs will build operator configs from a params map. +func (p Params) BuildConfigs(pluginRegistry operator.PluginRegistry, namespace string) ([]operator.Config, error) { + if operator.IsDefined(p.Type()) { return p.buildAsBuiltin(namespace) } - if customRegistry.IsDefined(p.Type()) { - return p.buildAsCustom(customRegistry, namespace) + if pluginRegistry.IsDefined(p.Type()) { + return p.buildPlugin(pluginRegistry, namespace) } return nil, errors.NewError( - "unsupported `type` for plugin config", - "ensure that all plugins have a supported builtin or custom type", + "unsupported `type` for operator config", + "ensure that all operators have a supported builtin or plugin type", "type", p.Type(), "id", p.ID(), ) } // buildAsBuiltin will build a builtin config from a params map. -func (p Params) buildAsBuiltin(namespace string) ([]plugin.Config, error) { +func (p Params) buildAsBuiltin(namespace string) ([]operator.Config, error) { bytes, err := yaml.Marshal(p) if err != nil { return nil, errors.NewError( @@ -206,17 +206,17 @@ func (p Params) buildAsBuiltin(namespace string) ([]plugin.Config, error) { ) } - var config plugin.Config + var config operator.Config if err := yaml.UnmarshalStrict(bytes, &config); err != nil { return nil, err } config.SetNamespace(namespace) - return []plugin.Config{config}, nil + return []operator.Config{config}, nil } -// buildAsCustom will build a custom config from a params map. -func (p Params) buildAsCustom(customRegistry plugin.CustomRegistry, namespace string) ([]plugin.Config, error) { +// buildPlugin will build a plugin config from a params map. +func (p Params) buildPlugin(pluginRegistry operator.PluginRegistry, namespace string) ([]operator.Config, error) { templateParams := map[string]interface{}{} for key, value := range p { templateParams[key] = value @@ -225,15 +225,15 @@ func (p Params) buildAsCustom(customRegistry plugin.CustomRegistry, namespace st templateParams["input"] = p.TemplateInput(namespace) templateParams["output"] = p.TemplateOutput(namespace) - config, err := customRegistry.Render(p.Type(), templateParams) + config, err := pluginRegistry.Render(p.Type(), templateParams) if err != nil { - return nil, errors.Wrap(err, "failed to render custom config") + return nil, errors.Wrap(err, "failed to render plugin config") } exclusions := p.NamespaceExclusions(namespace) - for _, pluginConfig := range config.Pipeline { + for _, operatorConfig := range config.Pipeline { innerNamespace := p.NamespacedID(namespace) - pluginConfig.SetNamespace(innerNamespace, exclusions...) + operatorConfig.SetNamespace(innerNamespace, exclusions...) } return config.Pipeline, nil diff --git a/pipeline/config_test.go b/pipeline/config_test.go index a00a4bbe6..d244d7af7 100644 --- a/pipeline/config_test.go +++ b/pipeline/config_test.go @@ -5,9 +5,9 @@ import ( "fmt" "testing" - "github.com/observiq/carbon/plugin" - _ "github.com/observiq/carbon/plugin/builtin" - "github.com/observiq/carbon/plugin/builtin/transformer" + "github.com/observiq/carbon/operator" + _ "github.com/observiq/carbon/operator/builtin" + "github.com/observiq/carbon/operator/builtin/transformer" "github.com/stretchr/testify/require" "go.uber.org/zap" yaml "gopkg.in/yaml.v2" @@ -188,7 +188,7 @@ func TestBuildBuiltinFromParamsWithUnsupportedYaml(t *testing.T) { "output": "test", "field": invalidMarshaller{}, } - _, err := params.BuildConfigs(plugin.CustomRegistry{}, "test_namespace") + _, err := params.BuildConfigs(operator.PluginRegistry{}, "test_namespace") require.Error(t, err) require.Contains(t, err.Error(), "failed to parse config map as yaml") } @@ -200,7 +200,7 @@ func TestBuildBuiltinFromParamsWithUnknownField(t *testing.T) { "unknown": true, "output": "test_output", } - _, err := params.BuildConfigs(plugin.CustomRegistry{}, "test_namespace") + _, err := params.BuildConfigs(operator.PluginRegistry{}, "test_namespace") require.Error(t, err) } @@ -210,43 +210,43 @@ func TestBuildBuiltinFromValidParams(t *testing.T) { "type": "noop", "output": "test_output", } - configs, err := params.BuildConfigs(plugin.CustomRegistry{}, "test_namespace") + configs, err := params.BuildConfigs(operator.PluginRegistry{}, "test_namespace") require.NoError(t, err) require.Equal(t, 1, len(configs)) - require.IsType(t, &transformer.NoopPluginConfig{}, configs[0].Builder) + require.IsType(t, &transformer.NoopOperatorConfig{}, configs[0].Builder) require.Equal(t, "test_namespace.noop", configs[0].ID()) } -func TestBuildCustomFromValidParams(t *testing.T) { - registry := plugin.CustomRegistry{} - customTemplate := ` +func TestBuildPluginFromValidParams(t *testing.T) { + registry := operator.PluginRegistry{} + pluginTemplate := ` pipeline: - - id: custom_noop + - id: plugin_noop type: noop output: {{.output}} ` - err := registry.Add("custom_plugin", customTemplate) + err := registry.Add("plugin", pluginTemplate) require.NoError(t, err) params := Params{ - "id": "custom_plugin", - "type": "custom_plugin", + "id": "plugin", + "type": "plugin", "output": "test_output", } configs, err := params.BuildConfigs(registry, "test_namespace") require.NoError(t, err) require.Equal(t, 1, len(configs)) - require.IsType(t, &transformer.NoopPluginConfig{}, configs[0].Builder) - require.Equal(t, "test_namespace.custom_plugin.custom_noop", configs[0].ID()) + require.IsType(t, &transformer.NoopOperatorConfig{}, configs[0].Builder) + require.Equal(t, "test_namespace.plugin.plugin_noop", configs[0].ID()) } func TestBuildValidPipeline(t *testing.T) { - registry := plugin.CustomRegistry{} - customTemplate := ` + registry := operator.PluginRegistry{} + pluginTemplate := ` pipeline: - - id: custom_generate + - id: plugin_generate type: generate_input count: 1 entry: @@ -254,22 +254,22 @@ pipeline: message: test output: {{.output}} ` - err := registry.Add("custom_plugin", customTemplate) + err := registry.Add("plugin", pluginTemplate) require.NoError(t, err) logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) - context := plugin.BuildContext{ - CustomRegistry: registry, + context := operator.BuildContext{ + PluginRegistry: registry, Logger: logger.Sugar(), } pipelineConfig := Config{ Params{ - "id": "custom_plugin", - "type": "custom_plugin", + "id": "plugin", + "type": "plugin", "output": "drop_output", }, Params{ @@ -283,20 +283,20 @@ pipeline: } func TestBuildInvalidPipelineInvalidType(t *testing.T) { - registry := plugin.CustomRegistry{} + registry := operator.PluginRegistry{} logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) - context := plugin.BuildContext{ - CustomRegistry: registry, + context := operator.BuildContext{ + PluginRegistry: registry, Logger: logger.Sugar(), } pipelineConfig := Config{ Params{ - "id": "custom_plugin", - "type": "custom_plugin", + "id": "plugin", + "type": "plugin", "output": "drop_output", }, Params{ @@ -307,36 +307,36 @@ func TestBuildInvalidPipelineInvalidType(t *testing.T) { _, err = pipelineConfig.BuildPipeline(context) require.Error(t, err) - require.Contains(t, err.Error(), "unsupported `type` for plugin config") + require.Contains(t, err.Error(), "unsupported `type` for operator config") } func TestBuildInvalidPipelineInvalidParam(t *testing.T) { - registry := plugin.CustomRegistry{} - customTemplate := ` + registry := operator.PluginRegistry{} + pluginTemplate := ` pipeline: - - id: custom_generate + - id: plugin_generate type: generate_input count: invalid_value record: message: test output: {{.output}} ` - err := registry.Add("custom_plugin", customTemplate) + err := registry.Add("plugin", pluginTemplate) require.NoError(t, err) logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) - context := plugin.BuildContext{ - CustomRegistry: registry, + context := operator.BuildContext{ + PluginRegistry: registry, Logger: logger.Sugar(), } pipelineConfig := Config{ Params{ - "id": "custom_plugin", - "type": "custom_plugin", + "id": "plugin", + "type": "plugin", "output": "drop_output", }, Params{ @@ -347,17 +347,17 @@ pipeline: _, err = pipelineConfig.BuildPipeline(context) require.Error(t, err) - require.Contains(t, err.Error(), "build plugin configs") + require.Contains(t, err.Error(), "build operator configs") } -func TestBuildInvalidPipelineInvalidPlugin(t *testing.T) { - registry := plugin.CustomRegistry{} +func TestBuildInvalidPipelineInvalidOperator(t *testing.T) { + registry := operator.PluginRegistry{} logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) - context := plugin.BuildContext{ - CustomRegistry: registry, + context := operator.BuildContext{ + PluginRegistry: registry, Logger: logger.Sugar(), } @@ -379,13 +379,13 @@ func TestBuildInvalidPipelineInvalidPlugin(t *testing.T) { } func TestBuildInvalidPipelineInvalidGraph(t *testing.T) { - registry := plugin.CustomRegistry{} + registry := operator.PluginRegistry{} logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) - context := plugin.BuildContext{ - CustomRegistry: registry, + context := operator.BuildContext{ + PluginRegistry: registry, Logger: logger.Sugar(), } diff --git a/pipeline/node.go b/pipeline/node.go index 6b027f72b..316a0822e 100644 --- a/pipeline/node.go +++ b/pipeline/node.go @@ -3,51 +3,51 @@ package pipeline import ( "hash/fnv" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" ) -// PluginNode is a basic node that represents a plugin in a pipeline. -type PluginNode struct { - plugin plugin.Plugin +// OperatorNode is a basic node that represents an operator in a pipeline. +type OperatorNode struct { + operator operator.Operator id int64 outputIDs map[string]int64 } -// Plugin returns the plugin of the node. -func (b PluginNode) Plugin() plugin.Plugin { - return b.plugin +// Operator returns the operator of the node. +func (b OperatorNode) Operator() operator.Operator { + return b.operator } // ID returns the node id. -func (b PluginNode) ID() int64 { +func (b OperatorNode) ID() int64 { return b.id } // DOTID returns the id used to represent this node in a dot graph. -func (b PluginNode) DOTID() string { - return b.plugin.ID() +func (b OperatorNode) DOTID() string { + return b.operator.ID() } -// OutputIDs returns a map of output plugin ids to node ids. -func (b PluginNode) OutputIDs() map[string]int64 { +// OutputIDs returns a map of output operator ids to node ids. +func (b OperatorNode) OutputIDs() map[string]int64 { return b.outputIDs } -// createPluginNode will create a plugin node. -func createPluginNode(plugin plugin.Plugin) PluginNode { - id := createNodeID(plugin.ID()) +// createOperatorNode will create an operator node. +func createOperatorNode(operator operator.Operator) OperatorNode { + id := createNodeID(operator.ID()) outputIDs := make(map[string]int64) - if plugin.CanOutput() { - for _, output := range plugin.Outputs() { + if operator.CanOutput() { + for _, output := range operator.Outputs() { outputIDs[output.ID()] = createNodeID(output.ID()) } } - return PluginNode{plugin, id, outputIDs} + return OperatorNode{operator, id, outputIDs} } -// createNodeID generates a node id from a plugin id. -func createNodeID(pluginID string) int64 { +// createNodeID generates a node id from an operator id. +func createNodeID(operatorID string) int64 { hash := fnv.New64a() - _, _ = hash.Write([]byte(pluginID)) + _, _ = hash.Write([]byte(operatorID)) return int64(hash.Sum64()) } diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index bd8ec0339..162444d26 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -5,19 +5,19 @@ import ( "strings" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "gonum.org/v1/gonum/graph/encoding/dot" "gonum.org/v1/gonum/graph/simple" "gonum.org/v1/gonum/graph/topo" ) -// Pipeline is a directed graph of connected plugins. +// Pipeline is a directed graph of connected operators. type Pipeline struct { Graph *simple.DirectedGraph running bool } -// Start will start the plugins in a pipeline in reverse topological order. +// Start will start the operators in a pipeline in reverse topological order. func (p *Pipeline) Start() error { if p.running { return nil @@ -25,19 +25,19 @@ func (p *Pipeline) Start() error { sortedNodes, _ := topo.Sort(p.Graph) for i := len(sortedNodes) - 1; i >= 0; i-- { - plugin := sortedNodes[i].(PluginNode).Plugin() - plugin.Logger().Debug("Starting plugin") - if err := plugin.Start(); err != nil { + operator := sortedNodes[i].(OperatorNode).Operator() + operator.Logger().Debug("Starting operator") + if err := operator.Start(); err != nil { return err } - plugin.Logger().Debug("Started plugin") + operator.Logger().Debug("Started operator") } p.running = true return nil } -// Stop will stop the plugins in a pipeline in topological order. +// Stop will stop the operators in a pipeline in topological order. func (p *Pipeline) Stop() { if !p.running { return @@ -45,10 +45,10 @@ func (p *Pipeline) Stop() { sortedNodes, _ := topo.Sort(p.Graph) for _, node := range sortedNodes { - plugin := node.(PluginNode).Plugin() - plugin.Logger().Debug("Stopping plugin") - _ = plugin.Stop() - plugin.Logger().Debug("Stopped plugin") + operator := node.(OperatorNode).Operator() + operator.Logger().Debug("Stopping operator") + _ = operator.Stop() + operator.Logger().Debug("Stopped operator") } p.running = false @@ -59,18 +59,18 @@ func (p *Pipeline) MarshalDot() ([]byte, error) { return dot.Marshal(p.Graph, "G", "", " ") } -// addNodes will add plugins as nodes to the supplied graph. -func addNodes(graph *simple.DirectedGraph, plugins []plugin.Plugin) error { - for _, plugin := range plugins { - pluginNode := createPluginNode(plugin) - if graph.Node(pluginNode.ID()) != nil { +// addNodes will add operators as nodes to the supplied graph. +func addNodes(graph *simple.DirectedGraph, operators []operator.Operator) error { + for _, operator := range operators { + operatorNode := createOperatorNode(operator) + if graph.Node(operatorNode.ID()) != nil { return errors.NewError( - fmt.Sprintf("plugin with id '%s' already exists in pipeline", pluginNode.Plugin().ID()), - "ensure that each plugin has a unique `type` or `id`", + fmt.Sprintf("operator with id '%s' already exists in pipeline", operatorNode.Operator().ID()), + "ensure that each operator has a unique `type` or `id`", ) } - graph.AddNode(pluginNode) + graph.AddNode(operatorNode) } return nil } @@ -79,7 +79,7 @@ func addNodes(graph *simple.DirectedGraph, plugins []plugin.Plugin) error { func connectNodes(graph *simple.DirectedGraph) error { nodes := graph.Nodes() for nodes.Next() { - node := nodes.Node().(PluginNode) + node := nodes.Node().(OperatorNode) if err := connectNode(graph, node); err != nil { return err } @@ -88,7 +88,7 @@ func connectNodes(graph *simple.DirectedGraph) error { if _, err := topo.Sort(graph); err != nil { return errors.NewError( "pipeline has a circular dependency", - "ensure that all plugins are connected in a straight, acyclic line", + "ensure that all operators are connected in a straight, acyclic line", "cycles", unorderableToCycles(err.(topo.Unorderable)), ) } @@ -97,33 +97,33 @@ func connectNodes(graph *simple.DirectedGraph) error { } // connectNode will connect a node to its outputs in the supplied graph. -func connectNode(graph *simple.DirectedGraph, inputNode PluginNode) error { - for outputPluginID, outputNodeID := range inputNode.OutputIDs() { +func connectNode(graph *simple.DirectedGraph, inputNode OperatorNode) error { + for outputOperatorID, outputNodeID := range inputNode.OutputIDs() { if graph.Node(outputNodeID) == nil { return errors.NewError( - "plugins cannot be connected, because the output does not exist in the pipeline", - "ensure that the output plugin is defined", - "input_plugin", inputNode.Plugin().ID(), - "output_plugin", outputPluginID, + "operators cannot be connected, because the output does not exist in the pipeline", + "ensure that the output operator is defined", + "input_operator", inputNode.Operator().ID(), + "output_operator", outputOperatorID, ) } - outputNode := graph.Node(outputNodeID).(PluginNode) - if !outputNode.Plugin().CanProcess() { + outputNode := graph.Node(outputNodeID).(OperatorNode) + if !outputNode.Operator().CanProcess() { return errors.NewError( - "plugins cannot be connected, because the output plugin can not process logs", - "ensure that the output plugin can process logs (like a parser or destination)", - "input_plugin", inputNode.Plugin().ID(), - "output_plugin", outputPluginID, + "operators cannot be connected, because the output operator can not process logs", + "ensure that the output operator can process logs (like a parser or destination)", + "input_operator", inputNode.Operator().ID(), + "output_operator", outputOperatorID, ) } if graph.HasEdgeFromTo(inputNode.ID(), outputNodeID) { return errors.NewError( - "plugins cannot be connected, because a connection already exists", - "ensure that only a single connection exists between the two plugins", - "input_plugin", inputNode.Plugin().ID(), - "output_plugin", outputPluginID, + "operators cannot be connected, because a connection already exists", + "ensure that only a single connection exists between the two operators", + "input_operator", inputNode.Operator().ID(), + "output_operator", outputOperatorID, ) } @@ -134,28 +134,28 @@ func connectNode(graph *simple.DirectedGraph, inputNode PluginNode) error { return nil } -// setPluginOutputs will set the outputs on plugins that can output. -func setPluginOutputs(plugins []plugin.Plugin) error { - for _, plugin := range plugins { - if !plugin.CanOutput() { +// setOperatorOutputs will set the outputs on operators that can output. +func setOperatorOutputs(operators []operator.Operator) error { + for _, operator := range operators { + if !operator.CanOutput() { continue } - if err := plugin.SetOutputs(plugins); err != nil { - return errors.WithDetails(err, "plugin_id", plugin.ID()) + if err := operator.SetOutputs(operators); err != nil { + return errors.WithDetails(err, "operator_id", operator.ID()) } } return nil } -// NewPipeline creates a new pipeline of connected plugins. -func NewPipeline(plugins []plugin.Plugin) (*Pipeline, error) { - if err := setPluginOutputs(plugins); err != nil { +// NewPipeline creates a new pipeline of connected operators. +func NewPipeline(operators []operator.Operator) (*Pipeline, error) { + if err := setOperatorOutputs(operators); err != nil { return nil, err } graph := simple.NewDirectedGraph() - if err := addNodes(graph, plugins); err != nil { + if err := addNodes(graph, operators); err != nil { return nil, err } @@ -174,10 +174,10 @@ func unorderableToCycles(err topo.Unorderable) string { } cycles.WriteByte('(') for _, node := range cycle { - cycles.WriteString(node.(PluginNode).plugin.ID()) + cycles.WriteString(node.(OperatorNode).operator.ID()) cycles.Write([]byte(` -> `)) } - cycles.WriteString(cycle[0].(PluginNode).plugin.ID()) + cycles.WriteString(cycle[0].(OperatorNode).operator.ID()) cycles.WriteByte(')') } return cycles.String() diff --git a/pipeline/pipeline_test.go b/pipeline/pipeline_test.go index ddb3a44a6..edd26abc3 100644 --- a/pipeline/pipeline_test.go +++ b/pipeline/pipeline_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "gonum.org/v1/gonum/graph" @@ -14,52 +14,52 @@ import ( func TestUnorderableToCycles(t *testing.T) { t.Run("SingleCycle", func(t *testing.T) { - mockPlugin1 := testutil.NewMockPlugin("plugin1") - mockPlugin2 := testutil.NewMockPlugin("plugin2") - mockPlugin3 := testutil.NewMockPlugin("plugin3") - mockPlugin1.On("Outputs").Return([]plugin.Plugin{mockPlugin2}) - mockPlugin2.On("Outputs").Return([]plugin.Plugin{mockPlugin3}) - mockPlugin3.On("Outputs").Return([]plugin.Plugin{mockPlugin1}) + mockOperator1 := testutil.NewMockOperator("operator1") + mockOperator2 := testutil.NewMockOperator("operator2") + mockOperator3 := testutil.NewMockOperator("operator3") + mockOperator1.On("Outputs").Return([]operator.Operator{mockOperator2}) + mockOperator2.On("Outputs").Return([]operator.Operator{mockOperator3}) + mockOperator3.On("Outputs").Return([]operator.Operator{mockOperator1}) err := topo.Unorderable([][]graph.Node{{ - createPluginNode(mockPlugin1), - createPluginNode(mockPlugin2), - createPluginNode(mockPlugin3), + createOperatorNode(mockOperator1), + createOperatorNode(mockOperator2), + createOperatorNode(mockOperator3), }}) output := unorderableToCycles(err) - expected := `(plugin1 -> plugin2 -> plugin3 -> plugin1)` + expected := `(operator1 -> operator2 -> operator3 -> operator1)` require.Equal(t, expected, output) }) t.Run("MultipleCycles", func(t *testing.T) { - mockPlugin1 := testutil.NewMockPlugin("plugin1") - mockPlugin2 := testutil.NewMockPlugin("plugin2") - mockPlugin3 := testutil.NewMockPlugin("plugin3") - mockPlugin1.On("Outputs").Return([]plugin.Plugin{mockPlugin2}) - mockPlugin2.On("Outputs").Return([]plugin.Plugin{mockPlugin3}) - mockPlugin3.On("Outputs").Return([]plugin.Plugin{mockPlugin1}) - - mockPlugin4 := testutil.NewMockPlugin("plugin4") - mockPlugin5 := testutil.NewMockPlugin("plugin5") - mockPlugin6 := testutil.NewMockPlugin("plugin6") - mockPlugin4.On("Outputs").Return([]plugin.Plugin{mockPlugin5}) - mockPlugin5.On("Outputs").Return([]plugin.Plugin{mockPlugin6}) - mockPlugin6.On("Outputs").Return([]plugin.Plugin{mockPlugin4}) + mockOperator1 := testutil.NewMockOperator("operator1") + mockOperator2 := testutil.NewMockOperator("operator2") + mockOperator3 := testutil.NewMockOperator("operator3") + mockOperator1.On("Outputs").Return([]operator.Operator{mockOperator2}) + mockOperator2.On("Outputs").Return([]operator.Operator{mockOperator3}) + mockOperator3.On("Outputs").Return([]operator.Operator{mockOperator1}) + + mockOperator4 := testutil.NewMockOperator("operator4") + mockOperator5 := testutil.NewMockOperator("operator5") + mockOperator6 := testutil.NewMockOperator("operator6") + mockOperator4.On("Outputs").Return([]operator.Operator{mockOperator5}) + mockOperator5.On("Outputs").Return([]operator.Operator{mockOperator6}) + mockOperator6.On("Outputs").Return([]operator.Operator{mockOperator4}) err := topo.Unorderable([][]graph.Node{{ - createPluginNode(mockPlugin1), - createPluginNode(mockPlugin2), - createPluginNode(mockPlugin3), + createOperatorNode(mockOperator1), + createOperatorNode(mockOperator2), + createOperatorNode(mockOperator3), }, { - createPluginNode(mockPlugin4), - createPluginNode(mockPlugin5), - createPluginNode(mockPlugin6), + createOperatorNode(mockOperator4), + createOperatorNode(mockOperator5), + createOperatorNode(mockOperator6), }}) output := unorderableToCycles(err) - expected := `(plugin1 -> plugin2 -> plugin3 -> plugin1),(plugin4 -> plugin5 -> plugin6 -> plugin4)` + expected := `(operator1 -> operator2 -> operator3 -> operator1),(operator4 -> operator5 -> operator6 -> operator4)` require.Equal(t, expected, output) }) @@ -67,7 +67,7 @@ func TestUnorderableToCycles(t *testing.T) { func TestPipeline(t *testing.T) { t.Run("MultipleStart", func(t *testing.T) { - pipeline, err := NewPipeline([]plugin.Plugin{}) + pipeline, err := NewPipeline([]operator.Operator{}) require.NoError(t, err) err = pipeline.Start() @@ -80,7 +80,7 @@ func TestPipeline(t *testing.T) { }) t.Run("MultipleStop", func(t *testing.T) { - pipeline, err := NewPipeline([]plugin.Plugin{}) + pipeline, err := NewPipeline([]operator.Operator{}) require.NoError(t, err) err = pipeline.Start() @@ -91,60 +91,60 @@ func TestPipeline(t *testing.T) { }) t.Run("DuplicateNodeIDs", func(t *testing.T) { - plugin1 := testutil.NewMockPlugin("plugin1") - plugin1.On("SetOutputs", mock.Anything).Return(nil) - plugin1.On("Outputs").Return(nil) - plugin2 := testutil.NewMockPlugin("plugin1") - plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return(nil) - - _, err := NewPipeline([]plugin.Plugin{plugin1, plugin2}) + operator1 := testutil.NewMockOperator("operator1") + operator1.On("SetOutputs", mock.Anything).Return(nil) + operator1.On("Outputs").Return(nil) + operator2 := testutil.NewMockOperator("operator1") + operator2.On("SetOutputs", mock.Anything).Return(nil) + operator2.On("Outputs").Return(nil) + + _, err := NewPipeline([]operator.Operator{operator1, operator2}) require.Error(t, err) require.Contains(t, err.Error(), "already exists") }) t.Run("OutputNotExist", func(t *testing.T) { - plugin1 := testutil.NewMockPlugin("plugin1") - plugin1.On("SetOutputs", mock.Anything).Return(nil) - plugin1.On("Outputs").Return() + operator1 := testutil.NewMockOperator("operator1") + operator1.On("SetOutputs", mock.Anything).Return(nil) + operator1.On("Outputs").Return() - plugin2 := testutil.NewMockPlugin("plugin2") - plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return([]plugin.Plugin{plugin1}) + operator2 := testutil.NewMockOperator("operator2") + operator2.On("SetOutputs", mock.Anything).Return(nil) + operator2.On("Outputs").Return([]operator.Operator{operator1}) - _, err := NewPipeline([]plugin.Plugin{plugin2}) + _, err := NewPipeline([]operator.Operator{operator2}) require.Error(t, err) require.Contains(t, err.Error(), "does not exist") }) t.Run("OutputNotProcessor", func(t *testing.T) { - plugin1 := &testutil.Plugin{} - plugin1.On("ID").Return("plugin1") - plugin1.On("CanProcess").Return(false) - plugin1.On("CanOutput").Return(true) - plugin1.On("SetOutputs", mock.Anything).Return(nil) - plugin1.On("Outputs").Return(nil) - - plugin2 := testutil.NewMockPlugin("plugin2") - plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return([]plugin.Plugin{plugin1}) - - _, err := NewPipeline([]plugin.Plugin{plugin1, plugin2}) + operator1 := &testutil.Operator{} + operator1.On("ID").Return("operator1") + operator1.On("CanProcess").Return(false) + operator1.On("CanOutput").Return(true) + operator1.On("SetOutputs", mock.Anything).Return(nil) + operator1.On("Outputs").Return(nil) + + operator2 := testutil.NewMockOperator("operator2") + operator2.On("SetOutputs", mock.Anything).Return(nil) + operator2.On("Outputs").Return([]operator.Operator{operator1}) + + _, err := NewPipeline([]operator.Operator{operator1, operator2}) require.Error(t, err) require.Contains(t, err.Error(), "can not process") }) t.Run("DuplicateEdges", func(t *testing.T) { - plugin1 := testutil.NewMockPlugin("plugin1") - plugin1.On("SetOutputs", mock.Anything).Return(nil) - plugin1.On("Outputs").Return(nil) + operator1 := testutil.NewMockOperator("operator1") + operator1.On("SetOutputs", mock.Anything).Return(nil) + operator1.On("Outputs").Return(nil) - plugin2 := testutil.NewMockPlugin("plugin2") - plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return([]plugin.Plugin{plugin1, plugin1}) + operator2 := testutil.NewMockOperator("operator2") + operator2.On("SetOutputs", mock.Anything).Return(nil) + operator2.On("Outputs").Return([]operator.Operator{operator1, operator1}) - node1 := createPluginNode(plugin1) - node2 := createPluginNode(plugin2) + node1 := createOperatorNode(operator1) + node2 := createOperatorNode(operator2) graph := simple.NewDirectedGraph() graph.AddNode(node1) @@ -158,17 +158,17 @@ func TestPipeline(t *testing.T) { }) t.Run("Cyclical", func(t *testing.T) { - mockPlugin1 := testutil.NewMockPlugin("plugin1") - mockPlugin2 := testutil.NewMockPlugin("plugin2") - mockPlugin3 := testutil.NewMockPlugin("plugin3") - mockPlugin1.On("Outputs").Return([]plugin.Plugin{mockPlugin2}) - mockPlugin1.On("SetOutputs", mock.Anything).Return(nil) - mockPlugin2.On("Outputs").Return([]plugin.Plugin{mockPlugin3}) - mockPlugin2.On("SetOutputs", mock.Anything).Return(nil) - mockPlugin3.On("Outputs").Return([]plugin.Plugin{mockPlugin1}) - mockPlugin3.On("SetOutputs", mock.Anything).Return(nil) - - _, err := NewPipeline([]plugin.Plugin{mockPlugin1, mockPlugin2, mockPlugin3}) + mockOperator1 := testutil.NewMockOperator("operator1") + mockOperator2 := testutil.NewMockOperator("operator2") + mockOperator3 := testutil.NewMockOperator("operator3") + mockOperator1.On("Outputs").Return([]operator.Operator{mockOperator2}) + mockOperator1.On("SetOutputs", mock.Anything).Return(nil) + mockOperator2.On("Outputs").Return([]operator.Operator{mockOperator3}) + mockOperator2.On("SetOutputs", mock.Anything).Return(nil) + mockOperator3.On("Outputs").Return([]operator.Operator{mockOperator1}) + mockOperator3.On("SetOutputs", mock.Anything).Return(nil) + + _, err := NewPipeline([]operator.Operator{mockOperator1, mockOperator2, mockOperator3}) require.Error(t, err) require.Contains(t, err.Error(), "circular dependency") }) diff --git a/plugin/buffer/buffer.go b/plugin/buffer/buffer.go deleted file mode 100644 index 11c731234..000000000 --- a/plugin/buffer/buffer.go +++ /dev/null @@ -1,76 +0,0 @@ -package buffer - -import ( - "context" - "fmt" - "time" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" -) - -// Buffer is an entity that buffers log entries to a plugin -type Buffer interface { - Flush(context.Context) error - Add(interface{}, int) error - AddWait(context.Context, interface{}, int) error - SetHandler(BundleHandler) - Process(context.Context, *entry.Entry) error -} - -func NewConfig() Config { - return Config{ - BufferType: "memory", - DelayThreshold: plugin.Duration{Duration: time.Second}, - BundleCountThreshold: 10_000, - BundleByteThreshold: 4 * 1024 * 1024 * 1024, // 4MB - BundleByteLimit: 4 * 1024 * 1024 * 1024, // 4MB - BufferedByteLimit: 500 * 1024 * 1024 * 1024, // 500MB - HandlerLimit: 32, - Retry: NewRetryConfig(), - } -} - -// Config is the configuration of a buffer -type Config struct { - BufferType string `json:"type,omitempty" yaml:"type,omitempty"` - DelayThreshold plugin.Duration `json:"delay_threshold,omitempty" yaml:"delay_threshold,omitempty"` - BundleCountThreshold int `json:"bundle_count_threshold,omitempty" yaml:"buffer_count_threshold,omitempty"` - BundleByteThreshold int `json:"bundle_byte_threshold,omitempty" yaml:"bundle_byte_threshold,omitempty"` - BundleByteLimit int `json:"bundle_byte_limit,omitempty" yaml:"bundle_byte_limit,omitempty"` - BufferedByteLimit int `json:"buffered_byte_limit,omitempty" yaml:"buffered_byte_limit,omitempty"` - HandlerLimit int `json:"handler_limit,omitempty" yaml:"handler_limit,omitempty"` - Retry RetryConfig `json:"retry,omitempty" yaml:"retry,omitempty"` -} - -// Build will build a buffer from the supplied configuration -func (config *Config) Build() (Buffer, error) { - switch config.BufferType { - case "memory", "": - return NewMemoryBuffer(config), nil - default: - return nil, errors.NewError( - fmt.Sprintf("Invalid buffer type %s", config.BufferType), - "The only supported buffer type is 'memory'", - ) - } -} - -func NewRetryConfig() RetryConfig { - return RetryConfig{ - InitialInterval: plugin.Duration{Duration: 500 * time.Millisecond}, - RandomizationFactor: 0.5, - Multiplier: 1.5, - MaxInterval: plugin.Duration{Duration: 15 * time.Minute}, - } -} - -// RetryConfig is the configuration of an entity that will retry processing after an error -type RetryConfig struct { - InitialInterval plugin.Duration `json:"initial_interval,omitempty" yaml:"initial_interval,omitempty"` - RandomizationFactor float64 `json:"randomization_factor,omitempty" yaml:"randomization_factor,omitempty"` - Multiplier float64 `json:"multiplier,omitempty" yaml:"multiplier,omitempty"` - MaxInterval plugin.Duration `json:"max_interval,omitempty" yaml:"max_interval,omitempty"` - MaxElapsedTime plugin.Duration `json:"max_elapsed_time,omitempty" yaml:"max_elapsed_time,omitempty"` -} diff --git a/plugin/builtin/builtin.go b/plugin/builtin/builtin.go deleted file mode 100644 index 694c54e92..000000000 --- a/plugin/builtin/builtin.go +++ /dev/null @@ -1,9 +0,0 @@ -package builtin - -import ( - // Load embedded packages when importing builtin plugins - _ "github.com/observiq/carbon/plugin/builtin/input" - _ "github.com/observiq/carbon/plugin/builtin/output" - _ "github.com/observiq/carbon/plugin/builtin/parser" - _ "github.com/observiq/carbon/plugin/builtin/transformer" -) diff --git a/plugin/builtin/input/input.go b/plugin/builtin/input/input.go deleted file mode 100644 index 6e7256552..000000000 --- a/plugin/builtin/input/input.go +++ /dev/null @@ -1,6 +0,0 @@ -package input - -import ( - // Load file package when importing input plugins - _ "github.com/observiq/carbon/plugin/builtin/input/file" -) diff --git a/plugin/builtin/output/drop.go b/plugin/builtin/output/drop.go deleted file mode 100644 index 0217a354a..000000000 --- a/plugin/builtin/output/drop.go +++ /dev/null @@ -1,48 +0,0 @@ -package output - -import ( - "context" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" -) - -func init() { - plugin.Register("drop_output", func() plugin.Builder { return NewDropOutputConfig("") }) -} - -func NewDropOutputConfig(pluginID string) *DropOutputConfig { - return &DropOutputConfig{ - OutputConfig: helper.NewOutputConfig(pluginID, "drop_output"), - } -} - -// DropOutputConfig is the configuration of a drop output plugin. -type DropOutputConfig struct { - helper.OutputConfig `yaml:",inline"` -} - -// Build will build a drop output plugin. -func (c DropOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - outputPlugin, err := c.OutputConfig.Build(context) - if err != nil { - return nil, err - } - - dropOutput := &DropOutput{ - OutputPlugin: outputPlugin, - } - - return dropOutput, nil -} - -// DropOutput is a plugin that consumes and ignores incoming entries. -type DropOutput struct { - helper.OutputPlugin -} - -// Process will drop the incoming entry. -func (p *DropOutput) Process(ctx context.Context, entry *entry.Entry) error { - return nil -} diff --git a/plugin/builtin/output/drop_test.go b/plugin/builtin/output/drop_test.go deleted file mode 100644 index 0cda288a4..000000000 --- a/plugin/builtin/output/drop_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package output - -import ( - "github.com/observiq/carbon/plugin/helper" -) - -func newFakeNullOutput() *DropOutput { - return &DropOutput{ - OutputPlugin: helper.OutputPlugin{ - BasicPlugin: helper.BasicPlugin{ - PluginID: "testnull", - PluginType: "drop_output", - }, - }, - } -} diff --git a/plugin/builtin/output/stdout.go b/plugin/builtin/output/stdout.go deleted file mode 100644 index 1157e191c..000000000 --- a/plugin/builtin/output/stdout.go +++ /dev/null @@ -1,64 +0,0 @@ -package output - -import ( - "context" - "encoding/json" - "io" - "os" - "sync" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" -) - -// Stdout is a global handle to standard output -var Stdout io.Writer = os.Stdout - -func init() { - plugin.Register("stdout", func() plugin.Builder { return NewStdoutConfig("") }) -} - -func NewStdoutConfig(pluginID string) *StdoutConfig { - return &StdoutConfig{ - OutputConfig: helper.NewOutputConfig(pluginID, "stdout"), - } -} - -// StdoutConfig is the configuration of the Stdout plugin -type StdoutConfig struct { - helper.OutputConfig `yaml:",inline"` -} - -// Build will build a stdout plugin. -func (c StdoutConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - outputPlugin, err := c.OutputConfig.Build(context) - if err != nil { - return nil, err - } - - return &StdoutPlugin{ - OutputPlugin: outputPlugin, - encoder: json.NewEncoder(Stdout), - }, nil -} - -// StdoutPlugin is a plugin that logs entries using stdout. -type StdoutPlugin struct { - helper.OutputPlugin - encoder *json.Encoder - mux sync.Mutex -} - -// Process will log entries received. -func (o *StdoutPlugin) Process(ctx context.Context, entry *entry.Entry) error { - o.mux.Lock() - err := o.encoder.Encode(entry) - if err != nil { - o.mux.Unlock() - o.Errorf("Failed to process entry: %s, $s", err, entry.Record) - return err - } - o.mux.Unlock() - return nil -} diff --git a/plugin/builtin/parser/severity.go b/plugin/builtin/parser/severity.go deleted file mode 100644 index 9e521ed35..000000000 --- a/plugin/builtin/parser/severity.go +++ /dev/null @@ -1,63 +0,0 @@ -package parser - -import ( - "context" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" -) - -func init() { - plugin.Register("severity_parser", func() plugin.Builder { return NewSeverityParserConfig("") }) -} - -func NewSeverityParserConfig(pluginID string) *SeverityParserConfig { - return &SeverityParserConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "severity_parser"), - SeverityParserConfig: helper.NewSeverityParserConfig(), - } -} - -// SeverityParserConfig is the configuration of a severity parser plugin. -type SeverityParserConfig struct { - helper.TransformerConfig `yaml:",inline"` - helper.SeverityParserConfig `yaml:",omitempty,inline"` -} - -// Build will build a time parser plugin. -func (c SeverityParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) - if err != nil { - return nil, err - } - - severityParser, err := c.SeverityParserConfig.Build(context) - if err != nil { - return nil, err - } - - severityPlugin := &SeverityParserPlugin{ - TransformerPlugin: transformerPlugin, - SeverityParser: severityParser, - } - - return severityPlugin, nil -} - -// SeverityParserPlugin is a plugin that parses time from a field to an entry. -type SeverityParserPlugin struct { - helper.TransformerPlugin - helper.SeverityParser -} - -// Process will parse time from an entry. -func (p *SeverityParserPlugin) Process(ctx context.Context, entry *entry.Entry) error { - if err := p.Parse(ctx, entry); err != nil { - return errors.Wrap(err, "parse severity") - } - - p.Write(ctx, entry) - return nil -} diff --git a/plugin/builtin/parser/time.go b/plugin/builtin/parser/time.go deleted file mode 100644 index 871e6f685..000000000 --- a/plugin/builtin/parser/time.go +++ /dev/null @@ -1,66 +0,0 @@ -package parser - -import ( - "context" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" -) - -func init() { - plugin.Register("time_parser", func() plugin.Builder { return NewTimeParserConfig("") }) -} - -func NewTimeParserConfig(pluginID string) *TimeParserConfig { - return &TimeParserConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "time_parser"), - TimeParser: helper.NewTimeParser(), - } -} - -// TimeParserConfig is the configuration of a time parser plugin. -type TimeParserConfig struct { - helper.TransformerConfig `yaml:",inline"` - helper.TimeParser `yaml:",omitempty,inline"` -} - -// Build will build a time parser plugin. -func (c TimeParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) - if err != nil { - return nil, err - } - - if err := c.TimeParser.Validate(context); err != nil { - return nil, err - } - - timeParser := &TimeParserPlugin{ - TransformerPlugin: transformerPlugin, - TimeParser: c.TimeParser, - } - - return timeParser, nil -} - -// TimeParserPlugin is a plugin that parses time from a field to an entry. -type TimeParserPlugin struct { - helper.TransformerPlugin - helper.TimeParser -} - -// CanOutput will always return true for a parser plugin. -func (t *TimeParserPlugin) CanOutput() bool { - return true -} - -// Process will parse time from an entry. -func (t *TimeParserPlugin) Process(ctx context.Context, entry *entry.Entry) error { - if err := t.Parse(ctx, entry); err != nil { - return errors.Wrap(err, "parse timestamp") - } - t.Write(ctx, entry) - return nil -} diff --git a/plugin/builtin/transformer/noop.go b/plugin/builtin/transformer/noop.go deleted file mode 100644 index 1172fa212..000000000 --- a/plugin/builtin/transformer/noop.go +++ /dev/null @@ -1,49 +0,0 @@ -package transformer - -import ( - "context" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" -) - -func init() { - plugin.Register("noop", func() plugin.Builder { return NewNoopPluginConfig("") }) -} - -func NewNoopPluginConfig(pluginID string) *NoopPluginConfig { - return &NoopPluginConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "noop"), - } -} - -// NoopPluginConfig is the configuration of a noop plugin. -type NoopPluginConfig struct { - helper.TransformerConfig `yaml:",inline"` -} - -// Build will build a noop plugin. -func (c NoopPluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) - if err != nil { - return nil, err - } - - noopPlugin := &NoopPlugin{ - TransformerPlugin: transformerPlugin, - } - - return noopPlugin, nil -} - -// NoopPlugin is a plugin that performs no operations on an entry. -type NoopPlugin struct { - helper.TransformerPlugin -} - -// Process will forward the entry to the next output without any alterations. -func (p *NoopPlugin) Process(ctx context.Context, entry *entry.Entry) error { - p.Write(ctx, entry) - return nil -} diff --git a/plugin/builtin/transformer/rate_limit.go b/plugin/builtin/transformer/rate_limit.go deleted file mode 100644 index f6c0e6dbe..000000000 --- a/plugin/builtin/transformer/rate_limit.go +++ /dev/null @@ -1,106 +0,0 @@ -package transformer - -import ( - "context" - "fmt" - "time" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" -) - -func init() { - plugin.Register("rate_limit", func() plugin.Builder { return NewRateLimitConfig("") }) -} - -func NewRateLimitConfig(pluginID string) *RateLimitConfig { - return &RateLimitConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "rate_limit"), - } -} - -// RateLimitConfig is the configuration of a rate filter plugin. -type RateLimitConfig struct { - helper.TransformerConfig `yaml:",inline"` - - Rate float64 `json:"rate,omitempty" yaml:"rate,omitempty"` - Interval plugin.Duration `json:"interval,omitempty" yaml:"interval,omitempty"` - Burst uint `json:"burst,omitempty" yaml:"burst,omitempty"` -} - -// Build will build a rate limit plugin. -func (c RateLimitConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) - if err != nil { - return nil, err - } - - var interval time.Duration - switch { - case c.Rate != 0 && c.Interval.Raw() != 0: - return nil, fmt.Errorf("only one of 'rate' or 'interval' can be defined") - case c.Rate < 0 || c.Interval.Raw() < 0: - return nil, fmt.Errorf("rate and interval must be greater than zero") - case c.Rate > 0: - interval = time.Second / time.Duration(c.Rate) - default: - interval = c.Interval.Raw() - } - - rateLimitPlugin := &RateLimitPlugin{ - TransformerPlugin: transformerPlugin, - interval: interval, - burst: c.Burst, - } - - return rateLimitPlugin, nil -} - -// RateLimitPlugin is a plugin that limits the rate of log consumption between plugins. -type RateLimitPlugin struct { - helper.TransformerPlugin - - interval time.Duration - burst uint - isReady chan struct{} - cancel context.CancelFunc -} - -// Process will wait until a rate is met before sending an entry to the output. -func (p *RateLimitPlugin) Process(ctx context.Context, entry *entry.Entry) error { - <-p.isReady - p.Write(ctx, entry) - return nil -} - -// Start will start the rate limit plugin. -func (p *RateLimitPlugin) Start() error { - p.isReady = make(chan struct{}, p.burst) - ticker := time.NewTicker(p.interval) - - ctx, cancel := context.WithCancel(context.Background()) - p.cancel = cancel - - // Buffer the ticker ticks in isReady to allow bursts - go func() { - defer ticker.Stop() - defer close(p.isReady) - for { - select { - case <-ticker.C: - p.isReady <- struct{}{} - case <-ctx.Done(): - return - } - } - }() - - return nil -} - -// Stop will stop the rate limit plugin. -func (p *RateLimitPlugin) Stop() error { - p.cancel() - return nil -} diff --git a/plugin/builtin/transformer/router.go b/plugin/builtin/transformer/router.go deleted file mode 100644 index 4f9060d6a..000000000 --- a/plugin/builtin/transformer/router.go +++ /dev/null @@ -1,166 +0,0 @@ -package transformer - -import ( - "context" - "fmt" - - "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" - "go.uber.org/zap" -) - -func init() { - plugin.Register("router", func() plugin.Builder { return NewRouterPluginConfig("") }) -} - -func NewRouterPluginConfig(pluginID string) *RouterPluginConfig { - return &RouterPluginConfig{ - BasicConfig: helper.NewBasicConfig(pluginID, "router"), - } -} - -// RouterPluginConfig is the configuration of a router plugin -type RouterPluginConfig struct { - helper.BasicConfig `yaml:",inline"` - Routes []*RouterPluginRouteConfig `json:"routes" yaml:"routes"` -} - -// RouterPluginRouteConfig is the configuration of a route on a router plugin -type RouterPluginRouteConfig struct { - Expression string `json:"expr" yaml:"expr"` - OutputIDs helper.OutputIDs `json:"output" yaml:"output"` -} - -// Build will build a router plugin from the supplied configuration -func (c RouterPluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - basicPlugin, err := c.BasicConfig.Build(context) - if err != nil { - return nil, err - } - - routes := make([]*RouterPluginRoute, 0, len(c.Routes)) - for _, routeConfig := range c.Routes { - compiled, err := expr.Compile(routeConfig.Expression, expr.AsBool(), expr.AllowUndefinedVariables()) - if err != nil { - return nil, fmt.Errorf("failed to compile expression '%s': %w", routeConfig.Expression, err) - } - route := RouterPluginRoute{ - Expression: compiled, - OutputIDs: routeConfig.OutputIDs, - } - routes = append(routes, &route) - } - - routerPlugin := &RouterPlugin{ - BasicPlugin: basicPlugin, - routes: routes, - } - - return routerPlugin, nil -} - -// SetNamespace will namespace the router plugin and the outputs contained in its routes -func (c *RouterPluginConfig) SetNamespace(namespace string, exclusions ...string) { - c.BasicConfig.SetNamespace(namespace, exclusions...) - for _, route := range c.Routes { - for i, outputID := range route.OutputIDs { - if helper.CanNamespace(outputID, exclusions) { - route.OutputIDs[i] = helper.AddNamespace(outputID, namespace) - } - } - } -} - -// RouterPlugin is a plugin that routes entries based on matching expressions -type RouterPlugin struct { - helper.BasicPlugin - routes []*RouterPluginRoute -} - -// RouterPluginRoute is a route on a router plugin -type RouterPluginRoute struct { - Expression *vm.Program - OutputIDs helper.OutputIDs - OutputPlugins []plugin.Plugin -} - -// CanProcess will always return true for a router plugin -func (p *RouterPlugin) CanProcess() bool { - return true -} - -// Process will route incoming entries based on matching expressions -func (p *RouterPlugin) Process(ctx context.Context, entry *entry.Entry) error { - env := helper.GetExprEnv(entry) - defer helper.PutExprEnv(env) - - for _, route := range p.routes { - matches, err := vm.Run(route.Expression, env) - if err != nil { - p.Warnw("Running expression returned an error", zap.Error(err)) - continue - } - - // we compile the expression with "AsBool", so this should be safe - if matches.(bool) { - for _, output := range route.OutputPlugins { - _ = output.Process(ctx, entry) - } - break - } - } - - return nil -} - -// CanOutput will always return true for a router plugin -func (p *RouterPlugin) CanOutput() bool { - return true -} - -// Outputs will return all connected plugins. -func (p *RouterPlugin) Outputs() []plugin.Plugin { - outputs := make([]plugin.Plugin, 0, len(p.routes)) - for _, route := range p.routes { - outputs = append(outputs, route.OutputPlugins...) - } - return outputs -} - -// SetOutputs will set the outputs of the router plugin. -func (p *RouterPlugin) SetOutputs(plugins []plugin.Plugin) error { - for _, route := range p.routes { - outputPlugins, err := p.findPlugins(plugins, route.OutputIDs) - if err != nil { - return fmt.Errorf("failed to set outputs on route: %s", err) - } - route.OutputPlugins = outputPlugins - } - return nil -} - -// findPlugins will find a subset of plugins from a collection. -func (p *RouterPlugin) findPlugins(plugins []plugin.Plugin, pluginIDs []string) ([]plugin.Plugin, error) { - result := make([]plugin.Plugin, 0) - for _, pluginID := range pluginIDs { - plugin, err := p.findPlugin(plugins, pluginID) - if err != nil { - return nil, err - } - result = append(result, plugin) - } - return result, nil -} - -// findPlugin will find a plugin from a collection. -func (p *RouterPlugin) findPlugin(plugins []plugin.Plugin, pluginID string) (plugin.Plugin, error) { - for _, plugin := range plugins { - if plugin.ID() == pluginID { - return plugin, nil - } - } - return nil, fmt.Errorf("plugin %s does not exist", pluginID) -} diff --git a/plugin/helper/input.go b/plugin/helper/input.go deleted file mode 100644 index 84fea7ce9..000000000 --- a/plugin/helper/input.go +++ /dev/null @@ -1,65 +0,0 @@ -package helper - -import ( - "context" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "go.uber.org/zap" -) - -func NewInputConfig(pluginID, pluginType string) InputConfig { - return InputConfig{ - WriterConfig: NewWriterConfig(pluginID, pluginType), - WriteTo: entry.NewRecordField(), - } -} - -// InputConfig provides a basic implementation of an input plugin config. -type InputConfig struct { - WriterConfig `yaml:",inline"` - WriteTo entry.Field `json:"write_to" yaml:"write_to"` -} - -// Build will build a base producer. -func (c InputConfig) Build(context plugin.BuildContext) (InputPlugin, error) { - writerPlugin, err := c.WriterConfig.Build(context) - if err != nil { - return InputPlugin{}, errors.WithDetails(err, "plugin_id", c.ID()) - } - - inputPlugin := InputPlugin{ - WriterPlugin: writerPlugin, - WriteTo: c.WriteTo, - } - - return inputPlugin, nil -} - -// InputPlugin provides a basic implementation of an input plugin. -type InputPlugin struct { - WriterPlugin - WriteTo entry.Field -} - -// NewEntry will create a new entry using the write_to field. -func (i *InputPlugin) NewEntry(value interface{}) *entry.Entry { - entry := entry.New() - entry.Set(i.WriteTo, value) - return entry -} - -// CanProcess will always return false for an input plugin. -func (i *InputPlugin) CanProcess() bool { - return false -} - -// Process will always return an error if called. -func (i *InputPlugin) Process(ctx context.Context, entry *entry.Entry) error { - i.Errorw("Plugin received an entry, but can not process", zap.Any("entry", entry)) - return errors.NewError( - "Plugin can not process logs.", - "Ensure that plugin is not configured to receive logs from other plugins", - ) -} diff --git a/plugin/helper/output.go b/plugin/helper/output.go deleted file mode 100644 index f1333a75d..000000000 --- a/plugin/helper/output.go +++ /dev/null @@ -1,66 +0,0 @@ -package helper - -import ( - "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" -) - -func NewOutputConfig(pluginID, pluginType string) OutputConfig { - return OutputConfig{ - BasicConfig: NewBasicConfig(pluginID, pluginType), - } -} - -// OutputConfig provides a basic implementation of an output plugin config. -type OutputConfig struct { - BasicConfig `mapstructure:",squash" yaml:",inline"` -} - -// Build will build an output plugin. -func (c OutputConfig) Build(context plugin.BuildContext) (OutputPlugin, error) { - basicPlugin, err := c.BasicConfig.Build(context) - if err != nil { - return OutputPlugin{}, err - } - - outputPlugin := OutputPlugin{ - BasicPlugin: basicPlugin, - } - - return outputPlugin, nil -} - -// SetNamespace will namespace the id and output of the plugin config. -func (c *OutputConfig) SetNamespace(namespace string, exclusions ...string) { - if CanNamespace(c.ID(), exclusions) { - c.PluginID = AddNamespace(c.ID(), namespace) - } -} - -// OutputPlugin provides a basic implementation of an output plugin. -type OutputPlugin struct { - BasicPlugin -} - -// CanProcess will always return true for an output plugin. -func (o *OutputPlugin) CanProcess() bool { - return true -} - -// CanOutput will always return false for an output plugin. -func (o *OutputPlugin) CanOutput() bool { - return false -} - -// Outputs will always return an empty array for an output plugin. -func (o *OutputPlugin) Outputs() []plugin.Plugin { - return []plugin.Plugin{} -} - -// SetOutputs will return an error if called. -func (o *OutputPlugin) SetOutputs(plugins []plugin.Plugin) error { - return errors.NewError( - "Plugin can not output, but is attempting to set an output.", - "This is an unexpected internal error. Please submit a bug/issue.", - ) -} diff --git a/plugin/helper/plugin.go b/plugin/helper/plugin.go deleted file mode 100644 index 2cbf3d622..000000000 --- a/plugin/helper/plugin.go +++ /dev/null @@ -1,103 +0,0 @@ -package helper - -import ( - "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "go.uber.org/zap" -) - -func NewBasicConfig(pluginID, pluginType string) BasicConfig { - return BasicConfig{ - PluginID: pluginID, - PluginType: pluginType, - } -} - -// BasicConfig provides a basic implemention for a plugin config. -type BasicConfig struct { - PluginID string `json:"id" yaml:"id"` - PluginType string `json:"type" yaml:"type"` -} - -// ID will return the plugin id. -func (c BasicConfig) ID() string { - if c.PluginID == "" { - return c.PluginType - } - return c.PluginID -} - -// Type will return the plugin type. -func (c BasicConfig) Type() string { - return c.PluginType -} - -// Build will build a basic plugin. -func (c BasicConfig) Build(context plugin.BuildContext) (BasicPlugin, error) { - if c.PluginType == "" { - return BasicPlugin{}, errors.NewError( - "missing required `type` field.", - "ensure that all plugins have a uniquely defined `type` field.", - "plugin_id", c.ID(), - ) - } - - if context.Logger == nil { - return BasicPlugin{}, errors.NewError( - "plugin build context is missing a logger.", - "this is an unexpected internal error", - "plugin_id", c.ID(), - "plugin_type", c.Type(), - ) - } - - plugin := BasicPlugin{ - PluginID: c.ID(), - PluginType: c.Type(), - SugaredLogger: context.Logger.With("plugin_id", c.ID(), "plugin_type", c.Type()), - } - - return plugin, nil -} - -// SetNamespace will namespace the plugin id. -func (c *BasicConfig) SetNamespace(namespace string, exclusions ...string) { - if CanNamespace(c.ID(), exclusions) { - c.PluginID = AddNamespace(c.ID(), namespace) - } -} - -// BasicPlugin provides a basic implementation of a plugin. -type BasicPlugin struct { - PluginID string - PluginType string - *zap.SugaredLogger -} - -// ID will return the plugin id. -func (p *BasicPlugin) ID() string { - if p.PluginID == "" { - return p.PluginType - } - return p.PluginID -} - -// Type will return the plugin type. -func (p *BasicPlugin) Type() string { - return p.PluginType -} - -// Logger returns the plugin's scoped logger. -func (p *BasicPlugin) Logger() *zap.SugaredLogger { - return p.SugaredLogger -} - -// Start will start the plugin. -func (p *BasicPlugin) Start() error { - return nil -} - -// Stop will stop the plugin. -func (p *BasicPlugin) Stop() error { - return nil -} diff --git a/plugin/helper/plugin_test.go b/plugin/helper/plugin_test.go deleted file mode 100644 index 95c21ab81..000000000 --- a/plugin/helper/plugin_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package helper - -import ( - "testing" - - "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestBasicConfigID(t *testing.T) { - config := BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", - } - require.Equal(t, "test-id", config.ID()) -} - -func TestBasicConfigType(t *testing.T) { - config := BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", - } - require.Equal(t, "test-type", config.Type()) -} - -func TestBasicConfigBuildWithoutID(t *testing.T) { - config := BasicConfig{ - PluginType: "test-type", - } - context := testutil.NewBuildContext(t) - _, err := config.Build(context) - require.NoError(t, err) -} - -func TestBasicConfigBuildWithoutType(t *testing.T) { - config := BasicConfig{ - PluginID: "test-id", - } - context := plugin.BuildContext{} - _, err := config.Build(context) - require.Error(t, err) - require.Contains(t, err.Error(), "missing required `type` field.") -} - -func TestBasicConfigBuildMissingLogger(t *testing.T) { - config := BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", - } - context := plugin.BuildContext{} - _, err := config.Build(context) - require.Error(t, err) - require.Contains(t, err.Error(), "plugin build context is missing a logger.") -} - -func TestBasicConfigBuildValid(t *testing.T) { - config := BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", - } - context := testutil.NewBuildContext(t) - plugin, err := config.Build(context) - require.NoError(t, err) - require.Equal(t, "test-id", plugin.PluginID) - require.Equal(t, "test-type", plugin.PluginType) -} - -func TestBasicPluginID(t *testing.T) { - plugin := BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", - } - require.Equal(t, "test-id", plugin.ID()) -} - -func TestBasicPluginType(t *testing.T) { - plugin := BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", - } - require.Equal(t, "test-type", plugin.Type()) -} - -func TestBasicPluginLogger(t *testing.T) { - logger := &zap.SugaredLogger{} - plugin := BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", - SugaredLogger: logger, - } - require.Equal(t, logger, plugin.Logger()) -} - -func TestBasicPluginStart(t *testing.T) { - plugin := BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", - } - err := plugin.Start() - require.NoError(t, err) -} - -func TestBasicPluginStop(t *testing.T) { - plugin := BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", - } - err := plugin.Stop() - require.NoError(t, err) -} diff --git a/plugin/helper/writer.go b/plugin/helper/writer.go deleted file mode 100644 index 8d8215150..000000000 --- a/plugin/helper/writer.go +++ /dev/null @@ -1,191 +0,0 @@ -package helper - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" -) - -func NewWriterConfig(pluginID, pluginType string) WriterConfig { - return WriterConfig{ - BasicConfig: NewBasicConfig(pluginID, pluginType), - } -} - -// WriterConfig is the configuration of a writer plugin. -type WriterConfig struct { - BasicConfig `yaml:",inline"` - OutputIDs OutputIDs `json:"output" yaml:"output"` -} - -// Build will build a writer plugin from the config. -func (c WriterConfig) Build(context plugin.BuildContext) (WriterPlugin, error) { - basicPlugin, err := c.BasicConfig.Build(context) - if err != nil { - return WriterPlugin{}, err - } - - writer := WriterPlugin{ - OutputIDs: c.OutputIDs, - BasicPlugin: basicPlugin, - } - return writer, nil -} - -// SetNamespace will namespace the output ids of the writer. -func (c *WriterConfig) SetNamespace(namespace string, exclusions ...string) { - c.BasicConfig.SetNamespace(namespace, exclusions...) - for i, outputID := range c.OutputIDs { - if CanNamespace(outputID, exclusions) { - c.OutputIDs[i] = AddNamespace(outputID, namespace) - } - } -} - -// WriterPlugin is a plugin that can write to other plugins. -type WriterPlugin struct { - BasicPlugin - OutputIDs OutputIDs - OutputPlugins []plugin.Plugin -} - -// Write will write an entry to the outputs of the plugin. -func (w *WriterPlugin) Write(ctx context.Context, e *entry.Entry) { - for i, plugin := range w.OutputPlugins { - if i == len(w.OutputPlugins)-1 { - _ = plugin.Process(ctx, e) - return - } - plugin.Process(ctx, e.Copy()) - } -} - -// CanOutput always returns true for a writer plugin. -func (w *WriterPlugin) CanOutput() bool { - return true -} - -// Outputs returns the outputs of the writer plugin. -func (w *WriterPlugin) Outputs() []plugin.Plugin { - return w.OutputPlugins -} - -// SetOutputs will set the outputs of the plugin. -func (w *WriterPlugin) SetOutputs(plugins []plugin.Plugin) error { - outputPlugins := make([]plugin.Plugin, 0) - - for _, pluginID := range w.OutputIDs { - plugin, ok := w.findPlugin(plugins, pluginID) - if !ok { - return fmt.Errorf("plugin '%s' does not exist", pluginID) - } - - if !plugin.CanProcess() { - return fmt.Errorf("plugin '%s' can not process entries", pluginID) - } - - outputPlugins = append(outputPlugins, plugin) - } - - // No outputs have been set, so use the next configured plugin - if len(w.OutputIDs) == 0 { - currentPluginIndex := -1 - for i, plugin := range plugins { - if plugin.ID() == w.ID() { - currentPluginIndex = i - break - } - } - if currentPluginIndex == -1 { - return fmt.Errorf("unexpectedly could not find self in array of plugins") - } - nextPluginIndex := currentPluginIndex + 1 - if nextPluginIndex == len(plugins) { - return fmt.Errorf("cannot omit output for the last plugin in the pipeline") - } - nextPlugin := plugins[nextPluginIndex] - if !nextPlugin.CanProcess() { - return fmt.Errorf("plugin '%s' cannot process entries, but it was selected as a receiver because 'output' was omitted", nextPlugin.ID()) - } - outputPlugins = append(outputPlugins, nextPlugin) - } - - w.OutputPlugins = outputPlugins - return nil -} - -// FindPlugin will find a plugin matching the supplied id. -func (w *WriterPlugin) findPlugin(plugins []plugin.Plugin, pluginID string) (plugin.Plugin, bool) { - for _, plugin := range plugins { - if plugin.ID() == pluginID { - return plugin, true - } - } - return nil, false -} - -// OutputIDs is a collection of plugin IDs used as outputs. -type OutputIDs []string - -// UnmarshalJSON will unmarshal a string or array of strings to OutputIDs. -func (o *OutputIDs) UnmarshalJSON(bytes []byte) error { - var value interface{} - err := json.Unmarshal(bytes, &value) - if err != nil { - return err - } - - ids, err := o.fromInterface(value) - if err != nil { - return err - } - - *o = ids - return nil -} - -// UnmarshalYAML will unmarshal a string or array of strings to OutputIDs. -func (o *OutputIDs) UnmarshalYAML(unmarshal func(interface{}) error) error { - var value interface{} - err := unmarshal(&value) - if err != nil { - return err - } - - ids, err := o.fromInterface(value) - if err != nil { - return err - } - - *o = ids - return nil -} - -// fromInterface will parse OutputIDs from a raw interface. -func (o *OutputIDs) fromInterface(value interface{}) (OutputIDs, error) { - if str, ok := value.(string); ok { - return OutputIDs{str}, nil - } - - if array, ok := value.([]interface{}); ok { - return o.fromArray(array) - } - - return nil, fmt.Errorf("value is not of type string or string array") -} - -// fromArray will parse OutputIDs from a raw array. -func (o *OutputIDs) fromArray(array []interface{}) (OutputIDs, error) { - ids := OutputIDs{} - for _, rawValue := range array { - strValue, ok := rawValue.(string) - if !ok { - return nil, fmt.Errorf("value in array is not of type string") - } - ids = append(ids, strValue) - } - return ids, nil -} diff --git a/plugin/plugin.go b/plugin/plugin.go deleted file mode 100644 index ead61d44a..000000000 --- a/plugin/plugin.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:generate mockery -name=^(Plugin)$ -output=../internal/testutil -outpkg=testutil -case=snake - -package plugin - -import ( - "context" - - "github.com/observiq/carbon/entry" - "go.uber.org/zap" -) - -// Plugin is a log monitoring component. -type Plugin interface { - // ID returns the id of the plugin. - ID() string - // Type returns the type of the plugin. - Type() string - - // Start will start the plugin. - Start() error - // Stop will stop the plugin. - Stop() error - - // CanOutput indicates if the plugin will output entries to other plugins. - CanOutput() bool - // Outputs returns the list of connected outputs. - Outputs() []Plugin - // SetOutputs will set the connected outputs. - SetOutputs([]Plugin) error - - // CanProcess indicates if the plugin will process entries from other plugins. - CanProcess() bool - // Process will process an entry from a plugin. - Process(context.Context, *entry.Entry) error - // Logger returns the plugin's logger - Logger() *zap.SugaredLogger -}