From 1c58ee8c3f86fa625f6b0c6ed5a8751feb73a5ab Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Wed, 22 Jul 2020 11:04:33 -0400 Subject: [PATCH 1/7] wip - renamed Plugin structs to Operator --- internal/testutil/mocks.go | 6 +- internal/testutil/plugin.go | 32 ++--- pipeline/config.go | 10 +- pipeline/config_test.go | 6 +- pipeline/node.go | 22 ++-- pipeline/pipeline.go | 44 +++---- pipeline/pipeline_test.go | 114 ++++++++--------- plugin/builtin/input/file/file.go | 32 ++--- plugin/builtin/input/file/file_test.go | 24 ++-- plugin/builtin/input/file/read_to_end.go | 20 +-- plugin/builtin/input/generate.go | 14 +- plugin/builtin/input/generate_test.go | 12 +- plugin/builtin/input/tcp.go | 10 +- plugin/builtin/input/tcp_test.go | 12 +- plugin/builtin/input/udp.go | 10 +- plugin/builtin/input/udp_test.go | 12 +- plugin/builtin/output/drop.go | 8 +- plugin/builtin/output/drop_test.go | 8 +- plugin/builtin/output/elastic.go | 16 +-- plugin/builtin/output/file.go | 12 +- plugin/builtin/output/google_cloud.go | 8 +- plugin/builtin/output/stdout.go | 18 +-- plugin/builtin/output/stdout_test.go | 8 +- plugin/builtin/parser/json.go | 12 +- plugin/builtin/parser/json_test.go | 26 ++-- plugin/builtin/parser/regex.go | 12 +- plugin/builtin/parser/regex_test.go | 22 ++-- plugin/builtin/parser/severity.go | 20 +-- plugin/builtin/parser/severity_test.go | 8 +- plugin/builtin/parser/syslog.go | 12 +- plugin/builtin/parser/syslog_test.go | 8 +- plugin/builtin/parser/time.go | 20 +-- plugin/builtin/parser/time_test.go | 8 +- .../transformer/k8s_metadata_decorator.go | 12 +- .../k8s_metadata_decorator_test.go | 14 +- plugin/builtin/transformer/metadata.go | 34 ++--- plugin/builtin/transformer/metadata_test.go | 26 ++-- plugin/builtin/transformer/noop.go | 28 ++-- plugin/builtin/transformer/noop_test.go | 4 +- plugin/builtin/transformer/rate_limit.go | 26 ++-- plugin/builtin/transformer/rate_limit_test.go | 4 +- plugin/builtin/transformer/restructure.go | 32 ++--- .../builtin/transformer/restructure_test.go | 28 ++-- plugin/builtin/transformer/router.go | 84 ++++++------ plugin/builtin/transformer/router_test.go | 32 ++--- plugin/config.go | 22 ++-- plugin/config_test.go | 26 ++-- plugin/helper/input.go | 30 ++--- plugin/helper/input_test.go | 52 ++++---- plugin/helper/output.go | 32 ++--- plugin/helper/output_test.go | 54 ++++---- plugin/helper/parser.go | 36 +++--- plugin/helper/parser_test.go | 120 +++++++++--------- plugin/helper/plugin.go | 58 ++++----- plugin/helper/plugin_test.go | 64 +++++----- plugin/helper/transformer.go | 28 ++-- plugin/helper/transformer_test.go | 52 ++++---- plugin/helper/writer.go | 66 +++++----- plugin/helper/writer_test.go | 50 ++++---- plugin/plugin.go | 26 ++-- 60 files changed, 823 insertions(+), 823 deletions(-) diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index d3b5b836a..1e7558c1e 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -1,8 +1,8 @@ package testutil -// NewMockPlugin will return a basic plugin mock -func NewMockPlugin(id string) *Plugin { - mockOutput := &Plugin{} +// NewMockOperator will return a basic plugin mock +func NewMockOperator(id string) *Operator { + mockOutput := &Operator{} mockOutput.On("ID").Return(id) mockOutput.On("CanProcess").Return(true) mockOutput.On("CanOutput").Return(true) diff --git a/internal/testutil/plugin.go b/internal/testutil/plugin.go index 2e6d6f9bf..69a32274f 100644 --- a/internal/testutil/plugin.go +++ b/internal/testutil/plugin.go @@ -13,13 +13,13 @@ import ( zap "go.uber.org/zap" ) -// Plugin is an autogenerated mock type for the Plugin type -type Plugin struct { +// Operator is an autogenerated mock type for the Operator type +type Operator struct { mock.Mock } // CanOutput provides a mock function with given fields: -func (_m *Plugin) CanOutput() bool { +func (_m *Operator) CanOutput() bool { ret := _m.Called() var r0 bool @@ -33,7 +33,7 @@ func (_m *Plugin) CanOutput() bool { } // CanProcess provides a mock function with given fields: -func (_m *Plugin) CanProcess() bool { +func (_m *Operator) CanProcess() bool { ret := _m.Called() var r0 bool @@ -47,7 +47,7 @@ func (_m *Plugin) CanProcess() bool { } // ID provides a mock function with given fields: -func (_m *Plugin) ID() string { +func (_m *Operator) ID() string { ret := _m.Called() var r0 string @@ -61,7 +61,7 @@ func (_m *Plugin) ID() string { } // Logger provides a mock function with given fields: -func (_m *Plugin) Logger() *zap.SugaredLogger { +func (_m *Operator) Logger() *zap.SugaredLogger { ret := _m.Called() var r0 *zap.SugaredLogger @@ -77,15 +77,15 @@ func (_m *Plugin) Logger() *zap.SugaredLogger { } // Outputs provides a mock function with given fields: -func (_m *Plugin) Outputs() []plugin.Plugin { +func (_m *Operator) Outputs() []plugin.Operator { ret := _m.Called() - var r0 []plugin.Plugin - if rf, ok := ret.Get(0).(func() []plugin.Plugin); ok { + var r0 []plugin.Operator + if rf, ok := ret.Get(0).(func() []plugin.Operator); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]plugin.Plugin) + r0 = ret.Get(0).([]plugin.Operator) } } @@ -93,7 +93,7 @@ func (_m *Plugin) Outputs() []plugin.Plugin { } // Process provides a mock function with given fields: _a0, _a1 -func (_m *Plugin) Process(_a0 context.Context, _a1 *entry.Entry) error { +func (_m *Operator) Process(_a0 context.Context, _a1 *entry.Entry) error { ret := _m.Called(_a0, _a1) var r0 error @@ -107,11 +107,11 @@ func (_m *Plugin) Process(_a0 context.Context, _a1 *entry.Entry) error { } // SetOutputs provides a mock function with given fields: _a0 -func (_m *Plugin) SetOutputs(_a0 []plugin.Plugin) error { +func (_m *Operator) SetOutputs(_a0 []plugin.Operator) error { ret := _m.Called(_a0) var r0 error - if rf, ok := ret.Get(0).(func([]plugin.Plugin) error); ok { + if rf, ok := ret.Get(0).(func([]plugin.Operator) error); ok { r0 = rf(_a0) } else { r0 = ret.Error(0) @@ -121,7 +121,7 @@ func (_m *Plugin) SetOutputs(_a0 []plugin.Plugin) error { } // Start provides a mock function with given fields: -func (_m *Plugin) Start() error { +func (_m *Operator) Start() error { ret := _m.Called() var r0 error @@ -135,7 +135,7 @@ func (_m *Plugin) Start() error { } // Stop provides a mock function with given fields: -func (_m *Plugin) Stop() error { +func (_m *Operator) Stop() error { ret := _m.Called() var r0 error @@ -149,7 +149,7 @@ func (_m *Plugin) Stop() error { } // Type provides a mock function with given fields: -func (_m *Plugin) Type() string { +func (_m *Operator) Type() string { ret := _m.Called() var r0 string diff --git a/pipeline/config.go b/pipeline/config.go index 65ee521e7..d94099a32 100644 --- a/pipeline/config.go +++ b/pipeline/config.go @@ -15,12 +15,12 @@ type Config []Params // BuildPipeline will build a pipeline from the config. func (c Config) BuildPipeline(context plugin.BuildContext) (*Pipeline, error) { - pluginConfigs, err := c.buildPluginConfigs(context.CustomRegistry) + pluginConfigs, err := c.buildOperatorConfigs(context.CustomRegistry) if err != nil { return nil, err } - plugins, err := c.buildPlugins(pluginConfigs, context) + plugins, err := c.buildOperators(pluginConfigs, context) if err != nil { return nil, err } @@ -33,8 +33,8 @@ func (c Config) BuildPipeline(context plugin.BuildContext) (*Pipeline, error) { return pipeline, nil } -func (c Config) buildPlugins(pluginConfigs []plugin.Config, context plugin.BuildContext) ([]plugin.Plugin, error) { - plugins := make([]plugin.Plugin, 0, len(pluginConfigs)) +func (c Config) buildOperators(pluginConfigs []plugin.Config, context plugin.BuildContext) ([]plugin.Operator, error) { + plugins := make([]plugin.Operator, 0, len(pluginConfigs)) for _, pluginConfig := range pluginConfigs { plugin, err := pluginConfig.Build(context) @@ -51,7 +51,7 @@ func (c Config) buildPlugins(pluginConfigs []plugin.Config, context plugin.Build return plugins, nil } -func (c Config) buildPluginConfigs(customRegistry plugin.CustomRegistry) ([]plugin.Config, error) { +func (c Config) buildOperatorConfigs(customRegistry plugin.CustomRegistry) ([]plugin.Config, error) { pluginConfigs := make([]plugin.Config, 0, len(c)) for _, params := range c { diff --git a/pipeline/config_test.go b/pipeline/config_test.go index a00a4bbe6..88aa6668a 100644 --- a/pipeline/config_test.go +++ b/pipeline/config_test.go @@ -214,7 +214,7 @@ func TestBuildBuiltinFromValidParams(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(configs)) - require.IsType(t, &transformer.NoopPluginConfig{}, configs[0].Builder) + require.IsType(t, &transformer.NoopOperatorConfig{}, configs[0].Builder) require.Equal(t, "test_namespace.noop", configs[0].ID()) } @@ -238,7 +238,7 @@ pipeline: configs, err := params.BuildConfigs(registry, "test_namespace") require.NoError(t, err) require.Equal(t, 1, len(configs)) - require.IsType(t, &transformer.NoopPluginConfig{}, configs[0].Builder) + require.IsType(t, &transformer.NoopOperatorConfig{}, configs[0].Builder) require.Equal(t, "test_namespace.custom_plugin.custom_noop", configs[0].ID()) } @@ -350,7 +350,7 @@ pipeline: require.Contains(t, err.Error(), "build plugin configs") } -func TestBuildInvalidPipelineInvalidPlugin(t *testing.T) { +func TestBuildInvalidPipelineInvalidOperator(t *testing.T) { registry := plugin.CustomRegistry{} logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() diff --git a/pipeline/node.go b/pipeline/node.go index 6b027f72b..f364bb437 100644 --- a/pipeline/node.go +++ b/pipeline/node.go @@ -6,35 +6,35 @@ import ( "github.com/observiq/carbon/plugin" ) -// PluginNode is a basic node that represents a plugin in a pipeline. -type PluginNode struct { - plugin plugin.Plugin +// OperatorNode is a basic node that represents a plugin in a pipeline. +type OperatorNode struct { + plugin plugin.Operator id int64 outputIDs map[string]int64 } -// Plugin returns the plugin of the node. -func (b PluginNode) Plugin() plugin.Plugin { +// Operator returns the plugin of the node. +func (b OperatorNode) Operator() plugin.Operator { return b.plugin } // ID returns the node id. -func (b PluginNode) ID() int64 { +func (b OperatorNode) ID() int64 { return b.id } // DOTID returns the id used to represent this node in a dot graph. -func (b PluginNode) DOTID() string { +func (b OperatorNode) DOTID() string { return b.plugin.ID() } // OutputIDs returns a map of output plugin ids to node ids. -func (b PluginNode) OutputIDs() map[string]int64 { +func (b OperatorNode) OutputIDs() map[string]int64 { return b.outputIDs } -// createPluginNode will create a plugin node. -func createPluginNode(plugin plugin.Plugin) PluginNode { +// createOperatorNode will create a plugin node. +func createOperatorNode(plugin plugin.Operator) OperatorNode { id := createNodeID(plugin.ID()) outputIDs := make(map[string]int64) if plugin.CanOutput() { @@ -42,7 +42,7 @@ func createPluginNode(plugin plugin.Plugin) PluginNode { outputIDs[output.ID()] = createNodeID(output.ID()) } } - return PluginNode{plugin, id, outputIDs} + return OperatorNode{plugin, id, outputIDs} } // createNodeID generates a node id from a plugin id. diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index bd8ec0339..b66a2d8c2 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -25,7 +25,7 @@ func (p *Pipeline) Start() error { sortedNodes, _ := topo.Sort(p.Graph) for i := len(sortedNodes) - 1; i >= 0; i-- { - plugin := sortedNodes[i].(PluginNode).Plugin() + plugin := sortedNodes[i].(OperatorNode).Operator() plugin.Logger().Debug("Starting plugin") if err := plugin.Start(); err != nil { return err @@ -45,7 +45,7 @@ func (p *Pipeline) Stop() { sortedNodes, _ := topo.Sort(p.Graph) for _, node := range sortedNodes { - plugin := node.(PluginNode).Plugin() + plugin := node.(OperatorNode).Operator() plugin.Logger().Debug("Stopping plugin") _ = plugin.Stop() plugin.Logger().Debug("Stopped plugin") @@ -60,12 +60,12 @@ func (p *Pipeline) MarshalDot() ([]byte, error) { } // addNodes will add plugins as nodes to the supplied graph. -func addNodes(graph *simple.DirectedGraph, plugins []plugin.Plugin) error { +func addNodes(graph *simple.DirectedGraph, plugins []plugin.Operator) error { for _, plugin := range plugins { - pluginNode := createPluginNode(plugin) + pluginNode := createOperatorNode(plugin) if graph.Node(pluginNode.ID()) != nil { return errors.NewError( - fmt.Sprintf("plugin with id '%s' already exists in pipeline", pluginNode.Plugin().ID()), + fmt.Sprintf("plugin with id '%s' already exists in pipeline", pluginNode.Operator().ID()), "ensure that each plugin has a unique `type` or `id`", ) } @@ -79,7 +79,7 @@ func addNodes(graph *simple.DirectedGraph, plugins []plugin.Plugin) error { func connectNodes(graph *simple.DirectedGraph) error { nodes := graph.Nodes() for nodes.Next() { - node := nodes.Node().(PluginNode) + node := nodes.Node().(OperatorNode) if err := connectNode(graph, node); err != nil { return err } @@ -97,24 +97,24 @@ func connectNodes(graph *simple.DirectedGraph) error { } // connectNode will connect a node to its outputs in the supplied graph. -func connectNode(graph *simple.DirectedGraph, inputNode PluginNode) error { - for outputPluginID, outputNodeID := range inputNode.OutputIDs() { +func connectNode(graph *simple.DirectedGraph, inputNode OperatorNode) error { + for outputOperatorID, outputNodeID := range inputNode.OutputIDs() { if graph.Node(outputNodeID) == nil { return errors.NewError( "plugins cannot be connected, because the output does not exist in the pipeline", "ensure that the output plugin is defined", - "input_plugin", inputNode.Plugin().ID(), - "output_plugin", outputPluginID, + "input_plugin", inputNode.Operator().ID(), + "output_plugin", outputOperatorID, ) } - outputNode := graph.Node(outputNodeID).(PluginNode) - if !outputNode.Plugin().CanProcess() { + outputNode := graph.Node(outputNodeID).(OperatorNode) + if !outputNode.Operator().CanProcess() { return errors.NewError( "plugins cannot be connected, because the output plugin can not process logs", "ensure that the output plugin can process logs (like a parser or destination)", - "input_plugin", inputNode.Plugin().ID(), - "output_plugin", outputPluginID, + "input_plugin", inputNode.Operator().ID(), + "output_plugin", outputOperatorID, ) } @@ -122,8 +122,8 @@ func connectNode(graph *simple.DirectedGraph, inputNode PluginNode) error { return errors.NewError( "plugins cannot be connected, because a connection already exists", "ensure that only a single connection exists between the two plugins", - "input_plugin", inputNode.Plugin().ID(), - "output_plugin", outputPluginID, + "input_plugin", inputNode.Operator().ID(), + "output_plugin", outputOperatorID, ) } @@ -134,8 +134,8 @@ func connectNode(graph *simple.DirectedGraph, inputNode PluginNode) error { return nil } -// setPluginOutputs will set the outputs on plugins that can output. -func setPluginOutputs(plugins []plugin.Plugin) error { +// setOperatorOutputs will set the outputs on plugins that can output. +func setOperatorOutputs(plugins []plugin.Operator) error { for _, plugin := range plugins { if !plugin.CanOutput() { continue @@ -149,8 +149,8 @@ func setPluginOutputs(plugins []plugin.Plugin) error { } // NewPipeline creates a new pipeline of connected plugins. -func NewPipeline(plugins []plugin.Plugin) (*Pipeline, error) { - if err := setPluginOutputs(plugins); err != nil { +func NewPipeline(plugins []plugin.Operator) (*Pipeline, error) { + if err := setOperatorOutputs(plugins); err != nil { return nil, err } @@ -174,10 +174,10 @@ func unorderableToCycles(err topo.Unorderable) string { } cycles.WriteByte('(') for _, node := range cycle { - cycles.WriteString(node.(PluginNode).plugin.ID()) + cycles.WriteString(node.(OperatorNode).plugin.ID()) cycles.Write([]byte(` -> `)) } - cycles.WriteString(cycle[0].(PluginNode).plugin.ID()) + cycles.WriteString(cycle[0].(OperatorNode).plugin.ID()) cycles.WriteByte(')') } return cycles.String() diff --git a/pipeline/pipeline_test.go b/pipeline/pipeline_test.go index ddb3a44a6..b8e34deeb 100644 --- a/pipeline/pipeline_test.go +++ b/pipeline/pipeline_test.go @@ -14,17 +14,17 @@ import ( func TestUnorderableToCycles(t *testing.T) { t.Run("SingleCycle", func(t *testing.T) { - mockPlugin1 := testutil.NewMockPlugin("plugin1") - mockPlugin2 := testutil.NewMockPlugin("plugin2") - mockPlugin3 := testutil.NewMockPlugin("plugin3") - mockPlugin1.On("Outputs").Return([]plugin.Plugin{mockPlugin2}) - mockPlugin2.On("Outputs").Return([]plugin.Plugin{mockPlugin3}) - mockPlugin3.On("Outputs").Return([]plugin.Plugin{mockPlugin1}) + mockOperator1 := testutil.NewMockOperator("plugin1") + mockOperator2 := testutil.NewMockOperator("plugin2") + mockOperator3 := testutil.NewMockOperator("plugin3") + mockOperator1.On("Outputs").Return([]plugin.Operator{mockOperator2}) + mockOperator2.On("Outputs").Return([]plugin.Operator{mockOperator3}) + mockOperator3.On("Outputs").Return([]plugin.Operator{mockOperator1}) err := topo.Unorderable([][]graph.Node{{ - createPluginNode(mockPlugin1), - createPluginNode(mockPlugin2), - createPluginNode(mockPlugin3), + createOperatorNode(mockOperator1), + createOperatorNode(mockOperator2), + createOperatorNode(mockOperator3), }}) output := unorderableToCycles(err) @@ -34,28 +34,28 @@ func TestUnorderableToCycles(t *testing.T) { }) t.Run("MultipleCycles", func(t *testing.T) { - mockPlugin1 := testutil.NewMockPlugin("plugin1") - mockPlugin2 := testutil.NewMockPlugin("plugin2") - mockPlugin3 := testutil.NewMockPlugin("plugin3") - mockPlugin1.On("Outputs").Return([]plugin.Plugin{mockPlugin2}) - mockPlugin2.On("Outputs").Return([]plugin.Plugin{mockPlugin3}) - mockPlugin3.On("Outputs").Return([]plugin.Plugin{mockPlugin1}) - - mockPlugin4 := testutil.NewMockPlugin("plugin4") - mockPlugin5 := testutil.NewMockPlugin("plugin5") - mockPlugin6 := testutil.NewMockPlugin("plugin6") - mockPlugin4.On("Outputs").Return([]plugin.Plugin{mockPlugin5}) - mockPlugin5.On("Outputs").Return([]plugin.Plugin{mockPlugin6}) - mockPlugin6.On("Outputs").Return([]plugin.Plugin{mockPlugin4}) + mockOperator1 := testutil.NewMockOperator("plugin1") + mockOperator2 := testutil.NewMockOperator("plugin2") + mockOperator3 := testutil.NewMockOperator("plugin3") + mockOperator1.On("Outputs").Return([]plugin.Operator{mockOperator2}) + mockOperator2.On("Outputs").Return([]plugin.Operator{mockOperator3}) + mockOperator3.On("Outputs").Return([]plugin.Operator{mockOperator1}) + + mockOperator4 := testutil.NewMockOperator("plugin4") + mockOperator5 := testutil.NewMockOperator("plugin5") + mockOperator6 := testutil.NewMockOperator("plugin6") + mockOperator4.On("Outputs").Return([]plugin.Operator{mockOperator5}) + mockOperator5.On("Outputs").Return([]plugin.Operator{mockOperator6}) + mockOperator6.On("Outputs").Return([]plugin.Operator{mockOperator4}) err := topo.Unorderable([][]graph.Node{{ - createPluginNode(mockPlugin1), - createPluginNode(mockPlugin2), - createPluginNode(mockPlugin3), + createOperatorNode(mockOperator1), + createOperatorNode(mockOperator2), + createOperatorNode(mockOperator3), }, { - createPluginNode(mockPlugin4), - createPluginNode(mockPlugin5), - createPluginNode(mockPlugin6), + createOperatorNode(mockOperator4), + createOperatorNode(mockOperator5), + createOperatorNode(mockOperator6), }}) output := unorderableToCycles(err) @@ -67,7 +67,7 @@ func TestUnorderableToCycles(t *testing.T) { func TestPipeline(t *testing.T) { t.Run("MultipleStart", func(t *testing.T) { - pipeline, err := NewPipeline([]plugin.Plugin{}) + pipeline, err := NewPipeline([]plugin.Operator{}) require.NoError(t, err) err = pipeline.Start() @@ -80,7 +80,7 @@ func TestPipeline(t *testing.T) { }) t.Run("MultipleStop", func(t *testing.T) { - pipeline, err := NewPipeline([]plugin.Plugin{}) + pipeline, err := NewPipeline([]plugin.Operator{}) require.NoError(t, err) err = pipeline.Start() @@ -91,60 +91,60 @@ func TestPipeline(t *testing.T) { }) t.Run("DuplicateNodeIDs", func(t *testing.T) { - plugin1 := testutil.NewMockPlugin("plugin1") + plugin1 := testutil.NewMockOperator("plugin1") plugin1.On("SetOutputs", mock.Anything).Return(nil) plugin1.On("Outputs").Return(nil) - plugin2 := testutil.NewMockPlugin("plugin1") + plugin2 := testutil.NewMockOperator("plugin1") plugin2.On("SetOutputs", mock.Anything).Return(nil) plugin2.On("Outputs").Return(nil) - _, err := NewPipeline([]plugin.Plugin{plugin1, plugin2}) + _, err := NewPipeline([]plugin.Operator{plugin1, plugin2}) require.Error(t, err) require.Contains(t, err.Error(), "already exists") }) t.Run("OutputNotExist", func(t *testing.T) { - plugin1 := testutil.NewMockPlugin("plugin1") + plugin1 := testutil.NewMockOperator("plugin1") plugin1.On("SetOutputs", mock.Anything).Return(nil) plugin1.On("Outputs").Return() - plugin2 := testutil.NewMockPlugin("plugin2") + plugin2 := testutil.NewMockOperator("plugin2") plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return([]plugin.Plugin{plugin1}) + plugin2.On("Outputs").Return([]plugin.Operator{plugin1}) - _, err := NewPipeline([]plugin.Plugin{plugin2}) + _, err := NewPipeline([]plugin.Operator{plugin2}) require.Error(t, err) require.Contains(t, err.Error(), "does not exist") }) t.Run("OutputNotProcessor", func(t *testing.T) { - plugin1 := &testutil.Plugin{} + plugin1 := &testutil.Operator{} plugin1.On("ID").Return("plugin1") plugin1.On("CanProcess").Return(false) plugin1.On("CanOutput").Return(true) plugin1.On("SetOutputs", mock.Anything).Return(nil) plugin1.On("Outputs").Return(nil) - plugin2 := testutil.NewMockPlugin("plugin2") + plugin2 := testutil.NewMockOperator("plugin2") plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return([]plugin.Plugin{plugin1}) + plugin2.On("Outputs").Return([]plugin.Operator{plugin1}) - _, err := NewPipeline([]plugin.Plugin{plugin1, plugin2}) + _, err := NewPipeline([]plugin.Operator{plugin1, plugin2}) require.Error(t, err) require.Contains(t, err.Error(), "can not process") }) t.Run("DuplicateEdges", func(t *testing.T) { - plugin1 := testutil.NewMockPlugin("plugin1") + plugin1 := testutil.NewMockOperator("plugin1") plugin1.On("SetOutputs", mock.Anything).Return(nil) plugin1.On("Outputs").Return(nil) - plugin2 := testutil.NewMockPlugin("plugin2") + plugin2 := testutil.NewMockOperator("plugin2") plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return([]plugin.Plugin{plugin1, plugin1}) + plugin2.On("Outputs").Return([]plugin.Operator{plugin1, plugin1}) - node1 := createPluginNode(plugin1) - node2 := createPluginNode(plugin2) + node1 := createOperatorNode(plugin1) + node2 := createOperatorNode(plugin2) graph := simple.NewDirectedGraph() graph.AddNode(node1) @@ -158,17 +158,17 @@ func TestPipeline(t *testing.T) { }) t.Run("Cyclical", func(t *testing.T) { - mockPlugin1 := testutil.NewMockPlugin("plugin1") - mockPlugin2 := testutil.NewMockPlugin("plugin2") - mockPlugin3 := testutil.NewMockPlugin("plugin3") - mockPlugin1.On("Outputs").Return([]plugin.Plugin{mockPlugin2}) - mockPlugin1.On("SetOutputs", mock.Anything).Return(nil) - mockPlugin2.On("Outputs").Return([]plugin.Plugin{mockPlugin3}) - mockPlugin2.On("SetOutputs", mock.Anything).Return(nil) - mockPlugin3.On("Outputs").Return([]plugin.Plugin{mockPlugin1}) - mockPlugin3.On("SetOutputs", mock.Anything).Return(nil) - - _, err := NewPipeline([]plugin.Plugin{mockPlugin1, mockPlugin2, mockPlugin3}) + mockOperator1 := testutil.NewMockOperator("plugin1") + mockOperator2 := testutil.NewMockOperator("plugin2") + mockOperator3 := testutil.NewMockOperator("plugin3") + mockOperator1.On("Outputs").Return([]plugin.Operator{mockOperator2}) + mockOperator1.On("SetOutputs", mock.Anything).Return(nil) + mockOperator2.On("Outputs").Return([]plugin.Operator{mockOperator3}) + mockOperator2.On("SetOutputs", mock.Anything).Return(nil) + mockOperator3.On("Outputs").Return([]plugin.Operator{mockOperator1}) + mockOperator3.On("SetOutputs", mock.Anything).Return(nil) + + _, err := NewPipeline([]plugin.Operator{mockOperator1, mockOperator2, mockOperator3}) require.Error(t, err) require.Contains(t, err.Error(), "circular dependency") }) diff --git a/plugin/builtin/input/file/file.go b/plugin/builtin/input/file/file.go index c3c2bfa2b..d780ca079 100644 --- a/plugin/builtin/input/file/file.go +++ b/plugin/builtin/input/file/file.go @@ -63,8 +63,8 @@ type MultilineConfig struct { } // Build will build a file input plugin from the supplied configuration -func (c InputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - inputPlugin, err := c.InputConfig.Build(context) +func (c InputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err } @@ -109,8 +109,8 @@ func (c InputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { return nil, fmt.Errorf("invalid start_at location '%s'", c.StartAt) } - plugin := &InputPlugin{ - InputPlugin: inputPlugin, + plugin := &InputOperator{ + InputOperator: inputOperator, Include: c.Include, Exclude: c.Exclude, SplitFunc: splitFunc, @@ -186,9 +186,9 @@ func (c InputConfig) getSplitFunc(encoding encoding.Encoding) (bufio.SplitFunc, return splitFunc, nil } -// InputPlugin is a plugin that monitors files for entries -type InputPlugin struct { - helper.InputPlugin +// InputOperator is a plugin that monitors files for entries +type InputOperator struct { + helper.InputOperator Include []string Exclude []string @@ -215,7 +215,7 @@ type InputPlugin struct { } // Start will start the file monitoring process -func (f *InputPlugin) Start() error { +func (f *InputOperator) Start() error { ctx, cancel := context.WithCancel(context.Background()) f.cancel = cancel f.wg = &sync.WaitGroup{} @@ -266,7 +266,7 @@ func (f *InputPlugin) Start() error { } // Stop will stop the file monitoring process -func (f *InputPlugin) Stop() error { +func (f *InputOperator) Stop() error { f.cancel() f.wg.Wait() f.syncKnownFiles() @@ -279,7 +279,7 @@ func (f *InputPlugin) Stop() error { // firstCheck indicates whether this is the first time checkFile has been called // after startup. This is important for the start_at parameter because, after initial // startup, we don't want to start at the end of newly-created files. -func (f *InputPlugin) checkFile(ctx context.Context, path string, firstCheck bool) { +func (f *InputOperator) checkFile(ctx context.Context, path string, firstCheck bool) { // Check if the file is currently being read if _, ok := f.runningFiles[path]; ok { @@ -314,14 +314,14 @@ func (f *InputPlugin) checkFile(ctx context.Context, path string, firstCheck boo go func(ctx context.Context, path string, offset, lastSeenSize int64) { defer f.readerWg.Done() messenger := f.newFileUpdateMessenger(path) - err := ReadToEnd(ctx, path, offset, lastSeenSize, messenger, f.SplitFunc, f.FilePathField, f.FileNameField, f.InputPlugin, f.MaxLogSize, f.encoding) + err := ReadToEnd(ctx, path, offset, lastSeenSize, messenger, f.SplitFunc, f.FilePathField, f.FileNameField, f.InputOperator, f.MaxLogSize, f.encoding) if err != nil { f.Warnw("Failed to read log file", zap.Error(err)) } }(ctx, path, knownFile.Offset, knownFile.LastSeenFileSize) } -func (f *InputPlugin) updateFile(message fileUpdateMessage) { +func (f *InputOperator) updateFile(message fileUpdateMessage) { if message.finished { delete(f.runningFiles, message.path) return @@ -385,7 +385,7 @@ func (f *InputPlugin) updateFile(message fileUpdateMessage) { knownFile.Offset = message.newOffset } -func (f *InputPlugin) drainMessages() { +func (f *InputOperator) drainMessages() { done := make(chan struct{}) go func() { f.readerWg.Wait() @@ -404,7 +404,7 @@ func (f *InputPlugin) drainMessages() { var knownFilesKey = "knownFiles" -func (f *InputPlugin) syncKnownFiles() { +func (f *InputOperator) syncKnownFiles() { var buf bytes.Buffer enc := gob.NewEncoder(&buf) err := enc.Encode(f.knownFiles) @@ -417,7 +417,7 @@ func (f *InputPlugin) syncKnownFiles() { f.persist.Sync() } -func (f *InputPlugin) readKnownFiles() (map[string]*knownFileInfo, error) { +func (f *InputOperator) readKnownFiles() (map[string]*knownFileInfo, error) { err := f.persist.Load() if err != nil { return nil, err @@ -439,7 +439,7 @@ func (f *InputPlugin) readKnownFiles() (map[string]*knownFileInfo, error) { return knownFiles, nil } -func (f *InputPlugin) newFileUpdateMessenger(path string) fileUpdateMessenger { +func (f *InputOperator) newFileUpdateMessenger(path string) fileUpdateMessenger { return fileUpdateMessenger{ path: path, c: f.fileUpdateChan, diff --git a/plugin/builtin/input/file/file_test.go b/plugin/builtin/input/file/file_test.go index 32ff023e0..f38d9a149 100644 --- a/plugin/builtin/input/file/file_test.go +++ b/plugin/builtin/input/file/file_test.go @@ -19,8 +19,8 @@ import ( "github.com/stretchr/testify/require" ) -func newTestFileSource(t *testing.T) (*InputPlugin, chan *entry.Entry) { - mockOutput := testutil.NewMockPlugin("output") +func newTestFileSource(t *testing.T) (*InputOperator, chan *entry.Entry) { + mockOutput := testutil.NewMockOperator("output") receivedEntries := make(chan *entry.Entry, 1000) mockOutput.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { receivedEntries <- args.Get(1).(*entry.Entry) @@ -35,15 +35,15 @@ func newTestFileSource(t *testing.T) (*InputPlugin, chan *entry.Entry) { if err != nil { t.Fatalf("Error building plugin: %s", err) } - source := pg.(*InputPlugin) - source.OutputPlugins = []plugin.Plugin{mockOutput} + source := pg.(*InputOperator) + source.OutputOperators = []plugin.Operator{mockOutput} return source, receivedEntries } func TestFileSource_Build(t *testing.T) { t.Parallel() - mockOutput := testutil.NewMockPlugin("mock") + mockOutput := testutil.NewMockOperator("mock") basicConfig := func() *InputConfig { cfg := NewInputConfig("testfile") @@ -59,14 +59,14 @@ func TestFileSource_Build(t *testing.T) { name string modifyBaseConfig func(*InputConfig) errorRequirement require.ErrorAssertionFunc - validate func(*testing.T, *InputPlugin) + validate func(*testing.T, *InputOperator) }{ { "Basic", func(f *InputConfig) { return }, require.NoError, - func(t *testing.T, f *InputPlugin) { - require.Equal(t, f.OutputPlugins[0], mockOutput) + func(t *testing.T, f *InputOperator) { + require.Equal(t, f.OutputOperators[0], mockOutput) require.Equal(t, f.Include, []string{"/var/log/testpath.*"}) require.Equal(t, f.FilePathField, entry.NewRecordField("testpath")) require.Equal(t, f.PollInterval, 10*time.Millisecond) @@ -107,7 +107,7 @@ func TestFileSource_Build(t *testing.T) { } }, require.NoError, - func(t *testing.T, f *InputPlugin) {}, + func(t *testing.T, f *InputOperator) {}, }, { "MultilineConfiguredEndPattern", @@ -117,7 +117,7 @@ func TestFileSource_Build(t *testing.T) { } }, require.NoError, - func(t *testing.T, f *InputPlugin) {}, + func(t *testing.T, f *InputOperator) {}, }, } @@ -132,10 +132,10 @@ func TestFileSource_Build(t *testing.T) { return } - err = plg.SetOutputs([]plugin.Plugin{mockOutput}) + err = plg.SetOutputs([]plugin.Operator{mockOutput}) require.NoError(t, err) - fileInput := plg.(*InputPlugin) + fileInput := plg.(*InputOperator) tc.validate(t, fileInput) }) } diff --git a/plugin/builtin/input/file/read_to_end.go b/plugin/builtin/input/file/read_to_end.go index 8e5a67217..7fac69e61 100644 --- a/plugin/builtin/input/file/read_to_end.go +++ b/plugin/builtin/input/file/read_to_end.go @@ -24,7 +24,7 @@ func ReadToEnd( splitFunc bufio.SplitFunc, filePathField entry.Field, fileNameField entry.Field, - inputPlugin helper.InputPlugin, + inputOperator helper.InputOperator, maxLogSize int, encoding encoding.Encoding, ) error { @@ -79,7 +79,7 @@ func ReadToEnd( // advanced since last cycle, read the rest of the file as an entry defer func() { if pos < stat.Size() && pos == startOffset && lastSeenFileSize == stat.Size() { - readRemaining(ctx, file, pos, stat.Size(), messenger, inputPlugin, filePathField, fileNameField, decoder, decodeBuffer) + readRemaining(ctx, file, pos, stat.Size(), messenger, inputOperator, filePathField, fileNameField, decoder, decodeBuffer) } }() @@ -104,37 +104,37 @@ func ReadToEnd( return err } - e := inputPlugin.NewEntry(string(decodeBuffer[:nDst])) + e := inputOperator.NewEntry(string(decodeBuffer[:nDst])) e.Set(filePathField, path) e.Set(fileNameField, filepath.Base(file.Name())) - inputPlugin.Write(ctx, e) + inputOperator.Write(ctx, e) messenger.SetOffset(pos) } } // readRemaining will read the remaining characters in a file as a log entry. -func readRemaining(ctx context.Context, file *os.File, filePos int64, fileSize int64, messenger fileUpdateMessenger, inputPlugin helper.InputPlugin, filePathField, fileNameField entry.Field, encoder *encoding.Decoder, decodeBuffer []byte) { +func readRemaining(ctx context.Context, file *os.File, filePos int64, fileSize int64, messenger fileUpdateMessenger, inputOperator helper.InputOperator, filePathField, fileNameField entry.Field, encoder *encoding.Decoder, decodeBuffer []byte) { _, err := file.Seek(filePos, 0) if err != nil { - inputPlugin.Errorf("failed to seek to read last log entry") + inputOperator.Errorf("failed to seek to read last log entry") return } msgBuf := make([]byte, fileSize-filePos) n, err := file.Read(msgBuf) if err != nil { - inputPlugin.Errorf("failed to read trailing log") + inputOperator.Errorf("failed to read trailing log") return } encoder.Reset() nDst, _, err := encoder.Transform(decodeBuffer, msgBuf, true) if err != nil { - inputPlugin.Errorw("failed to decode trailing log", zap.Error(err)) + inputOperator.Errorw("failed to decode trailing log", zap.Error(err)) } - e := inputPlugin.NewEntry(string(decodeBuffer[:nDst])) + e := inputOperator.NewEntry(string(decodeBuffer[:nDst])) e.Set(filePathField, file.Name()) e.Set(fileNameField, filepath.Base(file.Name())) - inputPlugin.Write(ctx, e) + inputOperator.Write(ctx, e) messenger.SetOffset(filePos + int64(n)) } diff --git a/plugin/builtin/input/generate.go b/plugin/builtin/input/generate.go index ebaa0ca72..98434e7e5 100644 --- a/plugin/builtin/input/generate.go +++ b/plugin/builtin/input/generate.go @@ -30,8 +30,8 @@ type GenerateInputConfig struct { } // Build will build a generate input plugin. -func (c *GenerateInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - inputPlugin, err := c.InputConfig.Build(context) +func (c *GenerateInputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err } @@ -39,17 +39,17 @@ func (c *GenerateInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, c.Entry.Record = recursiveMapInterfaceToMapString(c.Entry.Record) generateInput := &GenerateInput{ - InputPlugin: inputPlugin, - entry: c.Entry, - count: c.Count, - static: c.Static, + InputOperator: inputOperator, + entry: c.Entry, + count: c.Count, + static: c.Static, } return generateInput, nil } // GenerateInput is a plugin that generates log entries. type GenerateInput struct { - helper.InputPlugin + helper.InputOperator entry entry.Entry count int static bool diff --git a/plugin/builtin/input/generate_test.go b/plugin/builtin/input/generate_test.go index e7d78ac10..6125fa6a1 100644 --- a/plugin/builtin/input/generate_test.go +++ b/plugin/builtin/input/generate_test.go @@ -26,17 +26,17 @@ func TestInputGenerate(t *testing.T) { } buildContext := testutil.NewBuildContext(t) - newPlugin, err := basicConfig().Build(buildContext) + newOperator, err := basicConfig().Build(buildContext) require.NoError(t, err) receivedEntries := make(chan *entry.Entry) - mockOutput := testutil.NewMockPlugin("output1") + mockOutput := testutil.NewMockOperator("output1") mockOutput.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { receivedEntries <- args.Get(1).(*entry.Entry) }) - generateInput := newPlugin.(*GenerateInput) - err = generateInput.SetOutputs([]plugin.Plugin{mockOutput}) + generateInput := newOperator.(*GenerateInput) + err = generateInput.SetOutputs([]plugin.Operator{mockOutput}) require.NoError(t, err) err = generateInput.Start() @@ -84,8 +84,8 @@ pipeline: WriteTo: entry.NewRecordField(), WriterConfig: helper.WriterConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "my_generator", - PluginType: "generate_input", + OperatorID: "my_generator", + OperatorType: "generate_input", }, OutputIDs: []string{"sampleoutput"}, }, diff --git a/plugin/builtin/input/tcp.go b/plugin/builtin/input/tcp.go index 6a48fbd04..32252bac9 100644 --- a/plugin/builtin/input/tcp.go +++ b/plugin/builtin/input/tcp.go @@ -30,8 +30,8 @@ type TCPInputConfig struct { } // Build will build a tcp input plugin. -func (c TCPInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - inputPlugin, err := c.InputConfig.Build(context) +func (c TCPInputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err } @@ -46,15 +46,15 @@ func (c TCPInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error } tcpInput := &TCPInput{ - InputPlugin: inputPlugin, - address: address, + InputOperator: inputOperator, + address: address, } return tcpInput, nil } // TCPInput is a plugin that listens for log entries over tcp. type TCPInput struct { - helper.InputPlugin + helper.InputOperator address *net.TCPAddr listener *net.TCPListener diff --git a/plugin/builtin/input/tcp_test.go b/plugin/builtin/input/tcp_test.go index 8fd43cc76..a62303828 100644 --- a/plugin/builtin/input/tcp_test.go +++ b/plugin/builtin/input/tcp_test.go @@ -20,8 +20,8 @@ func TestTCPInput(t *testing.T) { WriteTo: entry.NewRecordField(), WriterConfig: helper.WriterConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "test_id", - PluginType: "tcp_input", + OperatorID: "test_id", + OperatorType: "tcp_input", }, OutputIDs: []string{"test_output_id"}, }, @@ -34,12 +34,12 @@ func TestTCPInput(t *testing.T) { cfg.ListenAddress = "127.0.0.1:64001" buildContext := testutil.NewBuildContext(t) - newPlugin, err := cfg.Build(buildContext) + newOperator, err := cfg.Build(buildContext) require.NoError(t, err) - mockOutput := testutil.Plugin{} - tcpInput := newPlugin.(*TCPInput) - tcpInput.InputPlugin.OutputPlugins = []plugin.Plugin{&mockOutput} + mockOutput := testutil.Operator{} + tcpInput := newOperator.(*TCPInput) + tcpInput.InputOperator.OutputOperators = []plugin.Operator{&mockOutput} entryChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { diff --git a/plugin/builtin/input/udp.go b/plugin/builtin/input/udp.go index b49034d11..299dac55f 100644 --- a/plugin/builtin/input/udp.go +++ b/plugin/builtin/input/udp.go @@ -29,8 +29,8 @@ type UDPInputConfig struct { } // Build will build a udp input plugin. -func (c UDPInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - inputPlugin, err := c.InputConfig.Build(context) +func (c UDPInputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err } @@ -45,15 +45,15 @@ func (c UDPInputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error } udpInput := &UDPInput{ - InputPlugin: inputPlugin, - address: address, + InputOperator: inputOperator, + address: address, } return udpInput, nil } // UDPInput is a plugin that listens to a socket for log entries. type UDPInput struct { - helper.InputPlugin + helper.InputOperator address *net.UDPAddr connection net.PacketConn diff --git a/plugin/builtin/input/udp_test.go b/plugin/builtin/input/udp_test.go index 0e160e1bf..3f87fdef1 100644 --- a/plugin/builtin/input/udp_test.go +++ b/plugin/builtin/input/udp_test.go @@ -20,8 +20,8 @@ func TestUDPInput(t *testing.T) { WriteTo: entry.NewRecordField(), WriterConfig: helper.WriterConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "test_id", - PluginType: "udp_input", + OperatorID: "test_id", + OperatorType: "udp_input", }, OutputIDs: []string{"test_output_id"}, }, @@ -34,14 +34,14 @@ func TestUDPInput(t *testing.T) { cfg.ListenAddress = "127.0.0.1:63001" buildContext := testutil.NewBuildContext(t) - newPlugin, err := cfg.Build(buildContext) + newOperator, err := cfg.Build(buildContext) require.NoError(t, err) - mockOutput := testutil.Plugin{} - udpInput, ok := newPlugin.(*UDPInput) + mockOutput := testutil.Operator{} + udpInput, ok := newOperator.(*UDPInput) require.True(t, ok) - udpInput.InputPlugin.OutputPlugins = []plugin.Plugin{&mockOutput} + udpInput.InputOperator.OutputOperators = []plugin.Operator{&mockOutput} entryChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { diff --git a/plugin/builtin/output/drop.go b/plugin/builtin/output/drop.go index 0217a354a..2f45c5d8d 100644 --- a/plugin/builtin/output/drop.go +++ b/plugin/builtin/output/drop.go @@ -24,14 +24,14 @@ type DropOutputConfig struct { } // Build will build a drop output plugin. -func (c DropOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - outputPlugin, err := c.OutputConfig.Build(context) +func (c DropOutputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + outputOperator, err := c.OutputConfig.Build(context) if err != nil { return nil, err } dropOutput := &DropOutput{ - OutputPlugin: outputPlugin, + OutputOperator: outputOperator, } return dropOutput, nil @@ -39,7 +39,7 @@ func (c DropOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, err // DropOutput is a plugin that consumes and ignores incoming entries. type DropOutput struct { - helper.OutputPlugin + helper.OutputOperator } // Process will drop the incoming entry. diff --git a/plugin/builtin/output/drop_test.go b/plugin/builtin/output/drop_test.go index 0cda288a4..a20b77271 100644 --- a/plugin/builtin/output/drop_test.go +++ b/plugin/builtin/output/drop_test.go @@ -6,10 +6,10 @@ import ( func newFakeNullOutput() *DropOutput { return &DropOutput{ - OutputPlugin: helper.OutputPlugin{ - BasicPlugin: helper.BasicPlugin{ - PluginID: "testnull", - PluginType: "drop_output", + OutputOperator: helper.OutputOperator{ + BasicOperator: helper.BasicOperator{ + OperatorID: "testnull", + OperatorType: "drop_output", }, }, } diff --git a/plugin/builtin/output/elastic.go b/plugin/builtin/output/elastic.go index 6115c32fd..cbaf58cb0 100644 --- a/plugin/builtin/output/elastic.go +++ b/plugin/builtin/output/elastic.go @@ -43,8 +43,8 @@ type ElasticOutputConfig struct { } // Build will build an elasticsearch output plugin. -func (c ElasticOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - outputPlugin, err := c.OutputConfig.Build(context) +func (c ElasticOutputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + outputOperator, err := c.OutputConfig.Build(context) if err != nil { return nil, err } @@ -72,11 +72,11 @@ func (c ElasticOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, } elasticOutput := &ElasticOutput{ - OutputPlugin: outputPlugin, - Buffer: buffer, - client: client, - indexField: c.IndexField, - idField: c.IDField, + OutputOperator: outputOperator, + Buffer: buffer, + client: client, + indexField: c.IndexField, + idField: c.IDField, } buffer.SetHandler(elasticOutput) @@ -86,7 +86,7 @@ func (c ElasticOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, // ElasticOutput is a plugin that sends entries to elasticsearch. type ElasticOutput struct { - helper.OutputPlugin + helper.OutputOperator buffer.Buffer client *elasticsearch.Client diff --git a/plugin/builtin/output/file.go b/plugin/builtin/output/file.go index 1734362d7..65285283e 100644 --- a/plugin/builtin/output/file.go +++ b/plugin/builtin/output/file.go @@ -32,8 +32,8 @@ type FileOutputConfig struct { } // Build will build a file output plugin. -func (c FileOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - outputPlugin, err := c.OutputConfig.Build(context) +func (c FileOutputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + outputOperator, err := c.OutputConfig.Build(context) if err != nil { return nil, err } @@ -51,9 +51,9 @@ func (c FileOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, err } fileOutput := &FileOutput{ - OutputPlugin: outputPlugin, - path: c.Path, - tmpl: tmpl, + OutputOperator: outputOperator, + path: c.Path, + tmpl: tmpl, } return fileOutput, nil @@ -61,7 +61,7 @@ func (c FileOutputConfig) Build(context plugin.BuildContext) (plugin.Plugin, err // FileOutput is a plugin that writes logs to a file. type FileOutput struct { - helper.OutputPlugin + helper.OutputOperator path string tmpl *template.Template diff --git a/plugin/builtin/output/google_cloud.go b/plugin/builtin/output/google_cloud.go index ae439f0e2..25bdc36f5 100644 --- a/plugin/builtin/output/google_cloud.go +++ b/plugin/builtin/output/google_cloud.go @@ -51,8 +51,8 @@ type GoogleCloudOutputConfig struct { } // Build will build a google cloud output plugin. -func (c GoogleCloudOutputConfig) Build(buildContext plugin.BuildContext) (plugin.Plugin, error) { - outputPlugin, err := c.OutputConfig.Build(buildContext) +func (c GoogleCloudOutputConfig) Build(buildContext plugin.BuildContext) (plugin.Operator, error) { + outputOperator, err := c.OutputConfig.Build(buildContext) if err != nil { return nil, err } @@ -63,7 +63,7 @@ func (c GoogleCloudOutputConfig) Build(buildContext plugin.BuildContext) (plugin } googleCloudOutput := &GoogleCloudOutput{ - OutputPlugin: outputPlugin, + OutputOperator: outputOperator, credentials: c.Credentials, credentialsFile: c.CredentialsFile, projectID: c.ProjectID, @@ -81,7 +81,7 @@ func (c GoogleCloudOutputConfig) Build(buildContext plugin.BuildContext) (plugin // GoogleCloudOutput is a plugin that sends logs to google cloud logging. type GoogleCloudOutput struct { - helper.OutputPlugin + helper.OutputOperator buffer.Buffer credentials string diff --git a/plugin/builtin/output/stdout.go b/plugin/builtin/output/stdout.go index 1157e191c..2769aa6b1 100644 --- a/plugin/builtin/output/stdout.go +++ b/plugin/builtin/output/stdout.go @@ -31,27 +31,27 @@ type StdoutConfig struct { } // Build will build a stdout plugin. -func (c StdoutConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - outputPlugin, err := c.OutputConfig.Build(context) +func (c StdoutConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + outputOperator, err := c.OutputConfig.Build(context) if err != nil { return nil, err } - return &StdoutPlugin{ - OutputPlugin: outputPlugin, - encoder: json.NewEncoder(Stdout), + return &StdoutOperator{ + OutputOperator: outputOperator, + encoder: json.NewEncoder(Stdout), }, nil } -// StdoutPlugin is a plugin that logs entries using stdout. -type StdoutPlugin struct { - helper.OutputPlugin +// StdoutOperator is a plugin that logs entries using stdout. +type StdoutOperator struct { + helper.OutputOperator encoder *json.Encoder mux sync.Mutex } // Process will log entries received. -func (o *StdoutPlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (o *StdoutOperator) Process(ctx context.Context, entry *entry.Entry) error { o.mux.Lock() err := o.encoder.Encode(entry) if err != nil { diff --git a/plugin/builtin/output/stdout_test.go b/plugin/builtin/output/stdout_test.go index eda366f0f..b7adb456f 100644 --- a/plugin/builtin/output/stdout_test.go +++ b/plugin/builtin/output/stdout_test.go @@ -13,12 +13,12 @@ import ( "github.com/stretchr/testify/require" ) -func TestStdoutPlugin(t *testing.T) { +func TestStdoutOperator(t *testing.T) { cfg := StdoutConfig{ OutputConfig: helper.OutputConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "test_plugin_id", - PluginType: "stdout", + OperatorID: "test_plugin_id", + OperatorType: "stdout", }, }, } @@ -27,7 +27,7 @@ func TestStdoutPlugin(t *testing.T) { require.NoError(t, err) var buf bytes.Buffer - plugin.(*StdoutPlugin).encoder = json.NewEncoder(&buf) + plugin.(*StdoutOperator).encoder = json.NewEncoder(&buf) ts := time.Unix(1591042864, 0) e := &entry.Entry{ diff --git a/plugin/builtin/parser/json.go b/plugin/builtin/parser/json.go index 497a69c90..66b317482 100644 --- a/plugin/builtin/parser/json.go +++ b/plugin/builtin/parser/json.go @@ -26,15 +26,15 @@ type JSONParserConfig struct { } // Build will build a JSON parser plugin. -func (c JSONParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - parserPlugin, err := c.ParserConfig.Build(context) +func (c JSONParserConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + parserOperator, err := c.ParserConfig.Build(context) if err != nil { return nil, err } jsonParser := &JSONParser{ - ParserPlugin: parserPlugin, - json: jsoniter.ConfigFastest, + ParserOperator: parserOperator, + json: jsoniter.ConfigFastest, } return jsonParser, nil @@ -42,13 +42,13 @@ func (c JSONParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, err // JSONParser is a plugin that parses JSON. type JSONParser struct { - helper.ParserPlugin + helper.ParserOperator json jsoniter.API } // Process will parse an entry for JSON. func (j *JSONParser) Process(ctx context.Context, entry *entry.Entry) error { - return j.ParserPlugin.ProcessWith(ctx, entry, j.parse) + return j.ParserOperator.ProcessWith(ctx, entry, j.parse) } // parse will parse a value as JSON. diff --git a/plugin/builtin/parser/json_test.go b/plugin/builtin/parser/json_test.go index d143fbbc9..6943bb493 100644 --- a/plugin/builtin/parser/json_test.go +++ b/plugin/builtin/parser/json_test.go @@ -16,19 +16,19 @@ import ( "go.uber.org/zap" ) -func NewFakeJSONPlugin() (*JSONParser, *testutil.Plugin) { - mock := testutil.Plugin{} +func NewFakeJSONOperator() (*JSONParser, *testutil.Operator) { + mock := testutil.Operator{} logger, _ := zap.NewProduction() return &JSONParser{ - ParserPlugin: helper.ParserPlugin{ - TransformerPlugin: helper.TransformerPlugin{ - WriterPlugin: helper.WriterPlugin{ - BasicPlugin: helper.BasicPlugin{ - PluginID: "test", - PluginType: "json_parser", + ParserOperator: helper.ParserOperator{ + TransformerOperator: helper.TransformerOperator{ + WriterOperator: helper.WriterOperator{ + BasicOperator: helper.BasicOperator{ + OperatorID: "test", + OperatorType: "json_parser", SugaredLogger: logger.Sugar(), }, - OutputPlugins: []plugin.Plugin{&mock}, + OutputOperators: []plugin.Operator{&mock}, }, }, ParseFrom: entry.NewRecordField("testfield"), @@ -39,7 +39,7 @@ func NewFakeJSONPlugin() (*JSONParser, *testutil.Plugin) { } func TestJSONImplementations(t *testing.T) { - require.Implements(t, (*plugin.Plugin)(nil), new(JSONParser)) + require.Implements(t, (*plugin.Operator)(nil), new(JSONParser)) } func TestJSONParser(t *testing.T) { @@ -81,7 +81,7 @@ func TestJSONParser(t *testing.T) { output := entry.New() output.Record = tc.expectedRecord - parser, mockOutput := NewFakeJSONPlugin() + parser, mockOutput := NewFakeJSONOperator() mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { e := args[1].(*entry.Entry) require.Equal(t, tc.expectedRecord, e.Record) @@ -151,9 +151,9 @@ func TestJSONParserWithEmbeddedTimeParser(t *testing.T) { output := entry.New() output.Record = tc.expectedRecord - parser, mockOutput := NewFakeJSONPlugin() + parser, mockOutput := NewFakeJSONOperator() parseFrom := entry.NewRecordField("testparsed", "timestamp") - parser.ParserPlugin.TimeParser = &helper.TimeParser{ + parser.ParserOperator.TimeParser = &helper.TimeParser{ ParseFrom: &parseFrom, LayoutType: "epoch", Layout: "s", diff --git a/plugin/builtin/parser/regex.go b/plugin/builtin/parser/regex.go index 85261ffc4..1e1cf2487 100644 --- a/plugin/builtin/parser/regex.go +++ b/plugin/builtin/parser/regex.go @@ -29,8 +29,8 @@ type RegexParserConfig struct { } // Build will build a regex parser plugin. -func (c RegexParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - parserPlugin, err := c.ParserConfig.Build(context) +func (c RegexParserConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + parserOperator, err := c.ParserConfig.Build(context) if err != nil { return nil, err } @@ -58,8 +58,8 @@ func (c RegexParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, er } regexParser := &RegexParser{ - ParserPlugin: parserPlugin, - regexp: r, + ParserOperator: parserOperator, + regexp: r, } return regexParser, nil @@ -67,13 +67,13 @@ func (c RegexParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, er // RegexParser is a plugin that parses regex in an entry. type RegexParser struct { - helper.ParserPlugin + helper.ParserOperator regexp *regexp.Regexp } // Process will parse an entry for regex. func (r *RegexParser) Process(ctx context.Context, entry *entry.Entry) error { - return r.ParserPlugin.ProcessWith(ctx, entry, r.parse) + return r.ParserOperator.ProcessWith(ctx, entry, r.parse) } // parse will parse a value using the supplied regex. diff --git a/plugin/builtin/parser/regex_test.go b/plugin/builtin/parser/regex_test.go index 01229cb01..a4d1a9a6c 100644 --- a/plugin/builtin/parser/regex_test.go +++ b/plugin/builtin/parser/regex_test.go @@ -13,24 +13,24 @@ import ( "github.com/stretchr/testify/require" ) -func newFakeRegexParser() (*RegexParser, *testutil.Plugin) { - mockPlugin := testutil.Plugin{} +func newFakeRegexParser() (*RegexParser, *testutil.Operator) { + mockOperator := testutil.Operator{} return &RegexParser{ - ParserPlugin: helper.ParserPlugin{ - TransformerPlugin: helper.TransformerPlugin{ - WriterPlugin: helper.WriterPlugin{ - BasicPlugin: helper.BasicPlugin{ - PluginID: "regex_parser", - PluginType: "regex_parser", + ParserOperator: helper.ParserOperator{ + TransformerOperator: helper.TransformerOperator{ + WriterOperator: helper.WriterOperator{ + BasicOperator: helper.BasicOperator{ + OperatorID: "regex_parser", + OperatorType: "regex_parser", }, - OutputIDs: []string{"mock_output"}, - OutputPlugins: []plugin.Plugin{&mockPlugin}, + OutputIDs: []string{"mock_output"}, + OutputOperators: []plugin.Operator{&mockOperator}, }, }, ParseFrom: entry.NewRecordField(), ParseTo: entry.NewRecordField(), }, - }, &mockPlugin + }, &mockOperator } func TestParserRegex(t *testing.T) { diff --git a/plugin/builtin/parser/severity.go b/plugin/builtin/parser/severity.go index 9e521ed35..f304051cb 100644 --- a/plugin/builtin/parser/severity.go +++ b/plugin/builtin/parser/severity.go @@ -27,8 +27,8 @@ type SeverityParserConfig struct { } // Build will build a time parser plugin. -func (c SeverityParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) +func (c SeverityParserConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err } @@ -38,22 +38,22 @@ func (c SeverityParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, return nil, err } - severityPlugin := &SeverityParserPlugin{ - TransformerPlugin: transformerPlugin, - SeverityParser: severityParser, + severityOperator := &SeverityParserOperator{ + TransformerOperator: transformerOperator, + SeverityParser: severityParser, } - return severityPlugin, nil + return severityOperator, nil } -// SeverityParserPlugin is a plugin that parses time from a field to an entry. -type SeverityParserPlugin struct { - helper.TransformerPlugin +// SeverityParserOperator is a plugin that parses time from a field to an entry. +type SeverityParserOperator struct { + helper.TransformerOperator helper.SeverityParser } // Process will parse time from an entry. -func (p *SeverityParserPlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (p *SeverityParserOperator) Process(ctx context.Context, entry *entry.Entry) error { if err := p.Parse(ctx, entry); err != nil { return errors.Wrap(err, "parse severity") } diff --git a/plugin/builtin/parser/severity_test.go b/plugin/builtin/parser/severity_test.go index 9720461ef..844eacd92 100644 --- a/plugin/builtin/parser/severity_test.go +++ b/plugin/builtin/parser/severity_test.go @@ -304,21 +304,21 @@ func runSeverityParseTest(t *testing.T, cfg *SeverityParserConfig, ent *entry.En return func(t *testing.T) { buildContext := testutil.NewBuildContext(t) - severityPlugin, err := cfg.Build(buildContext) + severityOperator, err := cfg.Build(buildContext) if buildErr { require.Error(t, err, "expected error when configuring plugin") return } require.NoError(t, err, "unexpected error when configuring plugin") - mockOutput := &testutil.Plugin{} + mockOutput := &testutil.Operator{} resultChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { resultChan <- args.Get(1).(*entry.Entry) }).Return(nil) - severityParser := severityPlugin.(*SeverityParserPlugin) - severityParser.OutputPlugins = []plugin.Plugin{mockOutput} + severityParser := severityOperator.(*SeverityParserOperator) + severityParser.OutputOperators = []plugin.Operator{mockOutput} err = severityParser.Process(context.Background(), ent) if parseErr { diff --git a/plugin/builtin/parser/syslog.go b/plugin/builtin/parser/syslog.go index a29fa1893..2f55054d1 100644 --- a/plugin/builtin/parser/syslog.go +++ b/plugin/builtin/parser/syslog.go @@ -31,7 +31,7 @@ type SyslogParserConfig struct { } // Build will build a JSON parser plugin. -func (c SyslogParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { +func (c SyslogParserConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { if c.ParserConfig.TimeParser == nil { parseFromField := entry.NewRecordField("timestamp") c.ParserConfig.TimeParser = &helper.TimeParser{ @@ -40,7 +40,7 @@ func (c SyslogParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, e } } - parserPlugin, err := c.ParserConfig.Build(context) + parserOperator, err := c.ParserConfig.Build(context) if err != nil { return nil, err } @@ -55,8 +55,8 @@ func (c SyslogParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, e } syslogParser := &SyslogParser{ - ParserPlugin: parserPlugin, - machine: machine, + ParserOperator: parserOperator, + machine: machine, } return syslogParser, nil @@ -75,13 +75,13 @@ func buildMachine(protocol string) (syslog.Machine, error) { // SyslogParser is a plugin that parses syslog. type SyslogParser struct { - helper.ParserPlugin + helper.ParserOperator machine syslog.Machine } // Process will parse an entry field as syslog. func (s *SyslogParser) Process(ctx context.Context, entry *entry.Entry) error { - return s.ParserPlugin.ProcessWith(ctx, entry, s.parse) + return s.ParserOperator.ProcessWith(ctx, entry, s.parse) } // parse will parse a value as syslog. diff --git a/plugin/builtin/parser/syslog_test.go b/plugin/builtin/parser/syslog_test.go index ebbd7c312..26837aacd 100644 --- a/plugin/builtin/parser/syslog_test.go +++ b/plugin/builtin/parser/syslog_test.go @@ -96,17 +96,17 @@ func TestSyslogParser(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { buildContext := testutil.NewBuildContext(t) - newPlugin, err := tc.config.Build(buildContext) + newOperator, err := tc.config.Build(buildContext) require.NoError(t, err) - syslogParser := newPlugin.(*SyslogParser) + syslogParser := newOperator.(*SyslogParser) - mockOutput := testutil.NewMockPlugin("output1") + mockOutput := testutil.NewMockOperator("output1") entryChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = syslogParser.SetOutputs([]plugin.Plugin{mockOutput}) + err = syslogParser.SetOutputs([]plugin.Operator{mockOutput}) require.NoError(t, err) newEntry := entry.New() diff --git a/plugin/builtin/parser/time.go b/plugin/builtin/parser/time.go index 871e6f685..d674a99b3 100644 --- a/plugin/builtin/parser/time.go +++ b/plugin/builtin/parser/time.go @@ -27,8 +27,8 @@ type TimeParserConfig struct { } // Build will build a time parser plugin. -func (c TimeParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) +func (c TimeParserConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err } @@ -37,27 +37,27 @@ func (c TimeParserConfig) Build(context plugin.BuildContext) (plugin.Plugin, err return nil, err } - timeParser := &TimeParserPlugin{ - TransformerPlugin: transformerPlugin, - TimeParser: c.TimeParser, + timeParser := &TimeParserOperator{ + TransformerOperator: transformerOperator, + TimeParser: c.TimeParser, } return timeParser, nil } -// TimeParserPlugin is a plugin that parses time from a field to an entry. -type TimeParserPlugin struct { - helper.TransformerPlugin +// TimeParserOperator is a plugin that parses time from a field to an entry. +type TimeParserOperator struct { + helper.TransformerOperator helper.TimeParser } // CanOutput will always return true for a parser plugin. -func (t *TimeParserPlugin) CanOutput() bool { +func (t *TimeParserOperator) CanOutput() bool { return true } // Process will parse time from an entry. -func (t *TimeParserPlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (t *TimeParserOperator) Process(ctx context.Context, entry *entry.Entry) error { if err := t.Parse(ctx, entry); err != nil { return errors.Wrap(err, "parse timestamp") } diff --git a/plugin/builtin/parser/time_test.go b/plugin/builtin/parser/time_test.go index 37158a63b..be187114d 100644 --- a/plugin/builtin/parser/time_test.go +++ b/plugin/builtin/parser/time_test.go @@ -392,21 +392,21 @@ func runLossyTimeParseTest(t *testing.T, cfg *TimeParserConfig, ent *entry.Entry return func(t *testing.T) { buildContext := testutil.NewBuildContext(t) - gotimePlugin, err := cfg.Build(buildContext) + gotimeOperator, err := cfg.Build(buildContext) if buildErr { require.Error(t, err, "expected error when configuring plugin") return } require.NoError(t, err) - mockOutput := &testutil.Plugin{} + mockOutput := &testutil.Operator{} resultChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { resultChan <- args.Get(1).(*entry.Entry) }).Return(nil) - timeParser := gotimePlugin.(*TimeParserPlugin) - timeParser.OutputPlugins = []plugin.Plugin{mockOutput} + timeParser := gotimeOperator.(*TimeParserOperator) + timeParser.OutputOperators = []plugin.Operator{mockOutput} err = timeParser.Process(context.Background(), ent) if parseErr { diff --git a/plugin/builtin/transformer/k8s_metadata_decorator.go b/plugin/builtin/transformer/k8s_metadata_decorator.go index 7a1df2170..71e9e4451 100644 --- a/plugin/builtin/transformer/k8s_metadata_decorator.go +++ b/plugin/builtin/transformer/k8s_metadata_decorator.go @@ -36,23 +36,23 @@ type K8sMetadataDecoratorConfig struct { } // Build will build a k8s_metadata_decorator plugin from the supplied configuration -func (c K8sMetadataDecoratorConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { +func (c K8sMetadataDecoratorConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { transformer, err := c.TransformerConfig.Build(context) if err != nil { return nil, errors.Wrap(err, "build transformer") } return &K8sMetadataDecorator{ - TransformerPlugin: transformer, - podNameField: c.PodNameField, - namespaceField: c.NamespaceField, - cacheTTL: c.CacheTTL.Raw(), + TransformerOperator: transformer, + podNameField: c.PodNameField, + namespaceField: c.NamespaceField, + cacheTTL: c.CacheTTL.Raw(), }, nil } // K8sMetadataDecorator is a plugin for decorating entries with kubernetes metadata type K8sMetadataDecorator struct { - helper.TransformerPlugin + helper.TransformerOperator podNameField entry.Field namespaceField entry.Field diff --git a/plugin/builtin/transformer/k8s_metadata_decorator_test.go b/plugin/builtin/transformer/k8s_metadata_decorator_test.go index 4b76a69f6..883a6d3e9 100644 --- a/plugin/builtin/transformer/k8s_metadata_decorator_test.go +++ b/plugin/builtin/transformer/k8s_metadata_decorator_test.go @@ -40,11 +40,11 @@ func TestK8sMetadataDecoratorBuildDefault(t *testing.T) { cfg := basicConfig() expected := &K8sMetadataDecorator{ - TransformerPlugin: helper.TransformerPlugin{ - WriterPlugin: helper.WriterPlugin{ - BasicPlugin: helper.BasicPlugin{ - PluginID: "testplugin", - PluginType: "k8s_metadata_decorator", + TransformerOperator: helper.TransformerOperator{ + WriterOperator: helper.WriterOperator{ + BasicOperator: helper.BasicOperator{ + OperatorID: "testplugin", + OperatorType: "k8s_metadata_decorator", }, OutputIDs: []string{"mock"}, }, @@ -68,8 +68,8 @@ func TestK8sMetadataDecoratorCachedMetadata(t *testing.T) { pg, err := cfg.Build(testutil.NewBuildContext(t)) require.NoError(t, err) - mockOutput := testutil.NewMockPlugin("mock") - pg.SetOutputs([]plugin.Plugin{mockOutput}) + mockOutput := testutil.NewMockOperator("mock") + pg.SetOutputs([]plugin.Operator{mockOutput}) // Preload cache so we don't hit the network k8s := pg.(*K8sMetadataDecorator) diff --git a/plugin/builtin/transformer/metadata.go b/plugin/builtin/transformer/metadata.go index d6c82833c..7a3ba2574 100644 --- a/plugin/builtin/transformer/metadata.go +++ b/plugin/builtin/transformer/metadata.go @@ -10,17 +10,17 @@ import ( ) func init() { - plugin.Register("metadata", func() plugin.Builder { return NewMetadataPluginConfig("") }) + plugin.Register("metadata", func() plugin.Builder { return NewMetadataOperatorConfig("") }) } -func NewMetadataPluginConfig(pluginID string) *MetadataPluginConfig { - return &MetadataPluginConfig{ +func NewMetadataOperatorConfig(pluginID string) *MetadataOperatorConfig { + return &MetadataOperatorConfig{ TransformerConfig: helper.NewTransformerConfig(pluginID, "metadata"), } } -// MetadataPluginConfig is the configuration of a metadata plugin -type MetadataPluginConfig struct { +// MetadataOperatorConfig is the configuration of a metadata plugin +type MetadataOperatorConfig struct { helper.TransformerConfig `yaml:",inline"` Labels map[string]helper.ExprStringConfig `json:"labels" yaml:"labels"` @@ -28,8 +28,8 @@ type MetadataPluginConfig struct { } // Build will build a metadata plugin from the supplied configuration -func (c MetadataPluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) +func (c MetadataOperatorConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err } @@ -44,29 +44,29 @@ func (c MetadataPluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, return nil, errors.Wrap(err, "validate labels") } - restructurePlugin := &MetadataPlugin{ - TransformerPlugin: transformerPlugin, - labeler: labeler, - tagger: tagger, + restructureOperator := &MetadataOperator{ + TransformerOperator: transformerOperator, + labeler: labeler, + tagger: tagger, } - return restructurePlugin, nil + return restructureOperator, nil } -// MetadataPlugin is a plugin that can add metadata to incoming entries -type MetadataPlugin struct { - helper.TransformerPlugin +// MetadataOperator is a plugin that can add metadata to incoming entries +type MetadataOperator struct { + helper.TransformerOperator labeler *labeler tagger *tagger } // Process will process an incoming entry using the metadata transform. -func (p *MetadataPlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (p *MetadataOperator) Process(ctx context.Context, entry *entry.Entry) error { return p.ProcessWith(ctx, entry, p.Transform) } // Transform will transform an entry using the labeler and tagger. -func (p *MetadataPlugin) Transform(entry *entry.Entry) (*entry.Entry, error) { +func (p *MetadataOperator) Transform(entry *entry.Entry) (*entry.Entry, error) { err := p.labeler.Label(entry) if err != nil { return entry, err diff --git a/plugin/builtin/transformer/metadata_test.go b/plugin/builtin/transformer/metadata_test.go index f17d163de..9ea97cf82 100644 --- a/plugin/builtin/transformer/metadata_test.go +++ b/plugin/builtin/transformer/metadata_test.go @@ -18,21 +18,21 @@ func TestMetadata(t *testing.T) { os.Setenv("TEST_METADATA_PLUGIN_ENV", "foo") defer os.Unsetenv("TEST_METADATA_PLUGIN_ENV") - baseConfig := func() *MetadataPluginConfig { - cfg := NewMetadataPluginConfig("test_plugin_id") + baseConfig := func() *MetadataOperatorConfig { + cfg := NewMetadataOperatorConfig("test_plugin_id") cfg.OutputIDs = []string{"output1"} return cfg } cases := []struct { name string - config *MetadataPluginConfig + config *MetadataOperatorConfig input *entry.Entry expected *entry.Entry }{ { "AddTagLiteral", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Tags = []helper.ExprStringConfig{"tag1"} return cfg @@ -46,7 +46,7 @@ func TestMetadata(t *testing.T) { }, { "AddTagExpr", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Tags = []helper.ExprStringConfig{`prefix-EXPR( 'test1' )`} return cfg @@ -60,7 +60,7 @@ func TestMetadata(t *testing.T) { }, { "AddLabelLiteral", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Labels = map[string]helper.ExprStringConfig{ "label1": "value1", @@ -78,7 +78,7 @@ func TestMetadata(t *testing.T) { }, { "AddLabelExpr", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Labels = map[string]helper.ExprStringConfig{ "label1": `EXPR("start" + "end")`, @@ -96,7 +96,7 @@ func TestMetadata(t *testing.T) { }, { "AddLabelEnv", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Labels = map[string]helper.ExprStringConfig{ "label1": `EXPR(env("TEST_METADATA_PLUGIN_ENV"))`, @@ -114,7 +114,7 @@ func TestMetadata(t *testing.T) { }, { "AddTagEnv", - func() *MetadataPluginConfig { + func() *MetadataOperatorConfig { cfg := baseConfig() cfg.Tags = []helper.ExprStringConfig{`EXPR(env("TEST_METADATA_PLUGIN_ENV"))`} return cfg @@ -130,19 +130,19 @@ func TestMetadata(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - metadataPlugin, err := tc.config.Build(testutil.NewBuildContext(t)) + metadataOperator, err := tc.config.Build(testutil.NewBuildContext(t)) require.NoError(t, err) - mockOutput := testutil.NewMockPlugin("output1") + mockOutput := testutil.NewMockOperator("output1") entryChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = metadataPlugin.SetOutputs([]plugin.Plugin{mockOutput}) + err = metadataOperator.SetOutputs([]plugin.Operator{mockOutput}) require.NoError(t, err) - err = metadataPlugin.Process(context.Background(), tc.input) + err = metadataOperator.Process(context.Background(), tc.input) require.NoError(t, err) select { diff --git a/plugin/builtin/transformer/noop.go b/plugin/builtin/transformer/noop.go index 1172fa212..e477cc4f1 100644 --- a/plugin/builtin/transformer/noop.go +++ b/plugin/builtin/transformer/noop.go @@ -9,41 +9,41 @@ import ( ) func init() { - plugin.Register("noop", func() plugin.Builder { return NewNoopPluginConfig("") }) + plugin.Register("noop", func() plugin.Builder { return NewNoopOperatorConfig("") }) } -func NewNoopPluginConfig(pluginID string) *NoopPluginConfig { - return &NoopPluginConfig{ +func NewNoopOperatorConfig(pluginID string) *NoopOperatorConfig { + return &NoopOperatorConfig{ TransformerConfig: helper.NewTransformerConfig(pluginID, "noop"), } } -// NoopPluginConfig is the configuration of a noop plugin. -type NoopPluginConfig struct { +// NoopOperatorConfig is the configuration of a noop plugin. +type NoopOperatorConfig struct { helper.TransformerConfig `yaml:",inline"` } // Build will build a noop plugin. -func (c NoopPluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) +func (c NoopOperatorConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err } - noopPlugin := &NoopPlugin{ - TransformerPlugin: transformerPlugin, + noopOperator := &NoopOperator{ + TransformerOperator: transformerOperator, } - return noopPlugin, nil + return noopOperator, nil } -// NoopPlugin is a plugin that performs no operations on an entry. -type NoopPlugin struct { - helper.TransformerPlugin +// NoopOperator is a plugin that performs no operations on an entry. +type NoopOperator struct { + helper.TransformerOperator } // Process will forward the entry to the next output without any alterations. -func (p *NoopPlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (p *NoopOperator) Process(ctx context.Context, entry *entry.Entry) error { p.Write(ctx, entry) return nil } diff --git a/plugin/builtin/transformer/noop_test.go b/plugin/builtin/transformer/noop_test.go index 8c2fc024a..cb56c07ba 100644 --- a/plugin/builtin/transformer/noop_test.go +++ b/plugin/builtin/transformer/noop_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/require" ) -func TestNoopPluginBuild(t *testing.T) { - cfg := NewNoopPluginConfig("test_plugin_id") +func TestNoopOperatorBuild(t *testing.T) { + cfg := NewNoopOperatorConfig("test_plugin_id") cfg.OutputIDs = []string{"output"} _, err := cfg.Build(testutil.NewBuildContext(t)) diff --git a/plugin/builtin/transformer/rate_limit.go b/plugin/builtin/transformer/rate_limit.go index f6c0e6dbe..7c9737f1d 100644 --- a/plugin/builtin/transformer/rate_limit.go +++ b/plugin/builtin/transformer/rate_limit.go @@ -30,8 +30,8 @@ type RateLimitConfig struct { } // Build will build a rate limit plugin. -func (c RateLimitConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) +func (c RateLimitConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err } @@ -48,18 +48,18 @@ func (c RateLimitConfig) Build(context plugin.BuildContext) (plugin.Plugin, erro interval = c.Interval.Raw() } - rateLimitPlugin := &RateLimitPlugin{ - TransformerPlugin: transformerPlugin, - interval: interval, - burst: c.Burst, + rateLimitOperator := &RateLimitOperator{ + TransformerOperator: transformerOperator, + interval: interval, + burst: c.Burst, } - return rateLimitPlugin, nil + return rateLimitOperator, nil } -// RateLimitPlugin is a plugin that limits the rate of log consumption between plugins. -type RateLimitPlugin struct { - helper.TransformerPlugin +// RateLimitOperator is a plugin that limits the rate of log consumption between plugins. +type RateLimitOperator struct { + helper.TransformerOperator interval time.Duration burst uint @@ -68,14 +68,14 @@ type RateLimitPlugin struct { } // Process will wait until a rate is met before sending an entry to the output. -func (p *RateLimitPlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (p *RateLimitOperator) Process(ctx context.Context, entry *entry.Entry) error { <-p.isReady p.Write(ctx, entry) return nil } // Start will start the rate limit plugin. -func (p *RateLimitPlugin) Start() error { +func (p *RateLimitOperator) Start() error { p.isReady = make(chan struct{}, p.burst) ticker := time.NewTicker(p.interval) @@ -100,7 +100,7 @@ func (p *RateLimitPlugin) Start() error { } // Stop will stop the rate limit plugin. -func (p *RateLimitPlugin) Stop() error { +func (p *RateLimitOperator) Stop() error { p.cancel() return nil } diff --git a/plugin/builtin/transformer/rate_limit_test.go b/plugin/builtin/transformer/rate_limit_test.go index 70c6e1c6b..6ac994850 100644 --- a/plugin/builtin/transformer/rate_limit_test.go +++ b/plugin/builtin/transformer/rate_limit_test.go @@ -25,12 +25,12 @@ func TestRateLimit(t *testing.T) { require.NoError(t, err) receivedLog := make(chan struct{}, 100) - mockOutput := testutil.NewMockPlugin("output1") + mockOutput := testutil.NewMockOperator("output1") mockOutput.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { receivedLog <- struct{}{} }) - err = rateLimit.SetOutputs([]plugin.Plugin{mockOutput}) + err = rateLimit.SetOutputs([]plugin.Operator{mockOutput}) require.NoError(t, err) err = rateLimit.Start() diff --git a/plugin/builtin/transformer/restructure.go b/plugin/builtin/transformer/restructure.go index eb56d11d4..3605b2d3b 100644 --- a/plugin/builtin/transformer/restructure.go +++ b/plugin/builtin/transformer/restructure.go @@ -14,50 +14,50 @@ import ( ) func init() { - plugin.Register("restructure", func() plugin.Builder { return NewRestructurePluginConfig("") }) + plugin.Register("restructure", func() plugin.Builder { return NewRestructureOperatorConfig("") }) } -func NewRestructurePluginConfig(pluginID string) *RestructurePluginConfig { - return &RestructurePluginConfig{ +func NewRestructureOperatorConfig(pluginID string) *RestructureOperatorConfig { + return &RestructureOperatorConfig{ TransformerConfig: helper.NewTransformerConfig(pluginID, "restructure"), } } -// RestructurePluginConfig is the configuration of a restructure plugin -type RestructurePluginConfig struct { +// RestructureOperatorConfig is the configuration of a restructure plugin +type RestructureOperatorConfig struct { helper.TransformerConfig `yaml:",inline"` Ops []Op `json:"ops" yaml:"ops"` } // Build will build a restructure plugin from the supplied configuration -func (c RestructurePluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) +func (c RestructureOperatorConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err } - restructurePlugin := &RestructurePlugin{ - TransformerPlugin: transformerPlugin, - ops: c.Ops, + restructureOperator := &RestructureOperator{ + TransformerOperator: transformerOperator, + ops: c.Ops, } - return restructurePlugin, nil + return restructureOperator, nil } -// RestructurePlugin is a plugin that can restructure incoming entries using operations -type RestructurePlugin struct { - helper.TransformerPlugin +// RestructureOperator is a plugin that can restructure incoming entries using operations +type RestructureOperator struct { + helper.TransformerOperator ops []Op } // Process will process an entry with a restructure transformation. -func (p *RestructurePlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (p *RestructureOperator) Process(ctx context.Context, entry *entry.Entry) error { return p.ProcessWith(ctx, entry, p.Transform) } // Transform will apply the restructure operations to an entry -func (p *RestructurePlugin) Transform(entry *entry.Entry) (*entry.Entry, error) { +func (p *RestructureOperator) Transform(entry *entry.Entry) (*entry.Entry, error) { for _, op := range p.ops { err := op.Apply(entry) if err != nil { diff --git a/plugin/builtin/transformer/restructure_test.go b/plugin/builtin/transformer/restructure_test.go index e519a0db1..0e30e34ca 100644 --- a/plugin/builtin/transformer/restructure_test.go +++ b/plugin/builtin/transformer/restructure_test.go @@ -19,24 +19,24 @@ import ( yaml "gopkg.in/yaml.v2" ) -func NewFakeRestructurePlugin() (*RestructurePlugin, *testutil.Plugin) { - mock := testutil.Plugin{} +func NewFakeRestructureOperator() (*RestructureOperator, *testutil.Operator) { + mock := testutil.Operator{} logger, _ := zap.NewProduction() - return &RestructurePlugin{ - TransformerPlugin: helper.TransformerPlugin{ - WriterPlugin: helper.WriterPlugin{ - BasicPlugin: helper.BasicPlugin{ - PluginID: "test", - PluginType: "restructure", + return &RestructureOperator{ + TransformerOperator: helper.TransformerOperator{ + WriterOperator: helper.WriterOperator{ + BasicOperator: helper.BasicOperator{ + OperatorID: "test", + OperatorType: "restructure", SugaredLogger: logger.Sugar(), }, - OutputPlugins: []plugin.Plugin{&mock}, + OutputOperators: []plugin.Operator{&mock}, }, }, }, &mock } -func TestRestructurePlugin(t *testing.T) { +func TestRestructureOperator(t *testing.T) { os.Setenv("TEST_RESTRUCTURE_PLUGIN_ENV", "foo") defer os.Unsetenv("TEST_RESTRUCTURE_PLUGIN_ENV") @@ -202,7 +202,7 @@ func TestRestructurePlugin(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - plugin, mockOutput := NewFakeRestructurePlugin() + plugin, mockOutput := NewFakeRestructureOperator() plugin.ops = tc.ops mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { @@ -343,12 +343,12 @@ ops: }` expected := plugin.Config(plugin.Config{ - Builder: &RestructurePluginConfig{ + Builder: &RestructureOperatorConfig{ TransformerConfig: helper.TransformerConfig{ WriterConfig: helper.WriterConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "my_restructure", - PluginType: "restructure", + OperatorID: "my_restructure", + OperatorType: "restructure", }, OutputIDs: []string{"test_output"}, }, diff --git a/plugin/builtin/transformer/router.go b/plugin/builtin/transformer/router.go index 4f9060d6a..eecd0ffe7 100644 --- a/plugin/builtin/transformer/router.go +++ b/plugin/builtin/transformer/router.go @@ -13,57 +13,57 @@ import ( ) func init() { - plugin.Register("router", func() plugin.Builder { return NewRouterPluginConfig("") }) + plugin.Register("router", func() plugin.Builder { return NewRouterOperatorConfig("") }) } -func NewRouterPluginConfig(pluginID string) *RouterPluginConfig { - return &RouterPluginConfig{ +func NewRouterOperatorConfig(pluginID string) *RouterOperatorConfig { + return &RouterOperatorConfig{ BasicConfig: helper.NewBasicConfig(pluginID, "router"), } } -// RouterPluginConfig is the configuration of a router plugin -type RouterPluginConfig struct { +// RouterOperatorConfig is the configuration of a router plugin +type RouterOperatorConfig struct { helper.BasicConfig `yaml:",inline"` - Routes []*RouterPluginRouteConfig `json:"routes" yaml:"routes"` + Routes []*RouterOperatorRouteConfig `json:"routes" yaml:"routes"` } -// RouterPluginRouteConfig is the configuration of a route on a router plugin -type RouterPluginRouteConfig struct { +// RouterOperatorRouteConfig is the configuration of a route on a router plugin +type RouterOperatorRouteConfig struct { Expression string `json:"expr" yaml:"expr"` OutputIDs helper.OutputIDs `json:"output" yaml:"output"` } // Build will build a router plugin from the supplied configuration -func (c RouterPluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - basicPlugin, err := c.BasicConfig.Build(context) +func (c RouterOperatorConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { + basicOperator, err := c.BasicConfig.Build(context) if err != nil { return nil, err } - routes := make([]*RouterPluginRoute, 0, len(c.Routes)) + routes := make([]*RouterOperatorRoute, 0, len(c.Routes)) for _, routeConfig := range c.Routes { compiled, err := expr.Compile(routeConfig.Expression, expr.AsBool(), expr.AllowUndefinedVariables()) if err != nil { return nil, fmt.Errorf("failed to compile expression '%s': %w", routeConfig.Expression, err) } - route := RouterPluginRoute{ + route := RouterOperatorRoute{ Expression: compiled, OutputIDs: routeConfig.OutputIDs, } routes = append(routes, &route) } - routerPlugin := &RouterPlugin{ - BasicPlugin: basicPlugin, - routes: routes, + routerOperator := &RouterOperator{ + BasicOperator: basicOperator, + routes: routes, } - return routerPlugin, nil + return routerOperator, nil } // SetNamespace will namespace the router plugin and the outputs contained in its routes -func (c *RouterPluginConfig) SetNamespace(namespace string, exclusions ...string) { +func (c *RouterOperatorConfig) SetNamespace(namespace string, exclusions ...string) { c.BasicConfig.SetNamespace(namespace, exclusions...) for _, route := range c.Routes { for i, outputID := range route.OutputIDs { @@ -74,26 +74,26 @@ func (c *RouterPluginConfig) SetNamespace(namespace string, exclusions ...string } } -// RouterPlugin is a plugin that routes entries based on matching expressions -type RouterPlugin struct { - helper.BasicPlugin - routes []*RouterPluginRoute +// RouterOperator is a plugin that routes entries based on matching expressions +type RouterOperator struct { + helper.BasicOperator + routes []*RouterOperatorRoute } -// RouterPluginRoute is a route on a router plugin -type RouterPluginRoute struct { - Expression *vm.Program - OutputIDs helper.OutputIDs - OutputPlugins []plugin.Plugin +// RouterOperatorRoute is a route on a router plugin +type RouterOperatorRoute struct { + Expression *vm.Program + OutputIDs helper.OutputIDs + OutputOperators []plugin.Operator } // CanProcess will always return true for a router plugin -func (p *RouterPlugin) CanProcess() bool { +func (p *RouterOperator) CanProcess() bool { return true } // Process will route incoming entries based on matching expressions -func (p *RouterPlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (p *RouterOperator) Process(ctx context.Context, entry *entry.Entry) error { env := helper.GetExprEnv(entry) defer helper.PutExprEnv(env) @@ -106,7 +106,7 @@ func (p *RouterPlugin) Process(ctx context.Context, entry *entry.Entry) error { // we compile the expression with "AsBool", so this should be safe if matches.(bool) { - for _, output := range route.OutputPlugins { + for _, output := range route.OutputOperators { _ = output.Process(ctx, entry) } break @@ -117,36 +117,36 @@ func (p *RouterPlugin) Process(ctx context.Context, entry *entry.Entry) error { } // CanOutput will always return true for a router plugin -func (p *RouterPlugin) CanOutput() bool { +func (p *RouterOperator) CanOutput() bool { return true } // Outputs will return all connected plugins. -func (p *RouterPlugin) Outputs() []plugin.Plugin { - outputs := make([]plugin.Plugin, 0, len(p.routes)) +func (p *RouterOperator) Outputs() []plugin.Operator { + outputs := make([]plugin.Operator, 0, len(p.routes)) for _, route := range p.routes { - outputs = append(outputs, route.OutputPlugins...) + outputs = append(outputs, route.OutputOperators...) } return outputs } // SetOutputs will set the outputs of the router plugin. -func (p *RouterPlugin) SetOutputs(plugins []plugin.Plugin) error { +func (p *RouterOperator) SetOutputs(plugins []plugin.Operator) error { for _, route := range p.routes { - outputPlugins, err := p.findPlugins(plugins, route.OutputIDs) + outputOperators, err := p.findOperators(plugins, route.OutputIDs) if err != nil { return fmt.Errorf("failed to set outputs on route: %s", err) } - route.OutputPlugins = outputPlugins + route.OutputOperators = outputOperators } return nil } -// findPlugins will find a subset of plugins from a collection. -func (p *RouterPlugin) findPlugins(plugins []plugin.Plugin, pluginIDs []string) ([]plugin.Plugin, error) { - result := make([]plugin.Plugin, 0) +// findOperators will find a subset of plugins from a collection. +func (p *RouterOperator) findOperators(plugins []plugin.Operator, pluginIDs []string) ([]plugin.Operator, error) { + result := make([]plugin.Operator, 0) for _, pluginID := range pluginIDs { - plugin, err := p.findPlugin(plugins, pluginID) + plugin, err := p.findOperator(plugins, pluginID) if err != nil { return nil, err } @@ -155,8 +155,8 @@ func (p *RouterPlugin) findPlugins(plugins []plugin.Plugin, pluginIDs []string) return result, nil } -// findPlugin will find a plugin from a collection. -func (p *RouterPlugin) findPlugin(plugins []plugin.Plugin, pluginID string) (plugin.Plugin, error) { +// findOperator will find a plugin from a collection. +func (p *RouterOperator) findOperator(plugins []plugin.Operator, pluginID string) (plugin.Operator, error) { for _, plugin := range plugins { if plugin.ID() == pluginID { return plugin, nil diff --git a/plugin/builtin/transformer/router_test.go b/plugin/builtin/transformer/router_test.go index d8c415499..ca142517c 100644 --- a/plugin/builtin/transformer/router_test.go +++ b/plugin/builtin/transformer/router_test.go @@ -13,15 +13,15 @@ import ( "github.com/stretchr/testify/require" ) -func TestRouterPlugin(t *testing.T) { +func TestRouterOperator(t *testing.T) { os.Setenv("TEST_ROUTER_PLUGIN_ENV", "foo") defer os.Unsetenv("TEST_ROUTER_PLUGIN_ENV") - basicConfig := func() *RouterPluginConfig { - return &RouterPluginConfig{ + basicConfig := func() *RouterOperatorConfig { + return &RouterOperatorConfig{ BasicConfig: helper.BasicConfig{ - PluginID: "test_plugin_id", - PluginType: "router", + OperatorID: "test_plugin_id", + OperatorType: "router", }, } } @@ -29,13 +29,13 @@ func TestRouterPlugin(t *testing.T) { cases := []struct { name string input *entry.Entry - routes []*RouterPluginRouteConfig + routes []*RouterOperatorRouteConfig expectedCounts map[string]int }{ { "DefaultRoute", entry.New(), - []*RouterPluginRouteConfig{ + []*RouterOperatorRouteConfig{ { "true", []string{"output1"}, @@ -46,7 +46,7 @@ func TestRouterPlugin(t *testing.T) { { "NoMatch", entry.New(), - []*RouterPluginRouteConfig{ + []*RouterOperatorRouteConfig{ { `false`, []string{"output1"}, @@ -61,7 +61,7 @@ func TestRouterPlugin(t *testing.T) { "message": "test_message", }, }, - []*RouterPluginRouteConfig{ + []*RouterOperatorRouteConfig{ { `$.message == "non_match"`, []string{"output1"}, @@ -80,7 +80,7 @@ func TestRouterPlugin(t *testing.T) { "message": "test_message", }, }, - []*RouterPluginRouteConfig{ + []*RouterOperatorRouteConfig{ { `env("TEST_ROUTER_PLUGIN_ENV") == "foo"`, []string{"output1"}, @@ -100,25 +100,25 @@ func TestRouterPlugin(t *testing.T) { cfg.Routes = tc.routes buildContext := testutil.NewBuildContext(t) - newPlugin, err := cfg.Build(buildContext) + newOperator, err := cfg.Build(buildContext) require.NoError(t, err) results := map[string]int{} - mock1 := testutil.NewMockPlugin("output1") + mock1 := testutil.NewMockOperator("output1") mock1.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { results["output1"] = results["output1"] + 1 }) - mock2 := testutil.NewMockPlugin("output2") + mock2 := testutil.NewMockOperator("output2") mock2.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { results["output2"] = results["output2"] + 1 }) - routerPlugin := newPlugin.(*RouterPlugin) - err = routerPlugin.SetOutputs([]plugin.Plugin{mock1, mock2}) + routerOperator := newOperator.(*RouterOperator) + err = routerOperator.SetOutputs([]plugin.Operator{mock1, mock2}) require.NoError(t, err) - err = routerPlugin.Process(context.Background(), tc.input) + err = routerOperator.Process(context.Background(), tc.input) require.NoError(t, err) require.Equal(t, tc.expectedCounts, results) diff --git a/plugin/config.go b/plugin/config.go index 7ad879c70..8515005cd 100644 --- a/plugin/config.go +++ b/plugin/config.go @@ -8,20 +8,20 @@ import ( "go.uber.org/zap" ) -// Config is the configuration of a plugin +// Config is the configuration of a operator type Config struct { Builder } -// Builder is an entity that can build plugins +// Builder is an entity that can build operators type Builder interface { ID() string Type() string - Build(BuildContext) (Plugin, error) + Build(BuildContext) (Operator, error) SetNamespace(namespace string, exclude ...string) } -// BuildContext supplies contextual resources when building a plugin. +// BuildContext supplies contextual resources when building an operator. type BuildContext struct { CustomRegistry CustomRegistry Database Database @@ -58,18 +58,18 @@ func NewStubDatabase() *StubDatabase { return &StubDatabase{} } -// registry is a global registry of plugin types to plugin builders. +// registry is a global registry of operator types to operator builders. var registry = make(map[string]func() Builder) -// Register will register a function to a plugin type. +// Register will register a function to a operator type. // This function will return a builder for the supplied type. -func Register(pluginType string, newBuilder func() Builder) { - registry[pluginType] = newBuilder +func Register(operatorType string, newBuilder func() Builder) { + registry[operatorType] = newBuilder } -// IsDefined will return a boolean indicating if a plugin type is registered and defined. -func IsDefined(pluginType string) bool { - _, ok := registry[pluginType] +// IsDefined will return a boolean indicating if a operator type is registered and defined. +func IsDefined(operatorType string) bool { + _, ok := registry[operatorType] return ok } diff --git a/plugin/config_test.go b/plugin/config_test.go index c9294ac3d..7f0e76db7 100644 --- a/plugin/config_test.go +++ b/plugin/config_test.go @@ -25,15 +25,15 @@ func TestStubDatabase(t *testing.T) { } type FakeBuilder struct { - PluginID string `json:"id" yaml:"id"` - PluginType string `json:"type" yaml:"type"` - Array []string `json:"array" yaml:"array"` + OperatorID string `json:"id" yaml:"id"` + OperatorType string `json:"type" yaml:"type"` + Array []string `json:"array" yaml:"array"` } -func (f *FakeBuilder) SetNamespace(s string, e ...string) {} -func (f *FakeBuilder) Build(context BuildContext) (Plugin, error) { return nil, nil } -func (f *FakeBuilder) ID() string { return "custom" } -func (f *FakeBuilder) Type() string { return "custom" } +func (f *FakeBuilder) SetNamespace(s string, e ...string) {} +func (f *FakeBuilder) Build(context BuildContext) (Operator, error) { return nil, nil } +func (f *FakeBuilder) ID() string { return "custom" } +func (f *FakeBuilder) Type() string { return "custom" } func TestUnmarshalJSONErrors(t *testing.T) { t.Run("InvalidJSON", func(t *testing.T) { @@ -73,9 +73,9 @@ func TestUnmarshalJSONErrors(t *testing.T) { func TestMarshalJSON(t *testing.T) { cfg := Config{ Builder: &FakeBuilder{ - PluginID: "custom", - PluginType: "custom", - Array: []string{"test"}, + OperatorID: "custom", + OperatorType: "custom", + Array: []string{"test"}, }, } out, err := json.Marshal(cfg) @@ -130,9 +130,9 @@ func TestUnmarshalYAMLErrors(t *testing.T) { func TestMarshalYAML(t *testing.T) { cfg := Config{ Builder: &FakeBuilder{ - PluginID: "custom", - PluginType: "custom", - Array: []string{"test"}, + OperatorID: "custom", + OperatorType: "custom", + Array: []string{"test"}, }, } out, err := yaml.Marshal(cfg) diff --git a/plugin/helper/input.go b/plugin/helper/input.go index 84fea7ce9..50826de03 100644 --- a/plugin/helper/input.go +++ b/plugin/helper/input.go @@ -23,43 +23,43 @@ type InputConfig struct { } // Build will build a base producer. -func (c InputConfig) Build(context plugin.BuildContext) (InputPlugin, error) { - writerPlugin, err := c.WriterConfig.Build(context) +func (c InputConfig) Build(context plugin.BuildContext) (InputOperator, error) { + writerOperator, err := c.WriterConfig.Build(context) if err != nil { - return InputPlugin{}, errors.WithDetails(err, "plugin_id", c.ID()) + return InputOperator{}, errors.WithDetails(err, "plugin_id", c.ID()) } - inputPlugin := InputPlugin{ - WriterPlugin: writerPlugin, - WriteTo: c.WriteTo, + inputOperator := InputOperator{ + WriterOperator: writerOperator, + WriteTo: c.WriteTo, } - return inputPlugin, nil + return inputOperator, nil } -// InputPlugin provides a basic implementation of an input plugin. -type InputPlugin struct { - WriterPlugin +// InputOperator provides a basic implementation of an input plugin. +type InputOperator struct { + WriterOperator WriteTo entry.Field } // NewEntry will create a new entry using the write_to field. -func (i *InputPlugin) NewEntry(value interface{}) *entry.Entry { +func (i *InputOperator) NewEntry(value interface{}) *entry.Entry { entry := entry.New() entry.Set(i.WriteTo, value) return entry } // CanProcess will always return false for an input plugin. -func (i *InputPlugin) CanProcess() bool { +func (i *InputOperator) CanProcess() bool { return false } // Process will always return an error if called. -func (i *InputPlugin) Process(ctx context.Context, entry *entry.Entry) error { - i.Errorw("Plugin received an entry, but can not process", zap.Any("entry", entry)) +func (i *InputOperator) Process(ctx context.Context, entry *entry.Entry) error { + i.Errorw("Operator received an entry, but can not process", zap.Any("entry", entry)) return errors.NewError( - "Plugin can not process logs.", + "Operator can not process logs.", "Ensure that plugin is not configured to receive logs from other plugins", ) } diff --git a/plugin/helper/input_test.go b/plugin/helper/input_test.go index 7480b5ef3..8a35fae2a 100644 --- a/plugin/helper/input_test.go +++ b/plugin/helper/input_test.go @@ -26,8 +26,8 @@ func TestInputConfigMissingOutput(t *testing.T) { config := InputConfig{ WriterConfig: WriterConfig{ BasicConfig: BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", }, }, WriteTo: entry.Field{}, @@ -42,8 +42,8 @@ func TestInputConfigValid(t *testing.T) { WriteTo: entry.Field{}, WriterConfig: WriterConfig{ BasicConfig: BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", }, OutputIDs: []string{"test-output"}, }, @@ -58,24 +58,24 @@ func TestInputConfigSetNamespace(t *testing.T) { WriteTo: entry.Field{}, WriterConfig: WriterConfig{ BasicConfig: BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", }, OutputIDs: []string{"test-output"}, }, } config.SetNamespace("test-namespace") - require.Equal(t, "test-namespace.test-id", config.PluginID) + require.Equal(t, "test-namespace.test-id", config.OperatorID) require.Equal(t, "test-namespace.test-output", config.OutputIDs[0]) } -func TestInputPluginCanProcess(t *testing.T) { +func TestInputOperatorCanProcess(t *testing.T) { buildContext := testutil.NewBuildContext(t) - input := InputPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + input := InputOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -83,13 +83,13 @@ func TestInputPluginCanProcess(t *testing.T) { require.False(t, input.CanProcess()) } -func TestInputPluginProcess(t *testing.T) { +func TestInputOperatorProcess(t *testing.T) { buildContext := testutil.NewBuildContext(t) - input := InputPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + input := InputOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -98,17 +98,17 @@ func TestInputPluginProcess(t *testing.T) { ctx := context.Background() err := input.Process(ctx, entry) require.Error(t, err) - require.Equal(t, err.Error(), "Plugin can not process logs.") + require.Equal(t, err.Error(), "Operator can not process logs.") } -func TestInputPluginNewEntry(t *testing.T) { +func TestInputOperatorNewEntry(t *testing.T) { buildContext := testutil.NewBuildContext(t) writeTo := entry.NewRecordField("test-field") - input := InputPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + input := InputOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, diff --git a/plugin/helper/output.go b/plugin/helper/output.go index f1333a75d..b35c6dacb 100644 --- a/plugin/helper/output.go +++ b/plugin/helper/output.go @@ -17,50 +17,50 @@ type OutputConfig struct { } // Build will build an output plugin. -func (c OutputConfig) Build(context plugin.BuildContext) (OutputPlugin, error) { - basicPlugin, err := c.BasicConfig.Build(context) +func (c OutputConfig) Build(context plugin.BuildContext) (OutputOperator, error) { + basicOperator, err := c.BasicConfig.Build(context) if err != nil { - return OutputPlugin{}, err + return OutputOperator{}, err } - outputPlugin := OutputPlugin{ - BasicPlugin: basicPlugin, + outputOperator := OutputOperator{ + BasicOperator: basicOperator, } - return outputPlugin, nil + return outputOperator, nil } // SetNamespace will namespace the id and output of the plugin config. func (c *OutputConfig) SetNamespace(namespace string, exclusions ...string) { if CanNamespace(c.ID(), exclusions) { - c.PluginID = AddNamespace(c.ID(), namespace) + c.OperatorID = AddNamespace(c.ID(), namespace) } } -// OutputPlugin provides a basic implementation of an output plugin. -type OutputPlugin struct { - BasicPlugin +// OutputOperator provides a basic implementation of an output plugin. +type OutputOperator struct { + BasicOperator } // CanProcess will always return true for an output plugin. -func (o *OutputPlugin) CanProcess() bool { +func (o *OutputOperator) CanProcess() bool { return true } // CanOutput will always return false for an output plugin. -func (o *OutputPlugin) CanOutput() bool { +func (o *OutputOperator) CanOutput() bool { return false } // Outputs will always return an empty array for an output plugin. -func (o *OutputPlugin) Outputs() []plugin.Plugin { - return []plugin.Plugin{} +func (o *OutputOperator) Outputs() []plugin.Operator { + return []plugin.Operator{} } // SetOutputs will return an error if called. -func (o *OutputPlugin) SetOutputs(plugins []plugin.Plugin) error { +func (o *OutputOperator) SetOutputs(plugins []plugin.Operator) error { return errors.NewError( - "Plugin can not output, but is attempting to set an output.", + "Operator can not output, but is attempting to set an output.", "This is an unexpected internal error. Please submit a bug/issue.", ) } diff --git a/plugin/helper/output_test.go b/plugin/helper/output_test.go index 643d3b81b..1e240f9a6 100644 --- a/plugin/helper/output_test.go +++ b/plugin/helper/output_test.go @@ -19,8 +19,8 @@ func TestOutputConfigMissingBase(t *testing.T) { func TestOutputConfigBuildValid(t *testing.T) { config := OutputConfig{ BasicConfig: BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", }, } context := testutil.NewBuildContext(t) @@ -31,61 +31,61 @@ func TestOutputConfigBuildValid(t *testing.T) { func TestOutputConfigNamespace(t *testing.T) { config := OutputConfig{ BasicConfig: BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", }, } config.SetNamespace("test-namespace") require.Equal(t, "test-namespace.test-id", config.ID()) } -func TestOutputPluginCanProcess(t *testing.T) { +func TestOutputOperatorCanProcess(t *testing.T) { buildContext := testutil.NewBuildContext(t) - output := OutputPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + output := OutputOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, } require.True(t, output.CanProcess()) } -func TestOutputPluginCanOutput(t *testing.T) { +func TestOutputOperatorCanOutput(t *testing.T) { buildContext := testutil.NewBuildContext(t) - output := OutputPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + output := OutputOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, } require.False(t, output.CanOutput()) } -func TestOutputPluginOutputs(t *testing.T) { +func TestOutputOperatorOutputs(t *testing.T) { buildContext := testutil.NewBuildContext(t) - output := OutputPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + output := OutputOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, } - require.Equal(t, []plugin.Plugin{}, output.Outputs()) + require.Equal(t, []plugin.Operator{}, output.Outputs()) } -func TestOutputPluginSetOutputs(t *testing.T) { +func TestOutputOperatorSetOutputs(t *testing.T) { buildContext := testutil.NewBuildContext(t) - output := OutputPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + output := OutputOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, } - err := output.SetOutputs([]plugin.Plugin{}) + err := output.SetOutputs([]plugin.Operator{}) require.Error(t, err) - require.Contains(t, err.Error(), "Plugin can not output") + require.Contains(t, err.Error(), "Operator can not output") } diff --git a/plugin/helper/parser.go b/plugin/helper/parser.go index bb0bada03..af1ef39c0 100644 --- a/plugin/helper/parser.go +++ b/plugin/helper/parser.go @@ -29,48 +29,48 @@ type ParserConfig struct { } // Build will build a parser plugin. -func (c ParserConfig) Build(context plugin.BuildContext) (ParserPlugin, error) { - transformerPlugin, err := c.TransformerConfig.Build(context) +func (c ParserConfig) Build(context plugin.BuildContext) (ParserOperator, error) { + transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { - return ParserPlugin{}, err + return ParserOperator{}, err } if c.ParseFrom.String() == c.ParseTo.String() && c.Preserve { - transformerPlugin.Warnw( + transformerOperator.Warnw( "preserve is true, but parse_to is set to the same field as parse_from, "+ "which will cause the original value to be overwritten", "plugin_id", c.ID(), ) } - parserPlugin := ParserPlugin{ - TransformerPlugin: transformerPlugin, - ParseFrom: c.ParseFrom, - ParseTo: c.ParseTo, - Preserve: c.Preserve, + parserOperator := ParserOperator{ + TransformerOperator: transformerOperator, + ParseFrom: c.ParseFrom, + ParseTo: c.ParseTo, + Preserve: c.Preserve, } if c.TimeParser != nil { if err := c.TimeParser.Validate(context); err != nil { - return ParserPlugin{}, err + return ParserOperator{}, err } - parserPlugin.TimeParser = c.TimeParser + parserOperator.TimeParser = c.TimeParser } if c.SeverityParserConfig != nil { severityParser, err := c.SeverityParserConfig.Build(context) if err != nil { - return ParserPlugin{}, err + return ParserOperator{}, err } - parserPlugin.SeverityParser = &severityParser + parserOperator.SeverityParser = &severityParser } - return parserPlugin, nil + return parserOperator, nil } -// ParserPlugin provides a basic implementation of a parser plugin. -type ParserPlugin struct { - TransformerPlugin +// ParserOperator provides a basic implementation of a parser plugin. +type ParserOperator struct { + TransformerOperator ParseFrom entry.Field ParseTo entry.Field Preserve bool @@ -79,7 +79,7 @@ type ParserPlugin struct { } // ProcessWith will process an entry with a parser function. -func (p *ParserPlugin) ProcessWith(ctx context.Context, entry *entry.Entry, parse ParseFunction) error { +func (p *ParserOperator) ProcessWith(ctx context.Context, entry *entry.Entry, parse ParseFunction) error { value, ok := entry.Get(p.ParseFrom) if !ok { err := errors.NewError( diff --git a/plugin/helper/parser_test.go b/plugin/helper/parser_test.go index b035f8283..98a3a7023 100644 --- a/plugin/helper/parser_test.go +++ b/plugin/helper/parser_test.go @@ -49,12 +49,12 @@ func TestParserConfigBuildValid(t *testing.T) { } func TestParserMissingField(t *testing.T) { - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: zaptest.NewLogger(t).Sugar(), }, }, @@ -74,12 +74,12 @@ func TestParserMissingField(t *testing.T) { func TestParserInvalidParse(t *testing.T) { buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -99,12 +99,12 @@ func TestParserInvalidParse(t *testing.T) { func TestParserInvalidTimeParse(t *testing.T) { buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -131,12 +131,12 @@ func TestParserInvalidTimeParse(t *testing.T) { func TestParserInvalidSeverityParse(t *testing.T) { buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -160,12 +160,12 @@ func TestParserInvalidSeverityParse(t *testing.T) { func TestParserInvalidTimeValidSeverityParse(t *testing.T) { buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -203,12 +203,12 @@ func TestParserInvalidTimeValidSeverityParse(t *testing.T) { func TestParserValidTimeInvalidSeverityParse(t *testing.T) { buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, }, @@ -246,20 +246,20 @@ func TestParserValidTimeInvalidSeverityParse(t *testing.T) { } func TestParserOutput(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ OnError: DropOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, + OutputOperators: []plugin.Operator{output}, }, }, ParseFrom: entry.NewRecordField(), @@ -276,20 +276,20 @@ func TestParserOutput(t *testing.T) { } func TestParserWithPreserve(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ OnError: DropOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, + OutputOperators: []plugin.Operator{output}, }, }, ParseFrom: entry.NewRecordField("parse_from"), @@ -315,20 +315,20 @@ func TestParserWithPreserve(t *testing.T) { } func TestParserWithoutPreserve(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - parser := ParserPlugin{ - TransformerPlugin: TransformerPlugin{ + parser := ParserOperator{ + TransformerOperator: TransformerOperator{ OnError: DropOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, + OutputOperators: []plugin.Operator{output}, }, }, ParseFrom: entry.NewRecordField("parse_from"), diff --git a/plugin/helper/plugin.go b/plugin/helper/plugin.go index 2cbf3d622..b11239b94 100644 --- a/plugin/helper/plugin.go +++ b/plugin/helper/plugin.go @@ -8,34 +8,34 @@ import ( func NewBasicConfig(pluginID, pluginType string) BasicConfig { return BasicConfig{ - PluginID: pluginID, - PluginType: pluginType, + OperatorID: pluginID, + OperatorType: pluginType, } } // BasicConfig provides a basic implemention for a plugin config. type BasicConfig struct { - PluginID string `json:"id" yaml:"id"` - PluginType string `json:"type" yaml:"type"` + OperatorID string `json:"id" yaml:"id"` + OperatorType string `json:"type" yaml:"type"` } // ID will return the plugin id. func (c BasicConfig) ID() string { - if c.PluginID == "" { - return c.PluginType + if c.OperatorID == "" { + return c.OperatorType } - return c.PluginID + return c.OperatorID } // Type will return the plugin type. func (c BasicConfig) Type() string { - return c.PluginType + return c.OperatorType } // Build will build a basic plugin. -func (c BasicConfig) Build(context plugin.BuildContext) (BasicPlugin, error) { - if c.PluginType == "" { - return BasicPlugin{}, errors.NewError( +func (c BasicConfig) Build(context plugin.BuildContext) (BasicOperator, error) { + if c.OperatorType == "" { + return BasicOperator{}, errors.NewError( "missing required `type` field.", "ensure that all plugins have a uniquely defined `type` field.", "plugin_id", c.ID(), @@ -43,7 +43,7 @@ func (c BasicConfig) Build(context plugin.BuildContext) (BasicPlugin, error) { } if context.Logger == nil { - return BasicPlugin{}, errors.NewError( + return BasicOperator{}, errors.NewError( "plugin build context is missing a logger.", "this is an unexpected internal error", "plugin_id", c.ID(), @@ -51,9 +51,9 @@ func (c BasicConfig) Build(context plugin.BuildContext) (BasicPlugin, error) { ) } - plugin := BasicPlugin{ - PluginID: c.ID(), - PluginType: c.Type(), + plugin := BasicOperator{ + OperatorID: c.ID(), + OperatorType: c.Type(), SugaredLogger: context.Logger.With("plugin_id", c.ID(), "plugin_type", c.Type()), } @@ -63,41 +63,41 @@ func (c BasicConfig) Build(context plugin.BuildContext) (BasicPlugin, error) { // SetNamespace will namespace the plugin id. func (c *BasicConfig) SetNamespace(namespace string, exclusions ...string) { if CanNamespace(c.ID(), exclusions) { - c.PluginID = AddNamespace(c.ID(), namespace) + c.OperatorID = AddNamespace(c.ID(), namespace) } } -// BasicPlugin provides a basic implementation of a plugin. -type BasicPlugin struct { - PluginID string - PluginType string +// BasicOperator provides a basic implementation of a plugin. +type BasicOperator struct { + OperatorID string + OperatorType string *zap.SugaredLogger } // ID will return the plugin id. -func (p *BasicPlugin) ID() string { - if p.PluginID == "" { - return p.PluginType +func (p *BasicOperator) ID() string { + if p.OperatorID == "" { + return p.OperatorType } - return p.PluginID + return p.OperatorID } // Type will return the plugin type. -func (p *BasicPlugin) Type() string { - return p.PluginType +func (p *BasicOperator) Type() string { + return p.OperatorType } // Logger returns the plugin's scoped logger. -func (p *BasicPlugin) Logger() *zap.SugaredLogger { +func (p *BasicOperator) Logger() *zap.SugaredLogger { return p.SugaredLogger } // Start will start the plugin. -func (p *BasicPlugin) Start() error { +func (p *BasicOperator) Start() error { return nil } // Stop will stop the plugin. -func (p *BasicPlugin) Stop() error { +func (p *BasicOperator) Stop() error { return nil } diff --git a/plugin/helper/plugin_test.go b/plugin/helper/plugin_test.go index 95c21ab81..d1b352821 100644 --- a/plugin/helper/plugin_test.go +++ b/plugin/helper/plugin_test.go @@ -11,23 +11,23 @@ import ( func TestBasicConfigID(t *testing.T) { config := BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", } require.Equal(t, "test-id", config.ID()) } func TestBasicConfigType(t *testing.T) { config := BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", } require.Equal(t, "test-type", config.Type()) } func TestBasicConfigBuildWithoutID(t *testing.T) { config := BasicConfig{ - PluginType: "test-type", + OperatorType: "test-type", } context := testutil.NewBuildContext(t) _, err := config.Build(context) @@ -36,7 +36,7 @@ func TestBasicConfigBuildWithoutID(t *testing.T) { func TestBasicConfigBuildWithoutType(t *testing.T) { config := BasicConfig{ - PluginID: "test-id", + OperatorID: "test-id", } context := plugin.BuildContext{} _, err := config.Build(context) @@ -46,8 +46,8 @@ func TestBasicConfigBuildWithoutType(t *testing.T) { func TestBasicConfigBuildMissingLogger(t *testing.T) { config := BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", } context := plugin.BuildContext{} _, err := config.Build(context) @@ -57,55 +57,55 @@ func TestBasicConfigBuildMissingLogger(t *testing.T) { func TestBasicConfigBuildValid(t *testing.T) { config := BasicConfig{ - PluginID: "test-id", - PluginType: "test-type", + OperatorID: "test-id", + OperatorType: "test-type", } context := testutil.NewBuildContext(t) plugin, err := config.Build(context) require.NoError(t, err) - require.Equal(t, "test-id", plugin.PluginID) - require.Equal(t, "test-type", plugin.PluginType) + require.Equal(t, "test-id", plugin.OperatorID) + require.Equal(t, "test-type", plugin.OperatorType) } -func TestBasicPluginID(t *testing.T) { - plugin := BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", +func TestBasicOperatorID(t *testing.T) { + plugin := BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", } require.Equal(t, "test-id", plugin.ID()) } -func TestBasicPluginType(t *testing.T) { - plugin := BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", +func TestBasicOperatorType(t *testing.T) { + plugin := BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", } require.Equal(t, "test-type", plugin.Type()) } -func TestBasicPluginLogger(t *testing.T) { +func TestBasicOperatorLogger(t *testing.T) { logger := &zap.SugaredLogger{} - plugin := BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + plugin := BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: logger, } require.Equal(t, logger, plugin.Logger()) } -func TestBasicPluginStart(t *testing.T) { - plugin := BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", +func TestBasicOperatorStart(t *testing.T) { + plugin := BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", } err := plugin.Start() require.NoError(t, err) } -func TestBasicPluginStop(t *testing.T) { - plugin := BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", +func TestBasicOperatorStop(t *testing.T) { + plugin := BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", } err := plugin.Stop() require.NoError(t, err) diff --git a/plugin/helper/transformer.go b/plugin/helper/transformer.go index b17ae5f8e..5713ac11c 100644 --- a/plugin/helper/transformer.go +++ b/plugin/helper/transformer.go @@ -23,43 +23,43 @@ type TransformerConfig struct { } // Build will build a transformer plugin. -func (c TransformerConfig) Build(context plugin.BuildContext) (TransformerPlugin, error) { - writerPlugin, err := c.WriterConfig.Build(context) +func (c TransformerConfig) Build(context plugin.BuildContext) (TransformerOperator, error) { + writerOperator, err := c.WriterConfig.Build(context) if err != nil { - return TransformerPlugin{}, errors.WithDetails(err, "plugin_id", c.ID()) + return TransformerOperator{}, errors.WithDetails(err, "plugin_id", c.ID()) } switch c.OnError { case SendOnError, DropOnError: default: - return TransformerPlugin{}, errors.NewError( + return TransformerOperator{}, errors.NewError( "plugin config has an invalid `on_error` field.", "ensure that the `on_error` field is set to either `send` or `drop`.", "on_error", c.OnError, ) } - transformerPlugin := TransformerPlugin{ - WriterPlugin: writerPlugin, - OnError: c.OnError, + transformerOperator := TransformerOperator{ + WriterOperator: writerOperator, + OnError: c.OnError, } - return transformerPlugin, nil + return transformerOperator, nil } -// TransformerPlugin provides a basic implementation of a transformer plugin. -type TransformerPlugin struct { - WriterPlugin +// TransformerOperator provides a basic implementation of a transformer plugin. +type TransformerOperator struct { + WriterOperator OnError string } // CanProcess will always return true for a transformer plugin. -func (t *TransformerPlugin) CanProcess() bool { +func (t *TransformerOperator) CanProcess() bool { return true } // ProcessWith will process an entry with a transform function. -func (t *TransformerPlugin) ProcessWith(ctx context.Context, entry *entry.Entry, transform TransformFunction) error { +func (t *TransformerOperator) ProcessWith(ctx context.Context, entry *entry.Entry, transform TransformFunction) error { newEntry, err := transform(entry) if err != nil { return t.HandleEntryError(ctx, entry, err) @@ -69,7 +69,7 @@ func (t *TransformerPlugin) ProcessWith(ctx context.Context, entry *entry.Entry, } // HandleEntryError will handle an entry error using the on_error strategy. -func (t *TransformerPlugin) HandleEntryError(ctx context.Context, entry *entry.Entry, err error) error { +func (t *TransformerOperator) HandleEntryError(ctx context.Context, entry *entry.Entry, err error) error { t.Errorw("Failed to process entry", zap.Any("error", err), zap.Any("action", t.OnError), zap.Any("entry", entry)) if t.OnError == SendOnError { t.Write(ctx, entry) diff --git a/plugin/helper/transformer_test.go b/plugin/helper/transformer_test.go index 447d0463a..ccf628a30 100644 --- a/plugin/helper/transformer_test.go +++ b/plugin/helper/transformer_test.go @@ -52,11 +52,11 @@ func TestTransformerConfigSetNamespace(t *testing.T) { cfg := NewTransformerConfig("test-id", "test-type") cfg.OutputIDs = []string{"test-output"} cfg.SetNamespace("test-namespace") - require.Equal(t, "test-namespace.test-id", cfg.PluginID) + require.Equal(t, "test-namespace.test-id", cfg.OperatorID) require.Equal(t, "test-namespace.test-output", cfg.OutputIDs[0]) } -func TestTransformerPluginCanProcess(t *testing.T) { +func TestTransformerOperatorCanProcess(t *testing.T) { cfg := NewTransformerConfig("test", "test") transformer, err := cfg.Build(testutil.NewBuildContext(t)) require.NoError(t, err) @@ -64,20 +64,20 @@ func TestTransformerPluginCanProcess(t *testing.T) { } func TestTransformerDropOnError(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - transformer := TransformerPlugin{ + transformer := TransformerOperator{ OnError: DropOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, - OutputIDs: []string{"test-output"}, + OutputOperators: []plugin.Operator{output}, + OutputIDs: []string{"test-output"}, }, } ctx := context.Background() @@ -92,20 +92,20 @@ func TestTransformerDropOnError(t *testing.T) { } func TestTransformerSendOnError(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - transformer := TransformerPlugin{ + transformer := TransformerOperator{ OnError: SendOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, - OutputIDs: []string{"test-output"}, + OutputOperators: []plugin.Operator{output}, + OutputIDs: []string{"test-output"}, }, } ctx := context.Background() @@ -120,20 +120,20 @@ func TestTransformerSendOnError(t *testing.T) { } func TestTransformerProcessWithValid(t *testing.T) { - output := &testutil.Plugin{} + output := &testutil.Operator{} output.On("ID").Return("test-output") output.On("Process", mock.Anything, mock.Anything).Return(nil) buildContext := testutil.NewBuildContext(t) - transformer := TransformerPlugin{ + transformer := TransformerOperator{ OnError: SendOnError, - WriterPlugin: WriterPlugin{ - BasicPlugin: BasicPlugin{ - PluginID: "test-id", - PluginType: "test-type", + WriterOperator: WriterOperator{ + BasicOperator: BasicOperator{ + OperatorID: "test-id", + OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputPlugins: []plugin.Plugin{output}, - OutputIDs: []string{"test-output"}, + OutputOperators: []plugin.Operator{output}, + OutputIDs: []string{"test-output"}, }, } ctx := context.Background() diff --git a/plugin/helper/writer.go b/plugin/helper/writer.go index 8d8215150..a79488cd6 100644 --- a/plugin/helper/writer.go +++ b/plugin/helper/writer.go @@ -22,15 +22,15 @@ type WriterConfig struct { } // Build will build a writer plugin from the config. -func (c WriterConfig) Build(context plugin.BuildContext) (WriterPlugin, error) { - basicPlugin, err := c.BasicConfig.Build(context) +func (c WriterConfig) Build(context plugin.BuildContext) (WriterOperator, error) { + basicOperator, err := c.BasicConfig.Build(context) if err != nil { - return WriterPlugin{}, err + return WriterOperator{}, err } - writer := WriterPlugin{ - OutputIDs: c.OutputIDs, - BasicPlugin: basicPlugin, + writer := WriterOperator{ + OutputIDs: c.OutputIDs, + BasicOperator: basicOperator, } return writer, nil } @@ -45,17 +45,17 @@ func (c *WriterConfig) SetNamespace(namespace string, exclusions ...string) { } } -// WriterPlugin is a plugin that can write to other plugins. -type WriterPlugin struct { - BasicPlugin - OutputIDs OutputIDs - OutputPlugins []plugin.Plugin +// WriterOperator is a plugin that can write to other plugins. +type WriterOperator struct { + BasicOperator + OutputIDs OutputIDs + OutputOperators []plugin.Operator } // Write will write an entry to the outputs of the plugin. -func (w *WriterPlugin) Write(ctx context.Context, e *entry.Entry) { - for i, plugin := range w.OutputPlugins { - if i == len(w.OutputPlugins)-1 { +func (w *WriterOperator) Write(ctx context.Context, e *entry.Entry) { + for i, plugin := range w.OutputOperators { + if i == len(w.OutputOperators)-1 { _ = plugin.Process(ctx, e) return } @@ -64,21 +64,21 @@ func (w *WriterPlugin) Write(ctx context.Context, e *entry.Entry) { } // CanOutput always returns true for a writer plugin. -func (w *WriterPlugin) CanOutput() bool { +func (w *WriterOperator) CanOutput() bool { return true } // Outputs returns the outputs of the writer plugin. -func (w *WriterPlugin) Outputs() []plugin.Plugin { - return w.OutputPlugins +func (w *WriterOperator) Outputs() []plugin.Operator { + return w.OutputOperators } // SetOutputs will set the outputs of the plugin. -func (w *WriterPlugin) SetOutputs(plugins []plugin.Plugin) error { - outputPlugins := make([]plugin.Plugin, 0) +func (w *WriterOperator) SetOutputs(plugins []plugin.Operator) error { + outputOperators := make([]plugin.Operator, 0) for _, pluginID := range w.OutputIDs { - plugin, ok := w.findPlugin(plugins, pluginID) + plugin, ok := w.findOperator(plugins, pluginID) if !ok { return fmt.Errorf("plugin '%s' does not exist", pluginID) } @@ -87,38 +87,38 @@ func (w *WriterPlugin) SetOutputs(plugins []plugin.Plugin) error { return fmt.Errorf("plugin '%s' can not process entries", pluginID) } - outputPlugins = append(outputPlugins, plugin) + outputOperators = append(outputOperators, plugin) } // No outputs have been set, so use the next configured plugin if len(w.OutputIDs) == 0 { - currentPluginIndex := -1 + currentOperatorIndex := -1 for i, plugin := range plugins { if plugin.ID() == w.ID() { - currentPluginIndex = i + currentOperatorIndex = i break } } - if currentPluginIndex == -1 { + if currentOperatorIndex == -1 { return fmt.Errorf("unexpectedly could not find self in array of plugins") } - nextPluginIndex := currentPluginIndex + 1 - if nextPluginIndex == len(plugins) { + nextOperatorIndex := currentOperatorIndex + 1 + if nextOperatorIndex == len(plugins) { return fmt.Errorf("cannot omit output for the last plugin in the pipeline") } - nextPlugin := plugins[nextPluginIndex] - if !nextPlugin.CanProcess() { - return fmt.Errorf("plugin '%s' cannot process entries, but it was selected as a receiver because 'output' was omitted", nextPlugin.ID()) + nextOperator := plugins[nextOperatorIndex] + if !nextOperator.CanProcess() { + return fmt.Errorf("plugin '%s' cannot process entries, but it was selected as a receiver because 'output' was omitted", nextOperator.ID()) } - outputPlugins = append(outputPlugins, nextPlugin) + outputOperators = append(outputOperators, nextOperator) } - w.OutputPlugins = outputPlugins + w.OutputOperators = outputOperators return nil } -// FindPlugin will find a plugin matching the supplied id. -func (w *WriterPlugin) findPlugin(plugins []plugin.Plugin, pluginID string) (plugin.Plugin, bool) { +// FindOperator will find a plugin matching the supplied id. +func (w *WriterOperator) findOperator(plugins []plugin.Operator, pluginID string) (plugin.Operator, bool) { for _, plugin := range plugins { if plugin.ID() == pluginID { return plugin, true diff --git a/plugin/helper/writer_test.go b/plugin/helper/writer_test.go index 8d5387292..a0a405909 100644 --- a/plugin/helper/writer_test.go +++ b/plugin/helper/writer_test.go @@ -16,7 +16,7 @@ import ( func TestWriterConfigMissingOutput(t *testing.T) { config := WriterConfig{ BasicConfig: BasicConfig{ - PluginType: "testtype", + OperatorType: "testtype", }, } context := testutil.NewBuildContext(t) @@ -28,7 +28,7 @@ func TestWriterConfigValidBuild(t *testing.T) { config := WriterConfig{ OutputIDs: OutputIDs{"output"}, BasicConfig: BasicConfig{ - PluginType: "testtype", + OperatorType: "testtype", }, } context := testutil.NewBuildContext(t) @@ -44,13 +44,13 @@ func TestWriterConfigSetNamespace(t *testing.T) { require.Equal(t, OutputIDs{"namespace.output1", "namespace.output2"}, config.OutputIDs) } -func TestWriterPluginWrite(t *testing.T) { - output1 := &testutil.Plugin{} +func TestWriterOperatorWrite(t *testing.T) { + output1 := &testutil.Operator{} output1.On("Process", mock.Anything, mock.Anything).Return(nil) - output2 := &testutil.Plugin{} + output2 := &testutil.Operator{} output2.On("Process", mock.Anything, mock.Anything).Return(nil) - writer := WriterPlugin{ - OutputPlugins: []plugin.Plugin{output1, output2}, + writer := WriterOperator{ + OutputOperators: []plugin.Operator{output1, output2}, } ctx := context.Background() @@ -61,18 +61,18 @@ func TestWriterPluginWrite(t *testing.T) { output2.AssertCalled(t, "Process", ctx, mock.Anything) } -func TestWriterPluginCanOutput(t *testing.T) { - writer := WriterPlugin{} +func TestWriterOperatorCanOutput(t *testing.T) { + writer := WriterOperator{} require.True(t, writer.CanOutput()) } -func TestWriterPluginOutputs(t *testing.T) { - output1 := &testutil.Plugin{} +func TestWriterOperatorOutputs(t *testing.T) { + output1 := &testutil.Operator{} output1.On("Process", mock.Anything, mock.Anything).Return(nil) - output2 := &testutil.Plugin{} + output2 := &testutil.Operator{} output2.On("Process", mock.Anything, mock.Anything).Return(nil) - writer := WriterPlugin{ - OutputPlugins: []plugin.Plugin{output1, output2}, + writer := WriterOperator{ + OutputOperators: []plugin.Operator{output1, output2}, } ctx := context.Background() @@ -84,44 +84,44 @@ func TestWriterPluginOutputs(t *testing.T) { } func TestWriterSetOutputsMissing(t *testing.T) { - output1 := &testutil.Plugin{} + output1 := &testutil.Operator{} output1.On("ID").Return("output1") - writer := WriterPlugin{ + writer := WriterOperator{ OutputIDs: OutputIDs{"output2"}, } - err := writer.SetOutputs([]plugin.Plugin{output1}) + err := writer.SetOutputs([]plugin.Operator{output1}) require.Error(t, err) require.Contains(t, err.Error(), "does not exist") } func TestWriterSetOutputsInvalid(t *testing.T) { - output1 := &testutil.Plugin{} + output1 := &testutil.Operator{} output1.On("ID").Return("output1") output1.On("CanProcess").Return(false) - writer := WriterPlugin{ + writer := WriterOperator{ OutputIDs: OutputIDs{"output1"}, } - err := writer.SetOutputs([]plugin.Plugin{output1}) + err := writer.SetOutputs([]plugin.Operator{output1}) require.Error(t, err) require.Contains(t, err.Error(), "can not process entries") } func TestWriterSetOutputsValid(t *testing.T) { - output1 := &testutil.Plugin{} + output1 := &testutil.Operator{} output1.On("ID").Return("output1") output1.On("CanProcess").Return(true) - output2 := &testutil.Plugin{} + output2 := &testutil.Operator{} output2.On("ID").Return("output2") output2.On("CanProcess").Return(true) - writer := WriterPlugin{ + writer := WriterOperator{ OutputIDs: OutputIDs{"output1", "output2"}, } - err := writer.SetOutputs([]plugin.Plugin{output1, output2}) + err := writer.SetOutputs([]plugin.Operator{output1, output2}) require.NoError(t, err) - require.Equal(t, []plugin.Plugin{output1, output2}, writer.Outputs()) + require.Equal(t, []plugin.Operator{output1, output2}, writer.Outputs()) } func TestUnmarshalJSONString(t *testing.T) { diff --git a/plugin/plugin.go b/plugin/plugin.go index ead61d44a..f1acdb9af 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -1,4 +1,4 @@ -//go:generate mockery -name=^(Plugin)$ -output=../internal/testutil -outpkg=testutil -case=snake +//go:generate mockery -name=^(Operator)$ -output=../internal/testutil -outpkg=testutil -case=snake package plugin @@ -9,29 +9,29 @@ import ( "go.uber.org/zap" ) -// Plugin is a log monitoring component. -type Plugin interface { - // ID returns the id of the plugin. +// Operator is a log monitoring component. +type Operator interface { + // ID returns the id of the operator. ID() string - // Type returns the type of the plugin. + // Type returns the type of the operator. Type() string - // Start will start the plugin. + // Start will start the operator. Start() error - // Stop will stop the plugin. + // Stop will stop the operator. Stop() error - // CanOutput indicates if the plugin will output entries to other plugins. + // CanOutput indicates if the operator will output entries to other operators. CanOutput() bool // Outputs returns the list of connected outputs. - Outputs() []Plugin + Outputs() []Operator // SetOutputs will set the connected outputs. - SetOutputs([]Plugin) error + SetOutputs([]Operator) error - // CanProcess indicates if the plugin will process entries from other plugins. + // CanProcess indicates if the operator will process entries from other operators. CanProcess() bool - // Process will process an entry from a plugin. + // Process will process an entry from a operator. Process(context.Context, *entry.Entry) error - // Logger returns the plugin's logger + // Logger returns the operator's logger Logger() *zap.SugaredLogger } From 06537f99fc90686649f249995983fc7456084daf Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Wed, 22 Jul 2020 11:45:06 -0400 Subject: [PATCH 2/7] wip - renamed plugin package to operator --- CONTRIBUTING.md | 38 ++--- agent/agent.go | 20 +-- commands/example_test.go | 2 +- commands/graph.go | 12 +- commands/offsets.go | 16 +-- commands/offsets_test.go | 16 +-- internal/testutil/mocks.go | 2 +- internal/testutil/plugin.go | 14 +- internal/testutil/util.go | 6 +- operator/buffer/buffer.go | 76 ++++++++++ {plugin => operator}/buffer/buffer_test.go | 4 +- {plugin => operator}/buffer/memory_buffer.go | 0 .../buffer/memory_buffer_test.go | 14 +- operator/builtin/builtin.go | 9 ++ {plugin => operator}/builtin/builtin_test.go | 0 .../builtin/input/file/file.go | 38 ++--- .../builtin/input/file/file_test.go | 12 +- .../builtin/input/file/line_splitter.go | 0 .../builtin/input/file/line_splitter_test.go | 0 .../builtin/input/file/read_to_end.go | 4 +- .../builtin/input/generate.go | 18 +-- .../builtin/input/generate_test.go | 14 +- operator/builtin/input/input.go | 6 + .../builtin/input/journald.go | 70 +++++----- .../builtin/input/journald_test.go | 6 +- {plugin => operator}/builtin/input/tcp.go | 18 +-- .../builtin/input/tcp_test.go | 6 +- {plugin => operator}/builtin/input/udp.go | 18 +-- .../builtin/input/udp_test.go | 6 +- operator/builtin/output/drop.go | 48 +++++++ .../builtin/output/drop_test.go | 2 +- .../builtin/output/elastic.go | 20 +-- .../builtin/output/elastic_test.go | 0 {plugin => operator}/builtin/output/file.go | 18 +-- .../builtin/output/google_cloud.go | 36 ++--- .../builtin/output/google_cloud_test.go | 4 +- {plugin => operator}/builtin/output/stdout.go | 18 +-- .../builtin/output/stdout_test.go | 10 +- {plugin => operator}/builtin/parser/json.go | 18 +-- .../builtin/parser/json_test.go | 8 +- {plugin => operator}/builtin/parser/regex.go | 18 +-- .../builtin/parser/regex_test.go | 6 +- .../builtin/parser/severity.go | 18 +-- .../builtin/parser/severity_test.go | 12 +- {plugin => operator}/builtin/parser/syslog.go | 18 +-- .../builtin/parser/syslog_test.go | 6 +- {plugin => operator}/builtin/parser/time.go | 20 +-- .../builtin/parser/time_test.go | 12 +- .../transformer/k8s_metadata_decorator.go | 32 ++--- .../k8s_metadata_decorator_test.go | 16 +-- .../builtin/transformer/metadata.go | 18 +-- .../builtin/transformer/metadata_test.go | 8 +- .../builtin/transformer/noop.go | 18 +-- .../builtin/transformer/noop_test.go | 2 +- .../builtin/transformer/rate_limit.go | 28 ++-- .../builtin/transformer/rate_limit_test.go | 4 +- .../builtin/transformer/restructure.go | 18 +-- .../builtin/transformer/restructure_test.go | 18 +-- .../builtin/transformer/router.go | 66 ++++----- .../builtin/transformer/router_test.go | 8 +- {plugin => operator}/config.go | 2 +- {plugin => operator}/config_test.go | 2 +- {plugin => operator}/custom.go | 2 +- {plugin => operator}/custom_test.go | 2 +- {plugin => operator}/duration.go | 2 +- {plugin => operator}/duration_test.go | 2 +- {plugin => operator}/helper/expr_string.go | 0 .../helper/expr_string_test.go | 0 {plugin => operator}/helper/input.go | 18 +-- {plugin => operator}/helper/input_test.go | 0 {plugin => operator}/helper/namespace.go | 0 {plugin => operator}/helper/output.go | 28 ++-- {plugin => operator}/helper/output_test.go | 6 +- {plugin => operator}/helper/parser.go | 14 +- {plugin => operator}/helper/parser_test.go | 8 +- {plugin => operator}/helper/persister.go | 6 +- {plugin => operator}/helper/plugin.go | 48 +++---- {plugin => operator}/helper/plugin_test.go | 34 ++--- {plugin => operator}/helper/severity.go | 0 .../helper/severity_builder.go | 12 +- {plugin => operator}/helper/time.go | 6 +- {plugin => operator}/helper/time_test.go | 0 {plugin => operator}/helper/transformer.go | 18 +-- .../helper/transformer_test.go | 10 +- {plugin => operator}/helper/writer.go | 76 +++++----- {plugin => operator}/helper/writer_test.go | 14 +- plugin/plugin.go => operator/operator.go | 2 +- pipeline/config.go | 68 ++++----- pipeline/config_test.go | 62 ++++---- pipeline/node.go | 34 ++--- pipeline/pipeline.go | 90 ++++++------ pipeline/pipeline_test.go | 132 +++++++++--------- plugin/buffer/buffer.go | 76 ---------- plugin/builtin/builtin.go | 9 -- plugin/builtin/input/input.go | 6 - plugin/builtin/output/drop.go | 48 ------- 96 files changed, 905 insertions(+), 905 deletions(-) create mode 100644 operator/buffer/buffer.go rename {plugin => operator}/buffer/buffer_test.go (95%) rename {plugin => operator}/buffer/memory_buffer.go (100%) rename {plugin => operator}/buffer/memory_buffer_test.go (90%) create mode 100644 operator/builtin/builtin.go rename {plugin => operator}/builtin/builtin_test.go (100%) rename {plugin => operator}/builtin/input/file/file.go (91%) rename {plugin => operator}/builtin/input/file/file_test.go (98%) rename {plugin => operator}/builtin/input/file/line_splitter.go (100%) rename {plugin => operator}/builtin/input/file/line_splitter_test.go (100%) rename {plugin => operator}/builtin/input/file/read_to_end.go (97%) rename {plugin => operator}/builtin/input/generate.go (80%) rename {plugin => operator}/builtin/input/generate_test.go (87%) create mode 100644 operator/builtin/input/input.go rename {plugin => operator}/builtin/input/journald.go (68%) rename {plugin => operator}/builtin/input/journald_test.go (96%) rename {plugin => operator}/builtin/input/tcp.go (85%) rename {plugin => operator}/builtin/input/tcp_test.go (90%) rename {plugin => operator}/builtin/input/udp.go (80%) rename {plugin => operator}/builtin/input/udp_test.go (90%) create mode 100644 operator/builtin/output/drop.go rename {plugin => operator}/builtin/output/drop_test.go (84%) rename {plugin => operator}/builtin/output/elastic.go (88%) rename {plugin => operator}/builtin/output/elastic_test.go (100%) rename {plugin => operator}/builtin/output/file.go (75%) rename {plugin => operator}/builtin/output/google_cloud.go (88%) rename {plugin => operator}/builtin/output/google_cloud_test.go (98%) rename {plugin => operator}/builtin/output/stdout.go (63%) rename {plugin => operator}/builtin/output/stdout_test.go (75%) rename {plugin => operator}/builtin/parser/json.go (68%) rename {plugin => operator}/builtin/parser/json_test.go (94%) rename {plugin => operator}/builtin/parser/regex.go (79%) rename {plugin => operator}/builtin/parser/regex_test.go (95%) rename {plugin => operator}/builtin/parser/severity.go (66%) rename {plugin => operator}/builtin/parser/severity_test.go (96%) rename {plugin => operator}/builtin/parser/syslog.go (88%) rename {plugin => operator}/builtin/parser/syslog_test.go (95%) rename {plugin => operator}/builtin/parser/time.go (61%) rename {plugin => operator}/builtin/parser/time_test.go (97%) rename {plugin => operator}/builtin/transformer/k8s_metadata_decorator.go (84%) rename {plugin => operator}/builtin/transformer/k8s_metadata_decorator_test.go (87%) rename {plugin => operator}/builtin/transformer/metadata.go (81%) rename {plugin => operator}/builtin/transformer/metadata_test.go (94%) rename {plugin => operator}/builtin/transformer/noop.go (53%) rename {plugin => operator}/builtin/transformer/noop_test.go (85%) rename {plugin => operator}/builtin/transformer/rate_limit.go (65%) rename {plugin => operator}/builtin/transformer/rate_limit_test.go (92%) rename {plugin => operator}/builtin/transformer/restructure.go (94%) rename {plugin => operator}/builtin/transformer/restructure_test.go (95%) rename {plugin => operator}/builtin/transformer/router.go (59%) rename {plugin => operator}/builtin/transformer/router_test.go (92%) rename {plugin => operator}/config.go (99%) rename {plugin => operator}/config_test.go (99%) rename {plugin => operator}/custom.go (99%) rename {plugin => operator}/custom_test.go (99%) rename {plugin => operator}/duration.go (98%) rename {plugin => operator}/duration_test.go (98%) rename {plugin => operator}/helper/expr_string.go (100%) rename {plugin => operator}/helper/expr_string_test.go (100%) rename {plugin => operator}/helper/input.go (65%) rename {plugin => operator}/helper/input_test.go (100%) rename {plugin => operator}/helper/namespace.go (100%) rename {plugin => operator}/helper/output.go (51%) rename {plugin => operator}/helper/output_test.go (93%) rename {plugin => operator}/helper/parser.go (89%) rename {plugin => operator}/helper/parser_test.go (98%) rename {plugin => operator}/helper/persister.go (93%) rename {plugin => operator}/helper/plugin.go (56%) rename {plugin => operator}/helper/plugin_test.go (74%) rename {plugin => operator}/helper/severity.go (100%) rename {plugin => operator}/helper/severity_builder.go (94%) rename {plugin => operator}/helper/time.go (97%) rename {plugin => operator}/helper/time_test.go (100%) rename {plugin => operator}/helper/transformer.go (79%) rename {plugin => operator}/helper/transformer_test.go (94%) rename {plugin => operator}/helper/writer.go (58%) rename {plugin => operator}/helper/writer_test.go (93%) rename plugin/plugin.go => operator/operator.go (98%) delete mode 100644 plugin/buffer/buffer.go delete mode 100644 plugin/builtin/builtin.go delete mode 100644 plugin/builtin/input/input.go delete mode 100644 plugin/builtin/output/drop.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35cefaf20..0504430d9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,57 +68,57 @@ A PR is considered to be **ready to merge** when: ## Design Choices -Best practices for developing a builtin plugin are documented below, but for changes to +Best practices for developing a builtin operators are documented below, but for changes to the core agent, we are happy to discuss proposals in the issue tracker. -### Builtin Plugin Development +### Builtin Operator Development -In order to write a builtin plugin, follow these three steps: -1. Build a unique struct that satisfies the [`Plugin`](plugin/plugin.go) interface. This struct will define what your plugin does when executed in the pipeline. +In order to write a builtin operator, follow these three steps: +1. Build a unique struct that satisfies the [`Operator`](operator/operator.go) interface. This struct will define what your operator does when executed in the pipeline. ```go -type ExamplePlugin struct { +type ExampleOperator struct { FilePath string } -func (p *ExamplePlugin) Process(ctx context.Context, entry *entry.Entry) error { +func (p *ExampleOperator) Process(ctx context.Context, entry *entry.Entry) error { // Processing logic } ``` -2. Build a unique config struct that satisfies the [`Config`](plugin/config.go) interface. This struct will define the parameters used to configure and build your plugin struct in step 1. +2. Build a unique config struct that satisfies the [`Config`](operator/config.go) interface. This struct will define the parameters used to configure and build your operator struct in step 1. ```go -type ExamplePluginConfig struct { +type ExampleOperatorConfig struct { filePath string } -func (c ExamplePluginConfig) Build(context plugin.BuildContext) (plugin.Plugin, error) { - return &ExamplePlugin{ +func (c ExampleOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { + return &ExampleOperator{ filePath: c.FilePath, }, nil } ``` -3. Register your config struct in the plugin registry using an `init()` hook. This will ensure that the agent knows about your plugin at runtime and can build it from a YAML config. +3. Register your config struct in the operator registry using an `init()` hook. This will ensure that the agent knows about your operator at runtime and can build it from a YAML config. ```go func init() { - plugin.Register("example_plugin", &ExamplePluginConfig{}) + operator.Register("example_operator", &ExampleOperatorConfig{}) } ``` -## Any tips for building plugins? -We highly recommend that developers take advantage of [helpers](plugin/helper) when building their plugins. Helpers are structs that help satisfy common behavior shared across many plugins. By embedding these structs, you can skip having to satisfy certain aspects of the `plugin` and `config` interfaces. +## Any tips for building operators? +We highly recommend that developers take advantage of [helpers](operator/helper) when building their operators. Helpers are structs that help satisfy common behavior shared across many operators. By embedding these structs, you can skip having to satisfy certain aspects of the `operator` and `config` interfaces. -For example, almost all plugins should embed the [BasicPlugin](plugin/helper/basic_plugin.go) helper, as it provides simple functionality for returning a plugin id and plugin type. +For example, almost all operators should embed the [BasicOperator](operator/helper/basic_operator.go) helper, as it provides simple functionality for returning a operator id and operator type. ```go -// ExamplePlugin is a basic plugin, with a basic lifecycle, that consumes -// but doesn't send log entries. Rather than implementing every part of the plugin +// ExampleOperator is a basic operator, with a basic lifecycle, that consumes +// but doesn't send log entries. Rather than implementing every part of the operator // interface, we can embed the following helpers to achieve this effect. -type ExamplePlugin struct { - helper.BasicPlugin +type ExampleOperator struct { + helper.BasicOperator helper.BasicLifecycle helper.BasicOutput } diff --git a/agent/agent.go b/agent/agent.go index 077086e1e..2b9e15b13 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -7,9 +7,9 @@ import ( "time" "github.com/observiq/carbon/errors" + "github.com/observiq/carbon/operator" + _ "github.com/observiq/carbon/operator/builtin" // register operators "github.com/observiq/carbon/pipeline" - pg "github.com/observiq/carbon/plugin" - _ "github.com/observiq/carbon/plugin/builtin" // register plugins "go.etcd.io/bbolt" "go.uber.org/zap" ) @@ -21,7 +21,7 @@ type LogAgent struct { Database string *zap.SugaredLogger - database pg.Database + database operator.Database pipeline *pipeline.Pipeline running bool } @@ -39,12 +39,12 @@ func (a *LogAgent) Start() error { } a.database = database - registry, err := pg.NewCustomRegistry(a.PluginDir) + registry, err := operator.NewCustomRegistry(a.PluginDir) if err != nil { - a.Errorw("Failed to load custom plugin registry", zap.Any("error", err)) + a.Errorw("Failed to load custom operator registry", zap.Any("error", err)) } - buildContext := pg.BuildContext{ + buildContext := operator.BuildContext{ CustomRegistry: registry, Logger: a.SugaredLogger, Database: a.database, @@ -83,9 +83,9 @@ func (a *LogAgent) Stop() { } // OpenDatabase will open and create a database. -func OpenDatabase(file string) (pg.Database, error) { +func OpenDatabase(file string) (operator.Database, error) { if file == "" { - return pg.NewStubDatabase(), nil + return operator.NewStubDatabase(), nil } if _, err := os.Stat(filepath.Dir(file)); err != nil { @@ -104,11 +104,11 @@ func OpenDatabase(file string) (pg.Database, error) { } // NewLogAgent creates a new carbon log agent. -func NewLogAgent(cfg *Config, logger *zap.SugaredLogger, pluginDir, databaseFile string) *LogAgent { +func NewLogAgent(cfg *Config, logger *zap.SugaredLogger, operatorDir, databaseFile string) *LogAgent { return &LogAgent{ Config: cfg, SugaredLogger: logger, - PluginDir: pluginDir, + PluginDir: operatorDir, Database: databaseFile, } } diff --git a/commands/example_test.go b/commands/example_test.go index 954709557..5e6206fb9 100644 --- a/commands/example_test.go +++ b/commands/example_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/observiq/carbon/plugin/builtin/output" + "github.com/observiq/carbon/operator/builtin/output" "github.com/stretchr/testify/require" ) diff --git a/commands/graph.go b/commands/graph.go index ee476296a..4e2845101 100644 --- a/commands/graph.go +++ b/commands/graph.go @@ -4,8 +4,8 @@ import ( "os" "github.com/observiq/carbon/agent" - "github.com/observiq/carbon/plugin" - pg "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" + pg "github.com/observiq/carbon/operator" "github.com/spf13/cobra" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -21,7 +21,7 @@ func NewGraphCommand(rootFlags *RootFlags) *cobra.Command { return &cobra.Command{ Use: "graph", Args: cobra.NoArgs, - Short: "Export a dot-formatted representation of the plugin graph", + Short: "Export a dot-formatted representation of the operator graph", Run: func(command *cobra.Command, args []string) { runGraph(command, args, rootFlags) }, } } @@ -43,9 +43,9 @@ func runGraph(_ *cobra.Command, _ []string, flags *RootFlags) { os.Exit(1) } - customRegistry, err := plugin.NewCustomRegistry(flags.PluginDir) + customRegistry, err := operator.NewCustomRegistry(flags.PluginDir) if err != nil { - logger.Errorw("Failed to load custom plugin registry", zap.Any("error", err)) + logger.Errorw("Failed to load custom operator registry", zap.Any("error", err)) } buildContext := pg.BuildContext{ @@ -55,7 +55,7 @@ func runGraph(_ *cobra.Command, _ []string, flags *RootFlags) { pipeline, err := cfg.Pipeline.BuildPipeline(buildContext) if err != nil { - logger.Errorw("Failed to build plugin pipeline", zap.Any("error", err)) + logger.Errorw("Failed to build operator pipeline", zap.Any("error", err)) os.Exit(1) } diff --git a/commands/offsets.go b/commands/offsets.go index 11582519b..3dc9d24a7 100644 --- a/commands/offsets.go +++ b/commands/offsets.go @@ -6,7 +6,7 @@ import ( "os" agent "github.com/observiq/carbon/agent" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator/helper" "github.com/spf13/cobra" "go.etcd.io/bbolt" ) @@ -17,7 +17,7 @@ var stdout io.Writer = os.Stdout func NewOffsetsCmd(rootFlags *RootFlags) *cobra.Command { offsets := &cobra.Command{ Use: "offsets", - Short: "Manage input plugin offsets", + Short: "Manage input operator offsets", Args: cobra.NoArgs, Run: func(command *cobra.Command, args []string) { stdout.Write([]byte("No offsets subcommand specified. See `carbon offsets help` for details\n")) @@ -35,7 +35,7 @@ func NewOffsetsClearCmd(rootFlags *RootFlags) *cobra.Command { var all bool offsetsClear := &cobra.Command{ - Use: "clear [flags] [plugin_ids]", + Use: "clear [flags] [operator_ids]", Short: "Clear persisted offsets from the database", Args: cobra.ArbitraryArgs, Run: func(command *cobra.Command, args []string) { @@ -46,7 +46,7 @@ func NewOffsetsClearCmd(rootFlags *RootFlags) *cobra.Command { if all { if len(args) != 0 { - stdout.Write([]byte("Providing a list of plugin IDs does nothing with the --all flag\n")) + stdout.Write([]byte("Providing a list of operator IDs does nothing with the --all flag\n")) } err := db.Update(func(tx *bbolt.Tx) error { @@ -59,18 +59,18 @@ func NewOffsetsClearCmd(rootFlags *RootFlags) *cobra.Command { exitOnErr("Failed to delete offsets", err) } else { if len(args) == 0 { - stdout.Write([]byte("Must either specify a list of plugins or the --all flag\n")) + stdout.Write([]byte("Must either specify a list of operators or the --all flag\n")) os.Exit(1) } - for _, pluginID := range args { + for _, operatorID := range args { err = db.Update(func(tx *bbolt.Tx) error { offsetBucket := tx.Bucket(helper.OffsetsBucket) if offsetBucket == nil { return nil } - return offsetBucket.DeleteBucket([]byte(pluginID)) + return offsetBucket.DeleteBucket([]byte(operatorID)) }) exitOnErr("Failed to delete offsets", err) } @@ -87,7 +87,7 @@ func NewOffsetsClearCmd(rootFlags *RootFlags) *cobra.Command { func NewOffsetsListCmd(rootFlags *RootFlags) *cobra.Command { offsetsList := &cobra.Command{ Use: "list", - Short: "List plugins with persisted offsets", + Short: "List operators with persisted offsets", Args: cobra.NoArgs, Run: func(command *cobra.Command, args []string) { db, err := agent.OpenDatabase(rootFlags.DatabaseFile) diff --git a/commands/offsets_test.go b/commands/offsets_test.go index a84a12e09..d848abc09 100644 --- a/commands/offsets_test.go +++ b/commands/offsets_test.go @@ -8,7 +8,7 @@ import ( "testing" agent "github.com/observiq/carbon/agent" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" ) @@ -33,15 +33,15 @@ func TestOffsets(t *testing.T) { bucket, err := tx.CreateBucketIfNotExists(helper.OffsetsBucket) require.NoError(t, err) - _, err = bucket.CreateBucket([]byte("$.testpluginid1")) + _, err = bucket.CreateBucket([]byte("$.testoperatorid1")) require.NoError(t, err) - _, err = bucket.CreateBucket([]byte("$.testpluginid2")) + _, err = bucket.CreateBucket([]byte("$.testoperatorid2")) require.NoError(t, err) return nil }) db.Close() - // check that offsets list actually lists the plugin + // check that offsets list actually lists the operator offsetsList := NewRootCmd() offsetsList.SetArgs([]string{ "offsets", "list", @@ -51,7 +51,7 @@ func TestOffsets(t *testing.T) { err = offsetsList.Execute() require.NoError(t, err) - require.Equal(t, "$.testpluginid1\n$.testpluginid2\n", buf.String()) + require.Equal(t, "$.testoperatorid1\n$.testoperatorid2\n", buf.String()) // clear the offsets offsetsClear := NewRootCmd() @@ -59,16 +59,16 @@ func TestOffsets(t *testing.T) { "offsets", "clear", "--database", databasePath, "--config", configPath, - "$.testpluginid2", + "$.testoperatorid2", }) err = offsetsClear.Execute() require.NoError(t, err) - // Check that offsets list only shows uncleared plugin id + // Check that offsets list only shows uncleared operator id buf.Reset() err = offsetsList.Execute() require.NoError(t, err) - require.Equal(t, "$.testpluginid1\n", buf.String()) + require.Equal(t, "$.testoperatorid1\n", buf.String()) } diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index 1e7558c1e..644375884 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -1,6 +1,6 @@ package testutil -// NewMockOperator will return a basic plugin mock +// NewMockOperator will return a basic operator mock func NewMockOperator(id string) *Operator { mockOutput := &Operator{} mockOutput.On("ID").Return(id) diff --git a/internal/testutil/plugin.go b/internal/testutil/plugin.go index 69a32274f..9b3c321b2 100644 --- a/internal/testutil/plugin.go +++ b/internal/testutil/plugin.go @@ -8,7 +8,7 @@ import ( entry "github.com/observiq/carbon/entry" mock "github.com/stretchr/testify/mock" - plugin "github.com/observiq/carbon/plugin" + operator "github.com/observiq/carbon/operator" zap "go.uber.org/zap" ) @@ -77,15 +77,15 @@ func (_m *Operator) Logger() *zap.SugaredLogger { } // Outputs provides a mock function with given fields: -func (_m *Operator) Outputs() []plugin.Operator { +func (_m *Operator) Outputs() []operator.Operator { ret := _m.Called() - var r0 []plugin.Operator - if rf, ok := ret.Get(0).(func() []plugin.Operator); ok { + var r0 []operator.Operator + if rf, ok := ret.Get(0).(func() []operator.Operator); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]plugin.Operator) + r0 = ret.Get(0).([]operator.Operator) } } @@ -107,11 +107,11 @@ func (_m *Operator) Process(_a0 context.Context, _a1 *entry.Entry) error { } // SetOutputs provides a mock function with given fields: _a0 -func (_m *Operator) SetOutputs(_a0 []plugin.Operator) error { +func (_m *Operator) SetOutputs(_a0 []operator.Operator) error { ret := _m.Called(_a0) var r0 error - if rf, ok := ret.Get(0).(func([]plugin.Operator) error); ok { + if rf, ok := ret.Get(0).(func([]operator.Operator) error); ok { r0 = rf(_a0) } else { r0 = ret.Error(0) diff --git a/internal/testutil/util.go b/internal/testutil/util.go index 5f91c07e6..f7dfb1386 100644 --- a/internal/testutil/util.go +++ b/internal/testutil/util.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "go.etcd.io/bbolt" "go.uber.org/zap/zaptest" ) @@ -50,8 +50,8 @@ func NewTestDatabase(t *testing.T) *bbolt.DB { } // NewBuildContext will return a new build context for testing -func NewBuildContext(t *testing.T) plugin.BuildContext { - return plugin.BuildContext{ +func NewBuildContext(t *testing.T) operator.BuildContext { + return operator.BuildContext{ Database: NewTestDatabase(t), Logger: zaptest.NewLogger(t).Sugar(), } diff --git a/operator/buffer/buffer.go b/operator/buffer/buffer.go new file mode 100644 index 000000000..28d1926bb --- /dev/null +++ b/operator/buffer/buffer.go @@ -0,0 +1,76 @@ +package buffer + +import ( + "context" + "fmt" + "time" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/errors" + "github.com/observiq/carbon/operator" +) + +// Buffer is an entity that buffers log entries to a operator +type Buffer interface { + Flush(context.Context) error + Add(interface{}, int) error + AddWait(context.Context, interface{}, int) error + SetHandler(BundleHandler) + Process(context.Context, *entry.Entry) error +} + +func NewConfig() Config { + return Config{ + BufferType: "memory", + DelayThreshold: operator.Duration{Duration: time.Second}, + BundleCountThreshold: 10_000, + BundleByteThreshold: 4 * 1024 * 1024 * 1024, // 4MB + BundleByteLimit: 4 * 1024 * 1024 * 1024, // 4MB + BufferedByteLimit: 500 * 1024 * 1024 * 1024, // 500MB + HandlerLimit: 32, + Retry: NewRetryConfig(), + } +} + +// Config is the configuration of a buffer +type Config struct { + BufferType string `json:"type,omitempty" yaml:"type,omitempty"` + DelayThreshold operator.Duration `json:"delay_threshold,omitempty" yaml:"delay_threshold,omitempty"` + BundleCountThreshold int `json:"bundle_count_threshold,omitempty" yaml:"buffer_count_threshold,omitempty"` + BundleByteThreshold int `json:"bundle_byte_threshold,omitempty" yaml:"bundle_byte_threshold,omitempty"` + BundleByteLimit int `json:"bundle_byte_limit,omitempty" yaml:"bundle_byte_limit,omitempty"` + BufferedByteLimit int `json:"buffered_byte_limit,omitempty" yaml:"buffered_byte_limit,omitempty"` + HandlerLimit int `json:"handler_limit,omitempty" yaml:"handler_limit,omitempty"` + Retry RetryConfig `json:"retry,omitempty" yaml:"retry,omitempty"` +} + +// Build will build a buffer from the supplied configuration +func (config *Config) Build() (Buffer, error) { + switch config.BufferType { + case "memory", "": + return NewMemoryBuffer(config), nil + default: + return nil, errors.NewError( + fmt.Sprintf("Invalid buffer type %s", config.BufferType), + "The only supported buffer type is 'memory'", + ) + } +} + +func NewRetryConfig() RetryConfig { + return RetryConfig{ + InitialInterval: operator.Duration{Duration: 500 * time.Millisecond}, + RandomizationFactor: 0.5, + Multiplier: 1.5, + MaxInterval: operator.Duration{Duration: 15 * time.Minute}, + } +} + +// RetryConfig is the configuration of an entity that will retry processing after an error +type RetryConfig struct { + InitialInterval operator.Duration `json:"initial_interval,omitempty" yaml:"initial_interval,omitempty"` + RandomizationFactor float64 `json:"randomization_factor,omitempty" yaml:"randomization_factor,omitempty"` + Multiplier float64 `json:"multiplier,omitempty" yaml:"multiplier,omitempty"` + MaxInterval operator.Duration `json:"max_interval,omitempty" yaml:"max_interval,omitempty"` + MaxElapsedTime operator.Duration `json:"max_elapsed_time,omitempty" yaml:"max_elapsed_time,omitempty"` +} diff --git a/plugin/buffer/buffer_test.go b/operator/buffer/buffer_test.go similarity index 95% rename from plugin/buffer/buffer_test.go rename to operator/buffer/buffer_test.go index 9eb71062f..11cded7e1 100644 --- a/plugin/buffer/buffer_test.go +++ b/operator/buffer/buffer_test.go @@ -8,7 +8,7 @@ import ( "time" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/require" "go.uber.org/zap" yaml "gopkg.in/yaml.v2" @@ -34,7 +34,7 @@ func (b *bufferHandler) Logger() *zap.SugaredLogger { func TestBuffer(t *testing.T) { config := NewConfig() - config.DelayThreshold = plugin.Duration{ + config.DelayThreshold = operator.Duration{ Duration: 100 * time.Millisecond, } diff --git a/plugin/buffer/memory_buffer.go b/operator/buffer/memory_buffer.go similarity index 100% rename from plugin/buffer/memory_buffer.go rename to operator/buffer/memory_buffer.go diff --git a/plugin/buffer/memory_buffer_test.go b/operator/buffer/memory_buffer_test.go similarity index 90% rename from plugin/buffer/memory_buffer_test.go rename to operator/buffer/memory_buffer_test.go index 829f51138..20e66f549 100644 --- a/plugin/buffer/memory_buffer_test.go +++ b/operator/buffer/memory_buffer_test.go @@ -7,7 +7,7 @@ import ( "time" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -47,7 +47,7 @@ func newMockHandler(t *testing.T) *mockHandler { func TestMemoryBufferRetry(t *testing.T) { t.Run("FailOnce", func(t *testing.T) { cfg := NewConfig() - cfg.DelayThreshold = plugin.Duration{Duration: 10 * time.Millisecond} + cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Millisecond} buffer, err := cfg.Build() require.NoError(t, err) handler := newMockHandler(t) @@ -68,7 +68,7 @@ func TestMemoryBufferRetry(t *testing.T) { t.Run("ContextCancelled", func(t *testing.T) { cfg := NewConfig() - cfg.DelayThreshold = plugin.Duration{Duration: 10 * time.Millisecond} + cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Millisecond} buffer, err := cfg.Build() require.NoError(t, err) handler := newMockHandler(t) @@ -95,8 +95,8 @@ func TestMemoryBufferRetry(t *testing.T) { t.Run("ExceededLimit", func(t *testing.T) { cfg := NewConfig() - cfg.DelayThreshold = plugin.Duration{Duration: 10 * time.Millisecond} - cfg.Retry.MaxElapsedTime = plugin.Duration{Duration: time.Nanosecond} + cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Millisecond} + cfg.Retry.MaxElapsedTime = operator.Duration{Duration: time.Nanosecond} buffer, err := cfg.Build() require.NoError(t, err) handler := newMockHandler(t) @@ -124,7 +124,7 @@ func TestMemoryBufferRetry(t *testing.T) { func TestMemoryBufferFlush(t *testing.T) { t.Run("Simple", func(t *testing.T) { cfg := NewConfig() - cfg.DelayThreshold = plugin.Duration{Duration: 10 * time.Hour} + cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Hour} buffer, err := cfg.Build() require.NoError(t, err) handler := newMockHandler(t) @@ -156,7 +156,7 @@ func TestMemoryBufferFlush(t *testing.T) { t.Run("ContextCancelled", func(t *testing.T) { cfg := NewConfig() - cfg.DelayThreshold = plugin.Duration{Duration: 10 * time.Hour} + cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Hour} buffer, err := cfg.Build() require.NoError(t, err) handler := newMockHandler(t) diff --git a/operator/builtin/builtin.go b/operator/builtin/builtin.go new file mode 100644 index 000000000..c52ad1286 --- /dev/null +++ b/operator/builtin/builtin.go @@ -0,0 +1,9 @@ +package builtin + +import ( + // Load embedded packages when importing builtin operators + _ "github.com/observiq/carbon/operator/builtin/input" + _ "github.com/observiq/carbon/operator/builtin/output" + _ "github.com/observiq/carbon/operator/builtin/parser" + _ "github.com/observiq/carbon/operator/builtin/transformer" +) diff --git a/plugin/builtin/builtin_test.go b/operator/builtin/builtin_test.go similarity index 100% rename from plugin/builtin/builtin_test.go rename to operator/builtin/builtin_test.go diff --git a/plugin/builtin/input/file/file.go b/operator/builtin/input/file/file.go similarity index 91% rename from plugin/builtin/input/file/file.go rename to operator/builtin/input/file/file.go index d780ca079..946fdce1d 100644 --- a/plugin/builtin/input/file/file.go +++ b/operator/builtin/input/file/file.go @@ -16,8 +16,8 @@ import ( "time" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" "golang.org/x/text/encoding" "golang.org/x/text/encoding/ianaindex" @@ -25,13 +25,13 @@ import ( ) func init() { - plugin.Register("file_input", func() plugin.Builder { return NewInputConfig("") }) + operator.Register("file_input", func() operator.Builder { return NewInputConfig("") }) } -func NewInputConfig(pluginID string) *InputConfig { +func NewInputConfig(operatorID string) *InputConfig { return &InputConfig{ - InputConfig: helper.NewInputConfig(pluginID, "file_input"), - PollInterval: plugin.Duration{Duration: 200 * time.Millisecond}, + InputConfig: helper.NewInputConfig(operatorID, "file_input"), + PollInterval: operator.Duration{Duration: 200 * time.Millisecond}, FilePathField: entry.NewNilField(), FileNameField: entry.NewNilField(), StartAt: "end", @@ -40,20 +40,20 @@ func NewInputConfig(pluginID string) *InputConfig { } } -// InputConfig is the configuration of a file input plugin +// InputConfig is the configuration of a file input operator type InputConfig struct { helper.InputConfig `yaml:",inline"` Include []string `json:"include,omitempty" yaml:"include,omitempty"` Exclude []string `json:"exclude,omitempty" yaml:"exclude,omitempty"` - PollInterval plugin.Duration `json:"poll_interval,omitempty" yaml:"poll_interval,omitempty"` - Multiline *MultilineConfig `json:"multiline,omitempty" yaml:"multiline,omitempty"` - FilePathField entry.Field `json:"file_path_field,omitempty" yaml:"file_path_field,omitempty"` - FileNameField entry.Field `json:"file_name_field,omitempty" yaml:"file_name_field,omitempty"` - StartAt string `json:"start_at,omitempty" yaml:"start_at,omitempty"` - MaxLogSize int `json:"max_log_size,omitempty" yaml:"max_log_size,omitempty"` - Encoding string `json:"encoding,omitempty" yaml:"encoding,omitempty"` + PollInterval operator.Duration `json:"poll_interval,omitempty" yaml:"poll_interval,omitempty"` + Multiline *MultilineConfig `json:"multiline,omitempty" yaml:"multiline,omitempty"` + FilePathField entry.Field `json:"file_path_field,omitempty" yaml:"file_path_field,omitempty"` + FileNameField entry.Field `json:"file_name_field,omitempty" yaml:"file_name_field,omitempty"` + StartAt string `json:"start_at,omitempty" yaml:"start_at,omitempty"` + MaxLogSize int `json:"max_log_size,omitempty" yaml:"max_log_size,omitempty"` + Encoding string `json:"encoding,omitempty" yaml:"encoding,omitempty"` } // MultilineConfig is the configuration a multiline operation @@ -62,8 +62,8 @@ type MultilineConfig struct { LineEndPattern string `json:"line_end_pattern" yaml:"line_end_pattern"` } -// Build will build a file input plugin from the supplied configuration -func (c InputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a file input operator from the supplied configuration +func (c InputConfig) Build(context operator.BuildContext) (operator.Operator, error) { inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err @@ -109,7 +109,7 @@ func (c InputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) return nil, fmt.Errorf("invalid start_at location '%s'", c.StartAt) } - plugin := &InputOperator{ + operator := &InputOperator{ InputOperator: inputOperator, Include: c.Include, Exclude: c.Exclude, @@ -126,7 +126,7 @@ func (c InputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) MaxLogSize: c.MaxLogSize, } - return plugin, nil + return operator, nil } var encodingOverrides = map[string]encoding.Encoding{ @@ -186,7 +186,7 @@ func (c InputConfig) getSplitFunc(encoding encoding.Encoding) (bufio.SplitFunc, return splitFunc, nil } -// InputOperator is a plugin that monitors files for entries +// InputOperator is a operator that monitors files for entries type InputOperator struct { helper.InputOperator diff --git a/plugin/builtin/input/file/file_test.go b/operator/builtin/input/file/file_test.go similarity index 98% rename from plugin/builtin/input/file/file_test.go rename to operator/builtin/input/file/file_test.go index f38d9a149..8a2b2e6dc 100644 --- a/plugin/builtin/input/file/file_test.go +++ b/operator/builtin/input/file/file_test.go @@ -14,7 +14,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -27,16 +27,16 @@ func newTestFileSource(t *testing.T) (*InputOperator, chan *entry.Entry) { }) cfg := NewInputConfig("testfile") - cfg.PollInterval = plugin.Duration{Duration: 50 * time.Millisecond} + cfg.PollInterval = operator.Duration{Duration: 50 * time.Millisecond} cfg.StartAt = "beginning" cfg.Include = []string{"should-be-overwritten"} pg, err := cfg.Build(testutil.NewBuildContext(t)) if err != nil { - t.Fatalf("Error building plugin: %s", err) + t.Fatalf("Error building operator: %s", err) } source := pg.(*InputOperator) - source.OutputOperators = []plugin.Operator{mockOutput} + source.OutputOperators = []operator.Operator{mockOutput} return source, receivedEntries } @@ -50,7 +50,7 @@ func TestFileSource_Build(t *testing.T) { cfg.OutputIDs = []string{"mock"} cfg.Include = []string{"/var/log/testpath.*"} cfg.Exclude = []string{"/var/log/testpath.ex*"} - cfg.PollInterval = plugin.Duration{Duration: 10 * time.Millisecond} + cfg.PollInterval = operator.Duration{Duration: 10 * time.Millisecond} cfg.FilePathField = entry.NewRecordField("testpath") return cfg } @@ -132,7 +132,7 @@ func TestFileSource_Build(t *testing.T) { return } - err = plg.SetOutputs([]plugin.Operator{mockOutput}) + err = plg.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) fileInput := plg.(*InputOperator) diff --git a/plugin/builtin/input/file/line_splitter.go b/operator/builtin/input/file/line_splitter.go similarity index 100% rename from plugin/builtin/input/file/line_splitter.go rename to operator/builtin/input/file/line_splitter.go diff --git a/plugin/builtin/input/file/line_splitter_test.go b/operator/builtin/input/file/line_splitter_test.go similarity index 100% rename from plugin/builtin/input/file/line_splitter_test.go rename to operator/builtin/input/file/line_splitter_test.go diff --git a/plugin/builtin/input/file/read_to_end.go b/operator/builtin/input/file/read_to_end.go similarity index 97% rename from plugin/builtin/input/file/read_to_end.go rename to operator/builtin/input/file/read_to_end.go index 7fac69e61..0f2bf76f9 100644 --- a/plugin/builtin/input/file/read_to_end.go +++ b/operator/builtin/input/file/read_to_end.go @@ -9,12 +9,12 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" "golang.org/x/text/encoding" ) -// ReadToEnd will read entries from a file and send them to the outputs of an input plugin +// ReadToEnd will read entries from a file and send them to the outputs of an input operator func ReadToEnd( ctx context.Context, path string, diff --git a/plugin/builtin/input/generate.go b/operator/builtin/input/generate.go similarity index 80% rename from plugin/builtin/input/generate.go rename to operator/builtin/input/generate.go index 98434e7e5..fbdc19ba2 100644 --- a/plugin/builtin/input/generate.go +++ b/operator/builtin/input/generate.go @@ -7,21 +7,21 @@ import ( "time" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("generate_input", func() plugin.Builder { return NewGenerateInputConfig("") }) + operator.Register("generate_input", func() operator.Builder { return NewGenerateInputConfig("") }) } -func NewGenerateInputConfig(pluginID string) *GenerateInputConfig { +func NewGenerateInputConfig(operatorID string) *GenerateInputConfig { return &GenerateInputConfig{ - InputConfig: helper.NewInputConfig(pluginID, "generate_input"), + InputConfig: helper.NewInputConfig(operatorID, "generate_input"), } } -// GenerateInputConfig is the configuration of a generate input plugin. +// GenerateInputConfig is the configuration of a generate input operator. type GenerateInputConfig struct { helper.InputConfig `yaml:",inline"` Entry entry.Entry `json:"entry" yaml:"entry"` @@ -29,8 +29,8 @@ type GenerateInputConfig struct { Static bool `json:"static" yaml:"static,omitempty"` } -// Build will build a generate input plugin. -func (c *GenerateInputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a generate input operator. +func (c *GenerateInputConfig) Build(context operator.BuildContext) (operator.Operator, error) { inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err @@ -47,7 +47,7 @@ func (c *GenerateInputConfig) Build(context plugin.BuildContext) (plugin.Operato return generateInput, nil } -// GenerateInput is a plugin that generates log entries. +// GenerateInput is a operator that generates log entries. type GenerateInput struct { helper.InputOperator entry entry.Entry diff --git a/plugin/builtin/input/generate_test.go b/operator/builtin/input/generate_test.go similarity index 87% rename from plugin/builtin/input/generate_test.go rename to operator/builtin/input/generate_test.go index 6125fa6a1..5ebf03d48 100644 --- a/plugin/builtin/input/generate_test.go +++ b/operator/builtin/input/generate_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -16,7 +16,7 @@ import ( func TestInputGenerate(t *testing.T) { count := 5 basicConfig := func() *GenerateInputConfig { - cfg := NewGenerateInputConfig("test_plugin_id") + cfg := NewGenerateInputConfig("test_operator_id") cfg.OutputIDs = []string{"output1"} cfg.Entry = entry.Entry{ Record: "test message", @@ -36,7 +36,7 @@ func TestInputGenerate(t *testing.T) { }) generateInput := newOperator.(*GenerateInput) - err = generateInput.SetOutputs([]plugin.Operator{mockOutput}) + err = generateInput.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) err = generateInput.Start() @@ -66,7 +66,7 @@ pipeline: tmpl, err := template.New("my_generator").Parse(templateText) require.NoError(t, err) - registry := plugin.CustomRegistry{ + registry := operator.CustomRegistry{ "sample": tmpl, } @@ -76,8 +76,8 @@ pipeline: config, err := registry.Render("sample", params) require.NoError(t, err) - expectedConfig := plugin.CustomConfig{ - Pipeline: []plugin.Config{ + expectedConfig := operator.CustomConfig{ + Pipeline: []operator.Config{ { Builder: &GenerateInputConfig{ InputConfig: helper.InputConfig{ diff --git a/operator/builtin/input/input.go b/operator/builtin/input/input.go new file mode 100644 index 000000000..dc5ce2719 --- /dev/null +++ b/operator/builtin/input/input.go @@ -0,0 +1,6 @@ +package input + +import ( + // Load file package when importing input operators + _ "github.com/observiq/carbon/operator/builtin/input/file" +) diff --git a/plugin/builtin/input/journald.go b/operator/builtin/input/journald.go similarity index 68% rename from plugin/builtin/input/journald.go rename to operator/builtin/input/journald.go index 1eb3faecb..fdf0fedc3 100644 --- a/plugin/builtin/input/journald.go +++ b/operator/builtin/input/journald.go @@ -15,22 +15,22 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" ) func init() { - plugin.Register("journald_input", func() plugin.Builder { return NewJournaldInputConfig("") }) + operator.Register("journald_input", func() operator.Builder { return NewJournaldInputConfig("") }) } -func NewJournaldInputConfig(pluginID string) *JournaldInputConfig { +func NewJournaldInputConfig(operatorID string) *JournaldInputConfig { return &JournaldInputConfig{ - InputConfig: helper.NewInputConfig(pluginID, "journald_input"), + InputConfig: helper.NewInputConfig(operatorID, "journald_input"), } } -// JournaldInputConfig is the configuration of a journald input plugin +// JournaldInputConfig is the configuration of a journald input operator type JournaldInputConfig struct { helper.InputConfig `yaml:",inline"` @@ -38,8 +38,8 @@ type JournaldInputConfig struct { Files []string `json:"files,omitempty" yaml:"files,omitempty"` } -// Build will build a journald input plugin from the supplied configuration -func (c JournaldInputConfig) Build(buildContext plugin.BuildContext) (plugin.Plugin, error) { +// Build will build a journald input operator from the supplied configuration +func (c JournaldInputConfig) Build(buildContext operator.BuildContext) (operator.Plugin, error) { inputPlugin, err := c.InputConfig.Build(buildContext) if err != nil { return nil, err @@ -79,7 +79,7 @@ func (c JournaldInputConfig) Build(buildContext plugin.BuildContext) (plugin.Plu return journaldInput, nil } -// JournaldInput is a plugin that process logs using journald +// JournaldInput is a operator that process logs using journald type JournaldInput struct { helper.InputPlugin @@ -99,21 +99,21 @@ type cmd interface { var lastReadCursorKey = "lastReadCursor" // Start will start generating log entries. -func (plugin *JournaldInput) Start() error { +func (operator *JournaldInput) Start() error { ctx, cancel := context.WithCancel(context.Background()) - plugin.cancel = cancel - plugin.wg = &sync.WaitGroup{} + operator.cancel = cancel + operator.wg = &sync.WaitGroup{} - err := plugin.persist.Load() + err := operator.persist.Load() if err != nil { return err } // Start from a cursor if there is a saved offset - cursor := plugin.persist.Get(lastReadCursorKey) + cursor := operator.persist.Get(lastReadCursorKey) // Start journalctl - cmd := plugin.newCmd(ctx, cursor) + cmd := operator.newCmd(ctx, cursor) stdout, err := cmd.StdoutPipe() if err != nil { return fmt.Errorf("failed to get journalctl stdout: %s", err) @@ -124,24 +124,24 @@ func (plugin *JournaldInput) Start() error { } // Start a goroutine to periodically flush the offsets - plugin.wg.Add(1) + operator.wg.Add(1) go func() { - defer plugin.wg.Done() + defer operator.wg.Done() for { select { case <-ctx.Done(): return case <-time.After(time.Second): - plugin.syncOffsets() + operator.syncOffsets() } } }() // Start the reader goroutine - plugin.wg.Add(1) + operator.wg.Add(1) go func() { - defer plugin.wg.Done() - defer plugin.syncOffsets() + defer operator.wg.Done() + defer operator.syncOffsets() stdoutBuf := bufio.NewReader(stdout) @@ -149,27 +149,27 @@ func (plugin *JournaldInput) Start() error { line, err := stdoutBuf.ReadBytes('\n') if err != nil { if err != io.EOF { - plugin.Errorw("Received error reading from journalctl stdout", zap.Error(err)) + operator.Errorw("Received error reading from journalctl stdout", zap.Error(err)) } return } - entry, cursor, err := plugin.parseJournalEntry(line) + entry, cursor, err := operator.parseJournalEntry(line) if err != nil { - plugin.Warnw("Failed to parse journal entry", zap.Error(err)) + operator.Warnw("Failed to parse journal entry", zap.Error(err)) continue } - plugin.persist.Set(lastReadCursorKey, []byte(cursor)) - plugin.Write(ctx, entry) + operator.persist.Set(lastReadCursorKey, []byte(cursor)) + operator.Write(ctx, entry) } }() return nil } -func (plugin *JournaldInput) parseJournalEntry(line []byte) (*entry.Entry, string, error) { +func (operator *JournaldInput) parseJournalEntry(line []byte) (*entry.Entry, string, error) { var record map[string]interface{} - err := plugin.json.Unmarshal(line, &record) + err := operator.json.Unmarshal(line, &record) if err != nil { return nil, "", err } @@ -201,21 +201,21 @@ func (plugin *JournaldInput) parseJournalEntry(line []byte) (*entry.Entry, strin return nil, "", errors.New("journald field for cursor is not a string") } - entry := plugin.NewEntry(record) + entry := operator.NewEntry(record) entry.Timestamp = time.Unix(0, timestampInt*1000) // in microseconds return entry, cursorString, nil } -func (plugin *JournaldInput) syncOffsets() { - err := plugin.persist.Sync() +func (operator *JournaldInput) syncOffsets() { + err := operator.persist.Sync() if err != nil { - plugin.Errorw("Failed to sync offsets", zap.Error(err)) + operator.Errorw("Failed to sync offsets", zap.Error(err)) } } // Stop will stop generating logs. -func (plugin *JournaldInput) Stop() error { - plugin.cancel() - plugin.wg.Wait() +func (operator *JournaldInput) Stop() error { + operator.cancel() + operator.wg.Wait() return nil } diff --git a/plugin/builtin/input/journald_test.go b/operator/builtin/input/journald_test.go similarity index 96% rename from plugin/builtin/input/journald_test.go rename to operator/builtin/input/journald_test.go index b0d5cafb9..2c63e8ec1 100644 --- a/plugin/builtin/input/journald_test.go +++ b/operator/builtin/input/journald_test.go @@ -12,7 +12,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -37,13 +37,13 @@ func TestInputJournald(t *testing.T) { journaldInput, err := cfg.Build(testutil.NewBuildContext(t)) require.NoError(t, err) - mockOutput := testutil.NewMockPlugin("output") + mockOutput := testutil.NewMockOperator("output") received := make(chan *entry.Entry) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { received <- args.Get(1).(*entry.Entry) }).Return(nil) - err = journaldInput.SetOutputs([]plugin.Plugin{mockOutput}) + err = journaldInput.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) journaldInput.(*JournaldInput).newCmd = func(ctx context.Context, cursor []byte) cmd { diff --git a/plugin/builtin/input/tcp.go b/operator/builtin/input/tcp.go similarity index 85% rename from plugin/builtin/input/tcp.go rename to operator/builtin/input/tcp.go index 32252bac9..cd555e3c1 100644 --- a/plugin/builtin/input/tcp.go +++ b/operator/builtin/input/tcp.go @@ -8,29 +8,29 @@ import ( "net" "sync" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("tcp_input", func() plugin.Builder { return NewTCPInputConfig("") }) + operator.Register("tcp_input", func() operator.Builder { return NewTCPInputConfig("") }) } -func NewTCPInputConfig(pluginID string) *TCPInputConfig { +func NewTCPInputConfig(operatorID string) *TCPInputConfig { return &TCPInputConfig{ - InputConfig: helper.NewInputConfig(pluginID, "tcp_input"), + InputConfig: helper.NewInputConfig(operatorID, "tcp_input"), } } -// TCPInputConfig is the configuration of a tcp input plugin. +// TCPInputConfig is the configuration of a tcp input operator. type TCPInputConfig struct { helper.InputConfig `yaml:",inline"` ListenAddress string `json:"listen_address,omitempty" yaml:"listen_address,omitempty"` } -// Build will build a tcp input plugin. -func (c TCPInputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a tcp input operator. +func (c TCPInputConfig) Build(context operator.BuildContext) (operator.Operator, error) { inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err @@ -52,7 +52,7 @@ func (c TCPInputConfig) Build(context plugin.BuildContext) (plugin.Operator, err return tcpInput, nil } -// TCPInput is a plugin that listens for log entries over tcp. +// TCPInput is a operator that listens for log entries over tcp. type TCPInput struct { helper.InputOperator address *net.TCPAddr diff --git a/plugin/builtin/input/tcp_test.go b/operator/builtin/input/tcp_test.go similarity index 90% rename from plugin/builtin/input/tcp_test.go rename to operator/builtin/input/tcp_test.go index a62303828..cd0cb0f08 100644 --- a/plugin/builtin/input/tcp_test.go +++ b/operator/builtin/input/tcp_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -39,7 +39,7 @@ func TestTCPInput(t *testing.T) { mockOutput := testutil.Operator{} tcpInput := newOperator.(*TCPInput) - tcpInput.InputOperator.OutputOperators = []plugin.Operator{&mockOutput} + tcpInput.InputOperator.OutputOperators = []operator.Operator{&mockOutput} entryChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { diff --git a/plugin/builtin/input/udp.go b/operator/builtin/input/udp.go similarity index 80% rename from plugin/builtin/input/udp.go rename to operator/builtin/input/udp.go index 299dac55f..1331e0d13 100644 --- a/plugin/builtin/input/udp.go +++ b/operator/builtin/input/udp.go @@ -7,29 +7,29 @@ import ( "strings" "sync" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("udp_input", func() plugin.Builder { return NewUDPInputConfig("") }) + operator.Register("udp_input", func() operator.Builder { return NewUDPInputConfig("") }) } -func NewUDPInputConfig(pluginID string) *UDPInputConfig { +func NewUDPInputConfig(operatorID string) *UDPInputConfig { return &UDPInputConfig{ - InputConfig: helper.NewInputConfig(pluginID, "udp_input"), + InputConfig: helper.NewInputConfig(operatorID, "udp_input"), } } -// UDPInputConfig is the configuration of a udp input plugin. +// UDPInputConfig is the configuration of a udp input operator. type UDPInputConfig struct { helper.InputConfig `yaml:",inline"` ListenAddress string `json:"listen_address,omitempty" yaml:"listen_address,omitempty"` } -// Build will build a udp input plugin. -func (c UDPInputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a udp input operator. +func (c UDPInputConfig) Build(context operator.BuildContext) (operator.Operator, error) { inputOperator, err := c.InputConfig.Build(context) if err != nil { return nil, err @@ -51,7 +51,7 @@ func (c UDPInputConfig) Build(context plugin.BuildContext) (plugin.Operator, err return udpInput, nil } -// UDPInput is a plugin that listens to a socket for log entries. +// UDPInput is a operator that listens to a socket for log entries. type UDPInput struct { helper.InputOperator address *net.UDPAddr diff --git a/plugin/builtin/input/udp_test.go b/operator/builtin/input/udp_test.go similarity index 90% rename from plugin/builtin/input/udp_test.go rename to operator/builtin/input/udp_test.go index 3f87fdef1..cf2af503e 100644 --- a/plugin/builtin/input/udp_test.go +++ b/operator/builtin/input/udp_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -41,7 +41,7 @@ func TestUDPInput(t *testing.T) { udpInput, ok := newOperator.(*UDPInput) require.True(t, ok) - udpInput.InputOperator.OutputOperators = []plugin.Operator{&mockOutput} + udpInput.InputOperator.OutputOperators = []operator.Operator{&mockOutput} entryChan := make(chan *entry.Entry, 1) mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { diff --git a/operator/builtin/output/drop.go b/operator/builtin/output/drop.go new file mode 100644 index 000000000..9d0d650ad --- /dev/null +++ b/operator/builtin/output/drop.go @@ -0,0 +1,48 @@ +package output + +import ( + "context" + + "github.com/observiq/carbon/entry" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" +) + +func init() { + operator.Register("drop_output", func() operator.Builder { return NewDropOutputConfig("") }) +} + +func NewDropOutputConfig(operatorID string) *DropOutputConfig { + return &DropOutputConfig{ + OutputConfig: helper.NewOutputConfig(operatorID, "drop_output"), + } +} + +// DropOutputConfig is the configuration of a drop output operator. +type DropOutputConfig struct { + helper.OutputConfig `yaml:",inline"` +} + +// Build will build a drop output operator. +func (c DropOutputConfig) Build(context operator.BuildContext) (operator.Operator, error) { + outputOperator, err := c.OutputConfig.Build(context) + if err != nil { + return nil, err + } + + dropOutput := &DropOutput{ + OutputOperator: outputOperator, + } + + return dropOutput, nil +} + +// DropOutput is a operator that consumes and ignores incoming entries. +type DropOutput struct { + helper.OutputOperator +} + +// Process will drop the incoming entry. +func (p *DropOutput) Process(ctx context.Context, entry *entry.Entry) error { + return nil +} diff --git a/plugin/builtin/output/drop_test.go b/operator/builtin/output/drop_test.go similarity index 84% rename from plugin/builtin/output/drop_test.go rename to operator/builtin/output/drop_test.go index a20b77271..0c3a125ac 100644 --- a/plugin/builtin/output/drop_test.go +++ b/operator/builtin/output/drop_test.go @@ -1,7 +1,7 @@ package output import ( - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator/helper" ) func newFakeNullOutput() *DropOutput { diff --git a/plugin/builtin/output/elastic.go b/operator/builtin/output/elastic.go similarity index 88% rename from plugin/builtin/output/elastic.go rename to operator/builtin/output/elastic.go index cbaf58cb0..e2c37d6ef 100644 --- a/plugin/builtin/output/elastic.go +++ b/operator/builtin/output/elastic.go @@ -11,24 +11,24 @@ import ( uuid "github.com/hashicorp/go-uuid" "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/buffer" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/buffer" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" ) func init() { - plugin.Register("elastic_output", func() plugin.Builder { return NewElasticOutputConfig("") }) + operator.Register("elastic_output", func() operator.Builder { return NewElasticOutputConfig("") }) } -func NewElasticOutputConfig(pluginID string) *ElasticOutputConfig { +func NewElasticOutputConfig(operatorID string) *ElasticOutputConfig { return &ElasticOutputConfig{ - OutputConfig: helper.NewOutputConfig(pluginID, "elastic_output"), + OutputConfig: helper.NewOutputConfig(operatorID, "elastic_output"), BufferConfig: buffer.NewConfig(), } } -// ElasticOutputConfig is the configuration of an elasticsearch output plugin. +// ElasticOutputConfig is the configuration of an elasticsearch output operator. type ElasticOutputConfig struct { helper.OutputConfig `yaml:",inline"` BufferConfig buffer.Config `json:"buffer" yaml:"buffer"` @@ -42,8 +42,8 @@ type ElasticOutputConfig struct { IDField *entry.Field `json:"id_field,omitempty" yaml:"id_field,omitempty"` } -// Build will build an elasticsearch output plugin. -func (c ElasticOutputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build an elasticsearch output operator. +func (c ElasticOutputConfig) Build(context operator.BuildContext) (operator.Operator, error) { outputOperator, err := c.OutputConfig.Build(context) if err != nil { return nil, err @@ -84,7 +84,7 @@ func (c ElasticOutputConfig) Build(context plugin.BuildContext) (plugin.Operator return elasticOutput, nil } -// ElasticOutput is a plugin that sends entries to elasticsearch. +// ElasticOutput is a operator that sends entries to elasticsearch. type ElasticOutput struct { helper.OutputOperator buffer.Buffer diff --git a/plugin/builtin/output/elastic_test.go b/operator/builtin/output/elastic_test.go similarity index 100% rename from plugin/builtin/output/elastic_test.go rename to operator/builtin/output/elastic_test.go diff --git a/plugin/builtin/output/file.go b/operator/builtin/output/file.go similarity index 75% rename from plugin/builtin/output/file.go rename to operator/builtin/output/file.go index 65285283e..00ba693c5 100644 --- a/plugin/builtin/output/file.go +++ b/operator/builtin/output/file.go @@ -9,21 +9,21 @@ import ( "sync" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("file_output", func() plugin.Builder { return NewFileOutputConfig("") }) + operator.Register("file_output", func() operator.Builder { return NewFileOutputConfig("") }) } -func NewFileOutputConfig(pluginID string) *FileOutputConfig { +func NewFileOutputConfig(operatorID string) *FileOutputConfig { return &FileOutputConfig{ - OutputConfig: helper.NewOutputConfig(pluginID, "file_output"), + OutputConfig: helper.NewOutputConfig(operatorID, "file_output"), } } -// FileOutputConfig is the configuration of a file output pluginn. +// FileOutputConfig is the configuration of a file output operatorn. type FileOutputConfig struct { helper.OutputConfig `yaml:",inline"` @@ -31,8 +31,8 @@ type FileOutputConfig struct { Format string `json:"format,omitempty" path:"format,omitempty"` } -// Build will build a file output plugin. -func (c FileOutputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a file output operator. +func (c FileOutputConfig) Build(context operator.BuildContext) (operator.Operator, error) { outputOperator, err := c.OutputConfig.Build(context) if err != nil { return nil, err @@ -59,7 +59,7 @@ func (c FileOutputConfig) Build(context plugin.BuildContext) (plugin.Operator, e return fileOutput, nil } -// FileOutput is a plugin that writes logs to a file. +// FileOutput is a operator that writes logs to a file. type FileOutput struct { helper.OutputOperator diff --git a/plugin/builtin/output/google_cloud.go b/operator/builtin/output/google_cloud.go similarity index 88% rename from plugin/builtin/output/google_cloud.go rename to operator/builtin/output/google_cloud.go index 25bdc36f5..38a691d76 100644 --- a/plugin/builtin/output/google_cloud.go +++ b/operator/builtin/output/google_cloud.go @@ -13,9 +13,9 @@ import ( structpb "github.com/golang/protobuf/ptypes/struct" gax "github.com/googleapis/gax-go" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/buffer" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/buffer" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" "golang.org/x/oauth2/google" "google.golang.org/api/option" @@ -25,33 +25,33 @@ import ( ) func init() { - plugin.Register("google_cloud_output", func() plugin.Builder { return NewGoogleCloudOutputConfig("") }) + operator.Register("google_cloud_output", func() operator.Builder { return NewGoogleCloudOutputConfig("") }) } -func NewGoogleCloudOutputConfig(pluginID string) *GoogleCloudOutputConfig { +func NewGoogleCloudOutputConfig(operatorID string) *GoogleCloudOutputConfig { return &GoogleCloudOutputConfig{ - OutputConfig: helper.NewOutputConfig(pluginID, "google_cloud_output"), + OutputConfig: helper.NewOutputConfig(operatorID, "google_cloud_output"), BufferConfig: buffer.NewConfig(), - Timeout: plugin.Duration{Duration: 10 * time.Second}, + Timeout: operator.Duration{Duration: 10 * time.Second}, } } -// GoogleCloudOutputConfig is the configuration of a google cloud output plugin. +// GoogleCloudOutputConfig is the configuration of a google cloud output operator. type GoogleCloudOutputConfig struct { helper.OutputConfig `yaml:",inline"` BufferConfig buffer.Config `json:"buffer,omitempty" yaml:"buffer,omitempty"` - Credentials string `json:"credentials,omitempty" yaml:"credentials,omitempty"` - CredentialsFile string `json:"credentials_file,omitempty" yaml:"credentials_file,omitempty"` - ProjectID string `json:"project_id" yaml:"project_id"` - LogNameField *entry.Field `json:"log_name_field,omitempty" yaml:"log_name_field,omitempty"` - TraceField *entry.Field `json:"trace_field,omitempty" yaml:"trace_field,omitempty"` - SpanIDField *entry.Field `json:"span_id_field,omitempty" yaml:"span_id_field,omitempty"` - Timeout plugin.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"` + Credentials string `json:"credentials,omitempty" yaml:"credentials,omitempty"` + CredentialsFile string `json:"credentials_file,omitempty" yaml:"credentials_file,omitempty"` + ProjectID string `json:"project_id" yaml:"project_id"` + LogNameField *entry.Field `json:"log_name_field,omitempty" yaml:"log_name_field,omitempty"` + TraceField *entry.Field `json:"trace_field,omitempty" yaml:"trace_field,omitempty"` + SpanIDField *entry.Field `json:"span_id_field,omitempty" yaml:"span_id_field,omitempty"` + Timeout operator.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"` } -// Build will build a google cloud output plugin. -func (c GoogleCloudOutputConfig) Build(buildContext plugin.BuildContext) (plugin.Operator, error) { +// Build will build a google cloud output operator. +func (c GoogleCloudOutputConfig) Build(buildContext operator.BuildContext) (operator.Operator, error) { outputOperator, err := c.OutputConfig.Build(buildContext) if err != nil { return nil, err @@ -79,7 +79,7 @@ func (c GoogleCloudOutputConfig) Build(buildContext plugin.BuildContext) (plugin return googleCloudOutput, nil } -// GoogleCloudOutput is a plugin that sends logs to google cloud logging. +// GoogleCloudOutput is a operator that sends logs to google cloud logging. type GoogleCloudOutput struct { helper.OutputOperator buffer.Buffer diff --git a/plugin/builtin/output/google_cloud_test.go b/operator/builtin/output/google_cloud_test.go similarity index 98% rename from plugin/builtin/output/google_cloud_test.go rename to operator/builtin/output/google_cloud_test.go index b002258af..61a13c2aa 100644 --- a/plugin/builtin/output/google_cloud_test.go +++ b/operator/builtin/output/google_cloud_test.go @@ -11,7 +11,7 @@ import ( gax "github.com/googleapis/gax-go" "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/require" "google.golang.org/genproto/googleapis/api/monitoredres" sev "google.golang.org/genproto/googleapis/logging/type" @@ -43,7 +43,7 @@ type googleCloudTestCase struct { func googleCloudBasicConfig() *GoogleCloudOutputConfig { cfg := NewGoogleCloudOutputConfig("test_id") cfg.ProjectID = "test_project_id" - cfg.BufferConfig.DelayThreshold = plugin.Duration{Duration: time.Millisecond} + cfg.BufferConfig.DelayThreshold = operator.Duration{Duration: time.Millisecond} return cfg } diff --git a/plugin/builtin/output/stdout.go b/operator/builtin/output/stdout.go similarity index 63% rename from plugin/builtin/output/stdout.go rename to operator/builtin/output/stdout.go index 2769aa6b1..a8be92ce8 100644 --- a/plugin/builtin/output/stdout.go +++ b/operator/builtin/output/stdout.go @@ -8,30 +8,30 @@ import ( "sync" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) // Stdout is a global handle to standard output var Stdout io.Writer = os.Stdout func init() { - plugin.Register("stdout", func() plugin.Builder { return NewStdoutConfig("") }) + operator.Register("stdout", func() operator.Builder { return NewStdoutConfig("") }) } -func NewStdoutConfig(pluginID string) *StdoutConfig { +func NewStdoutConfig(operatorID string) *StdoutConfig { return &StdoutConfig{ - OutputConfig: helper.NewOutputConfig(pluginID, "stdout"), + OutputConfig: helper.NewOutputConfig(operatorID, "stdout"), } } -// StdoutConfig is the configuration of the Stdout plugin +// StdoutConfig is the configuration of the Stdout operator type StdoutConfig struct { helper.OutputConfig `yaml:",inline"` } -// Build will build a stdout plugin. -func (c StdoutConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a stdout operator. +func (c StdoutConfig) Build(context operator.BuildContext) (operator.Operator, error) { outputOperator, err := c.OutputConfig.Build(context) if err != nil { return nil, err @@ -43,7 +43,7 @@ func (c StdoutConfig) Build(context plugin.BuildContext) (plugin.Operator, error }, nil } -// StdoutOperator is a plugin that logs entries using stdout. +// StdoutOperator is a operator that logs entries using stdout. type StdoutOperator struct { helper.OutputOperator encoder *json.Encoder diff --git a/plugin/builtin/output/stdout_test.go b/operator/builtin/output/stdout_test.go similarity index 75% rename from plugin/builtin/output/stdout_test.go rename to operator/builtin/output/stdout_test.go index b7adb456f..aa0f890c5 100644 --- a/plugin/builtin/output/stdout_test.go +++ b/operator/builtin/output/stdout_test.go @@ -9,7 +9,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/require" ) @@ -17,24 +17,24 @@ func TestStdoutOperator(t *testing.T) { cfg := StdoutConfig{ OutputConfig: helper.OutputConfig{ BasicConfig: helper.BasicConfig{ - OperatorID: "test_plugin_id", + OperatorID: "test_operator_id", OperatorType: "stdout", }, }, } - plugin, err := cfg.Build(testutil.NewBuildContext(t)) + operator, err := cfg.Build(testutil.NewBuildContext(t)) require.NoError(t, err) var buf bytes.Buffer - plugin.(*StdoutOperator).encoder = json.NewEncoder(&buf) + operator.(*StdoutOperator).encoder = json.NewEncoder(&buf) ts := time.Unix(1591042864, 0) e := &entry.Entry{ Timestamp: ts, Record: "test record", } - err = plugin.Process(context.Background(), e) + err = operator.Process(context.Background(), e) require.NoError(t, err) marshalledTimestamp, err := json.Marshal(ts) diff --git a/plugin/builtin/parser/json.go b/operator/builtin/parser/json.go similarity index 68% rename from plugin/builtin/parser/json.go rename to operator/builtin/parser/json.go index 66b317482..3daac0000 100644 --- a/plugin/builtin/parser/json.go +++ b/operator/builtin/parser/json.go @@ -6,27 +6,27 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("json_parser", func() plugin.Builder { return NewJSONParserConfig("") }) + operator.Register("json_parser", func() operator.Builder { return NewJSONParserConfig("") }) } -func NewJSONParserConfig(pluginID string) *JSONParserConfig { +func NewJSONParserConfig(operatorID string) *JSONParserConfig { return &JSONParserConfig{ - ParserConfig: helper.NewParserConfig(pluginID, "json_parser"), + ParserConfig: helper.NewParserConfig(operatorID, "json_parser"), } } -// JSONParserConfig is the configuration of a JSON parser plugin. +// JSONParserConfig is the configuration of a JSON parser operator. type JSONParserConfig struct { helper.ParserConfig `yaml:",inline"` } -// Build will build a JSON parser plugin. -func (c JSONParserConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a JSON parser operator. +func (c JSONParserConfig) Build(context operator.BuildContext) (operator.Operator, error) { parserOperator, err := c.ParserConfig.Build(context) if err != nil { return nil, err @@ -40,7 +40,7 @@ func (c JSONParserConfig) Build(context plugin.BuildContext) (plugin.Operator, e return jsonParser, nil } -// JSONParser is a plugin that parses JSON. +// JSONParser is a operator that parses JSON. type JSONParser struct { helper.ParserOperator json jsoniter.API diff --git a/plugin/builtin/parser/json_test.go b/operator/builtin/parser/json_test.go similarity index 94% rename from plugin/builtin/parser/json_test.go rename to operator/builtin/parser/json_test.go index 6943bb493..cf900ca8a 100644 --- a/plugin/builtin/parser/json_test.go +++ b/operator/builtin/parser/json_test.go @@ -8,8 +8,8 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -28,7 +28,7 @@ func NewFakeJSONOperator() (*JSONParser, *testutil.Operator) { OperatorType: "json_parser", SugaredLogger: logger.Sugar(), }, - OutputOperators: []plugin.Operator{&mock}, + OutputOperators: []operator.Operator{&mock}, }, }, ParseFrom: entry.NewRecordField("testfield"), @@ -39,7 +39,7 @@ func NewFakeJSONOperator() (*JSONParser, *testutil.Operator) { } func TestJSONImplementations(t *testing.T) { - require.Implements(t, (*plugin.Operator)(nil), new(JSONParser)) + require.Implements(t, (*operator.Operator)(nil), new(JSONParser)) } func TestJSONParser(t *testing.T) { diff --git a/plugin/builtin/parser/regex.go b/operator/builtin/parser/regex.go similarity index 79% rename from plugin/builtin/parser/regex.go rename to operator/builtin/parser/regex.go index 1e1cf2487..8392dd196 100644 --- a/plugin/builtin/parser/regex.go +++ b/operator/builtin/parser/regex.go @@ -7,29 +7,29 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("regex_parser", func() plugin.Builder { return NewRegexParserConfig("") }) + operator.Register("regex_parser", func() operator.Builder { return NewRegexParserConfig("") }) } -func NewRegexParserConfig(pluginID string) *RegexParserConfig { +func NewRegexParserConfig(operatorID string) *RegexParserConfig { return &RegexParserConfig{ - ParserConfig: helper.NewParserConfig(pluginID, "regex_parser"), + ParserConfig: helper.NewParserConfig(operatorID, "regex_parser"), } } -// RegexParserConfig is the configuration of a regex parser plugin. +// RegexParserConfig is the configuration of a regex parser operator. type RegexParserConfig struct { helper.ParserConfig `yaml:",inline"` Regex string `json:"regex" yaml:"regex"` } -// Build will build a regex parser plugin. -func (c RegexParserConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a regex parser operator. +func (c RegexParserConfig) Build(context operator.BuildContext) (operator.Operator, error) { parserOperator, err := c.ParserConfig.Build(context) if err != nil { return nil, err @@ -65,7 +65,7 @@ func (c RegexParserConfig) Build(context plugin.BuildContext) (plugin.Operator, return regexParser, nil } -// RegexParser is a plugin that parses regex in an entry. +// RegexParser is a operator that parses regex in an entry. type RegexParser struct { helper.ParserOperator regexp *regexp.Regexp diff --git a/plugin/builtin/parser/regex_test.go b/operator/builtin/parser/regex_test.go similarity index 95% rename from plugin/builtin/parser/regex_test.go rename to operator/builtin/parser/regex_test.go index a4d1a9a6c..41acfa95d 100644 --- a/plugin/builtin/parser/regex_test.go +++ b/operator/builtin/parser/regex_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -24,7 +24,7 @@ func newFakeRegexParser() (*RegexParser, *testutil.Operator) { OperatorType: "regex_parser", }, OutputIDs: []string{"mock_output"}, - OutputOperators: []plugin.Operator{&mockOperator}, + OutputOperators: []operator.Operator{&mockOperator}, }, }, ParseFrom: entry.NewRecordField(), diff --git a/plugin/builtin/parser/severity.go b/operator/builtin/parser/severity.go similarity index 66% rename from plugin/builtin/parser/severity.go rename to operator/builtin/parser/severity.go index f304051cb..8a7bc2d5d 100644 --- a/plugin/builtin/parser/severity.go +++ b/operator/builtin/parser/severity.go @@ -5,29 +5,29 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("severity_parser", func() plugin.Builder { return NewSeverityParserConfig("") }) + operator.Register("severity_parser", func() operator.Builder { return NewSeverityParserConfig("") }) } -func NewSeverityParserConfig(pluginID string) *SeverityParserConfig { +func NewSeverityParserConfig(operatorID string) *SeverityParserConfig { return &SeverityParserConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "severity_parser"), + TransformerConfig: helper.NewTransformerConfig(operatorID, "severity_parser"), SeverityParserConfig: helper.NewSeverityParserConfig(), } } -// SeverityParserConfig is the configuration of a severity parser plugin. +// SeverityParserConfig is the configuration of a severity parser operator. type SeverityParserConfig struct { helper.TransformerConfig `yaml:",inline"` helper.SeverityParserConfig `yaml:",omitempty,inline"` } -// Build will build a time parser plugin. -func (c SeverityParserConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a time parser operator. +func (c SeverityParserConfig) Build(context operator.BuildContext) (operator.Operator, error) { transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err @@ -46,7 +46,7 @@ func (c SeverityParserConfig) Build(context plugin.BuildContext) (plugin.Operato return severityOperator, nil } -// SeverityParserOperator is a plugin that parses time from a field to an entry. +// SeverityParserOperator is a operator that parses time from a field to an entry. type SeverityParserOperator struct { helper.TransformerOperator helper.SeverityParser diff --git a/plugin/builtin/parser/severity_test.go b/operator/builtin/parser/severity_test.go similarity index 96% rename from plugin/builtin/parser/severity_test.go rename to operator/builtin/parser/severity_test.go index 844eacd92..c90ff0aa7 100644 --- a/plugin/builtin/parser/severity_test.go +++ b/operator/builtin/parser/severity_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -306,10 +306,10 @@ func runSeverityParseTest(t *testing.T, cfg *SeverityParserConfig, ent *entry.En severityOperator, err := cfg.Build(buildContext) if buildErr { - require.Error(t, err, "expected error when configuring plugin") + require.Error(t, err, "expected error when configuring operator") return } - require.NoError(t, err, "unexpected error when configuring plugin") + require.NoError(t, err, "unexpected error when configuring operator") mockOutput := &testutil.Operator{} resultChan := make(chan *entry.Entry, 1) @@ -318,7 +318,7 @@ func runSeverityParseTest(t *testing.T, cfg *SeverityParserConfig, ent *entry.En }).Return(nil) severityParser := severityOperator.(*SeverityParserOperator) - severityParser.OutputOperators = []plugin.Operator{mockOutput} + severityParser.OutputOperators = []operator.Operator{mockOutput} err = severityParser.Process(context.Background(), ent) if parseErr { @@ -337,7 +337,7 @@ func runSeverityParseTest(t *testing.T, cfg *SeverityParserConfig, ent *entry.En } func parseSeverityTestConfig(parseFrom entry.Field, preset string, mapping map[interface{}]interface{}) *SeverityParserConfig { - cfg := NewSeverityParserConfig("test_plugin_id") + cfg := NewSeverityParserConfig("test_operator_id") cfg.OutputIDs = []string{"output1"} cfg.SeverityParserConfig = helper.SeverityParserConfig{ ParseFrom: &parseFrom, diff --git a/plugin/builtin/parser/syslog.go b/operator/builtin/parser/syslog.go similarity index 88% rename from plugin/builtin/parser/syslog.go rename to operator/builtin/parser/syslog.go index 2f55054d1..87b2c390e 100644 --- a/plugin/builtin/parser/syslog.go +++ b/operator/builtin/parser/syslog.go @@ -9,29 +9,29 @@ import ( "github.com/influxdata/go-syslog/v3/rfc3164" "github.com/influxdata/go-syslog/v3/rfc5424" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("syslog_parser", func() plugin.Builder { return NewSyslogParserConfig("") }) + operator.Register("syslog_parser", func() operator.Builder { return NewSyslogParserConfig("") }) } -func NewSyslogParserConfig(pluginID string) *SyslogParserConfig { +func NewSyslogParserConfig(operatorID string) *SyslogParserConfig { return &SyslogParserConfig{ - ParserConfig: helper.NewParserConfig(pluginID, "syslog_parser"), + ParserConfig: helper.NewParserConfig(operatorID, "syslog_parser"), } } -// SyslogParserConfig is the configuration of a syslog parser plugin. +// SyslogParserConfig is the configuration of a syslog parser operator. type SyslogParserConfig struct { helper.ParserConfig `yaml:",inline"` Protocol string `json:"protocol,omitempty" yaml:"protocol,omitempty"` } -// Build will build a JSON parser plugin. -func (c SyslogParserConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a JSON parser operator. +func (c SyslogParserConfig) Build(context operator.BuildContext) (operator.Operator, error) { if c.ParserConfig.TimeParser == nil { parseFromField := entry.NewRecordField("timestamp") c.ParserConfig.TimeParser = &helper.TimeParser{ @@ -73,7 +73,7 @@ func buildMachine(protocol string) (syslog.Machine, error) { } } -// SyslogParser is a plugin that parses syslog. +// SyslogParser is a operator that parses syslog. type SyslogParser struct { helper.ParserOperator machine syslog.Machine diff --git a/plugin/builtin/parser/syslog_test.go b/operator/builtin/parser/syslog_test.go similarity index 95% rename from plugin/builtin/parser/syslog_test.go rename to operator/builtin/parser/syslog_test.go index 26837aacd..05f441090 100644 --- a/plugin/builtin/parser/syslog_test.go +++ b/operator/builtin/parser/syslog_test.go @@ -7,14 +7,14 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) func TestSyslogParser(t *testing.T) { basicConfig := func() *SyslogParserConfig { - cfg := NewSyslogParserConfig("test_plugin_id") + cfg := NewSyslogParserConfig("test_operator_id") cfg.OutputIDs = []string{"output1"} return cfg } @@ -106,7 +106,7 @@ func TestSyslogParser(t *testing.T) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = syslogParser.SetOutputs([]plugin.Operator{mockOutput}) + err = syslogParser.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) newEntry := entry.New() diff --git a/plugin/builtin/parser/time.go b/operator/builtin/parser/time.go similarity index 61% rename from plugin/builtin/parser/time.go rename to operator/builtin/parser/time.go index d674a99b3..7453d45bf 100644 --- a/plugin/builtin/parser/time.go +++ b/operator/builtin/parser/time.go @@ -5,29 +5,29 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("time_parser", func() plugin.Builder { return NewTimeParserConfig("") }) + operator.Register("time_parser", func() operator.Builder { return NewTimeParserConfig("") }) } -func NewTimeParserConfig(pluginID string) *TimeParserConfig { +func NewTimeParserConfig(operatorID string) *TimeParserConfig { return &TimeParserConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "time_parser"), + TransformerConfig: helper.NewTransformerConfig(operatorID, "time_parser"), TimeParser: helper.NewTimeParser(), } } -// TimeParserConfig is the configuration of a time parser plugin. +// TimeParserConfig is the configuration of a time parser operator. type TimeParserConfig struct { helper.TransformerConfig `yaml:",inline"` helper.TimeParser `yaml:",omitempty,inline"` } -// Build will build a time parser plugin. -func (c TimeParserConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a time parser operator. +func (c TimeParserConfig) Build(context operator.BuildContext) (operator.Operator, error) { transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err @@ -45,13 +45,13 @@ func (c TimeParserConfig) Build(context plugin.BuildContext) (plugin.Operator, e return timeParser, nil } -// TimeParserOperator is a plugin that parses time from a field to an entry. +// TimeParserOperator is a operator that parses time from a field to an entry. type TimeParserOperator struct { helper.TransformerOperator helper.TimeParser } -// CanOutput will always return true for a parser plugin. +// CanOutput will always return true for a parser operator. func (t *TimeParserOperator) CanOutput() bool { return true } diff --git a/plugin/builtin/parser/time_test.go b/operator/builtin/parser/time_test.go similarity index 97% rename from plugin/builtin/parser/time_test.go rename to operator/builtin/parser/time_test.go index be187114d..deb3b107d 100644 --- a/plugin/builtin/parser/time_test.go +++ b/operator/builtin/parser/time_test.go @@ -8,8 +8,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -394,7 +394,7 @@ func runLossyTimeParseTest(t *testing.T, cfg *TimeParserConfig, ent *entry.Entry gotimeOperator, err := cfg.Build(buildContext) if buildErr { - require.Error(t, err, "expected error when configuring plugin") + require.Error(t, err, "expected error when configuring operator") return } require.NoError(t, err) @@ -406,11 +406,11 @@ func runLossyTimeParseTest(t *testing.T, cfg *TimeParserConfig, ent *entry.Entry }).Return(nil) timeParser := gotimeOperator.(*TimeParserOperator) - timeParser.OutputOperators = []plugin.Operator{mockOutput} + timeParser.OutputOperators = []operator.Operator{mockOutput} err = timeParser.Process(context.Background(), ent) if parseErr { - require.Error(t, err, "expected error when configuring plugin") + require.Error(t, err, "expected error when configuring operator") return } require.NoError(t, err) @@ -426,7 +426,7 @@ func runLossyTimeParseTest(t *testing.T, cfg *TimeParserConfig, ent *entry.Entry } func parseTimeTestConfig(layoutType, layout string, parseFrom entry.Field) *TimeParserConfig { - cfg := NewTimeParserConfig("test_plugin_id") + cfg := NewTimeParserConfig("test_operator_id") cfg.OutputIDs = []string{"output1"} cfg.TimeParser = helper.TimeParser{ LayoutType: layoutType, diff --git a/plugin/builtin/transformer/k8s_metadata_decorator.go b/operator/builtin/transformer/k8s_metadata_decorator.go similarity index 84% rename from plugin/builtin/transformer/k8s_metadata_decorator.go rename to operator/builtin/transformer/k8s_metadata_decorator.go index 71e9e4451..dc1d6a5d7 100644 --- a/plugin/builtin/transformer/k8s_metadata_decorator.go +++ b/operator/builtin/transformer/k8s_metadata_decorator.go @@ -7,36 +7,36 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" ) func init() { - plugin.Register("k8s_metadata_decorator", func() plugin.Builder { return NewK8smetadataDecoratorConfig("") }) + operator.Register("k8s_metadata_decorator", func() operator.Builder { return NewK8smetadataDecoratorConfig("") }) } -func NewK8smetadataDecoratorConfig(pluginID string) *K8sMetadataDecoratorConfig { +func NewK8smetadataDecoratorConfig(operatorID string) *K8sMetadataDecoratorConfig { return &K8sMetadataDecoratorConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "k8s_metadata_decorator"), + TransformerConfig: helper.NewTransformerConfig(operatorID, "k8s_metadata_decorator"), PodNameField: entry.NewRecordField("pod_name"), NamespaceField: entry.NewRecordField("namespace"), - CacheTTL: plugin.Duration{Duration: 10 * time.Minute}, + CacheTTL: operator.Duration{Duration: 10 * time.Minute}, } } -// K8sMetadataDecoratorConfig is the configuration of k8s_metadata_decorator plugin +// K8sMetadataDecoratorConfig is the configuration of k8s_metadata_decorator operator type K8sMetadataDecoratorConfig struct { helper.TransformerConfig `yaml:",inline"` - PodNameField entry.Field `json:"pod_name_field,omitempty" yaml:"pod_name_field,omitempty"` - NamespaceField entry.Field `json:"namespace_field,omitempty" yaml:"namespace_field,omitempty"` - CacheTTL plugin.Duration `json:"cache_ttl,omitempty" yaml:"cache_ttl,omitempty"` + PodNameField entry.Field `json:"pod_name_field,omitempty" yaml:"pod_name_field,omitempty"` + NamespaceField entry.Field `json:"namespace_field,omitempty" yaml:"namespace_field,omitempty"` + CacheTTL operator.Duration `json:"cache_ttl,omitempty" yaml:"cache_ttl,omitempty"` } -// Build will build a k8s_metadata_decorator plugin from the supplied configuration -func (c K8sMetadataDecoratorConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a k8s_metadata_decorator operator from the supplied configuration +func (c K8sMetadataDecoratorConfig) Build(context operator.BuildContext) (operator.Operator, error) { transformer, err := c.TransformerConfig.Build(context) if err != nil { return nil, errors.Wrap(err, "build transformer") @@ -50,7 +50,7 @@ func (c K8sMetadataDecoratorConfig) Build(context plugin.BuildContext) (plugin.O }, nil } -// K8sMetadataDecorator is a plugin for decorating entries with kubernetes metadata +// K8sMetadataDecorator is a operator for decorating entries with kubernetes metadata type K8sMetadataDecorator struct { helper.TransformerOperator podNameField entry.Field @@ -89,13 +89,13 @@ func (m *MetadataCache) Store(key string, entry MetadataCacheEntry) { m.m.Store(key, entry) } -// Start will start the k8s_metadata_decorator plugin +// Start will start the k8s_metadata_decorator operator func (k *K8sMetadataDecorator) Start() error { config, err := rest.InClusterConfig() if err != nil { return errors.NewError( "agent not in kubernetes cluster", - "the k8s_metadata_decorator plugin only supports running in a pod inside a kubernetes cluster", + "the k8s_metadata_decorator operator only supports running in a pod inside a kubernetes cluster", ) } @@ -126,7 +126,7 @@ func (k *K8sMetadataDecorator) Start() error { return nil } -// Process will process an entry received by the k8s_metadata_decorator plugin +// Process will process an entry received by the k8s_metadata_decorator operator func (k *K8sMetadataDecorator) Process(ctx context.Context, entry *entry.Entry) error { var podName string err := entry.Read(k.podNameField, &podName) diff --git a/plugin/builtin/transformer/k8s_metadata_decorator_test.go b/operator/builtin/transformer/k8s_metadata_decorator_test.go similarity index 87% rename from plugin/builtin/transformer/k8s_metadata_decorator_test.go rename to operator/builtin/transformer/k8s_metadata_decorator_test.go index 883a6d3e9..076720ced 100644 --- a/plugin/builtin/transformer/k8s_metadata_decorator_test.go +++ b/operator/builtin/transformer/k8s_metadata_decorator_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -31,7 +31,7 @@ func TestMetadataCache(t *testing.T) { } func basicConfig() *K8sMetadataDecoratorConfig { - cfg := NewK8smetadataDecoratorConfig("testplugin") + cfg := NewK8smetadataDecoratorConfig("testoperator") cfg.OutputIDs = []string{"mock"} return cfg } @@ -43,7 +43,7 @@ func TestK8sMetadataDecoratorBuildDefault(t *testing.T) { TransformerOperator: helper.TransformerOperator{ WriterOperator: helper.WriterOperator{ BasicOperator: helper.BasicOperator{ - OperatorID: "testplugin", + OperatorID: "testoperator", OperatorType: "k8s_metadata_decorator", }, OutputIDs: []string{"mock"}, @@ -55,10 +55,10 @@ func TestK8sMetadataDecoratorBuildDefault(t *testing.T) { cacheTTL: 10 * time.Minute, } - plugin, err := cfg.Build(testutil.NewBuildContext(t)) - plugin.(*K8sMetadataDecorator).SugaredLogger = nil + operator, err := cfg.Build(testutil.NewBuildContext(t)) + operator.(*K8sMetadataDecorator).SugaredLogger = nil require.NoError(t, err) - require.Equal(t, expected, plugin) + require.Equal(t, expected, operator) } @@ -69,7 +69,7 @@ func TestK8sMetadataDecoratorCachedMetadata(t *testing.T) { require.NoError(t, err) mockOutput := testutil.NewMockOperator("mock") - pg.SetOutputs([]plugin.Operator{mockOutput}) + pg.SetOutputs([]operator.Operator{mockOutput}) // Preload cache so we don't hit the network k8s := pg.(*K8sMetadataDecorator) diff --git a/plugin/builtin/transformer/metadata.go b/operator/builtin/transformer/metadata.go similarity index 81% rename from plugin/builtin/transformer/metadata.go rename to operator/builtin/transformer/metadata.go index 7a3ba2574..9aaab64be 100644 --- a/plugin/builtin/transformer/metadata.go +++ b/operator/builtin/transformer/metadata.go @@ -5,21 +5,21 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("metadata", func() plugin.Builder { return NewMetadataOperatorConfig("") }) + operator.Register("metadata", func() operator.Builder { return NewMetadataOperatorConfig("") }) } -func NewMetadataOperatorConfig(pluginID string) *MetadataOperatorConfig { +func NewMetadataOperatorConfig(operatorID string) *MetadataOperatorConfig { return &MetadataOperatorConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "metadata"), + TransformerConfig: helper.NewTransformerConfig(operatorID, "metadata"), } } -// MetadataOperatorConfig is the configuration of a metadata plugin +// MetadataOperatorConfig is the configuration of a metadata operator type MetadataOperatorConfig struct { helper.TransformerConfig `yaml:",inline"` @@ -27,8 +27,8 @@ type MetadataOperatorConfig struct { Tags []helper.ExprStringConfig `json:"tags" yaml:"tags"` } -// Build will build a metadata plugin from the supplied configuration -func (c MetadataOperatorConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a metadata operator from the supplied configuration +func (c MetadataOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err @@ -53,7 +53,7 @@ func (c MetadataOperatorConfig) Build(context plugin.BuildContext) (plugin.Opera return restructureOperator, nil } -// MetadataOperator is a plugin that can add metadata to incoming entries +// MetadataOperator is a operator that can add metadata to incoming entries type MetadataOperator struct { helper.TransformerOperator labeler *labeler diff --git a/plugin/builtin/transformer/metadata_test.go b/operator/builtin/transformer/metadata_test.go similarity index 94% rename from plugin/builtin/transformer/metadata_test.go rename to operator/builtin/transformer/metadata_test.go index 9ea97cf82..2bb5c5280 100644 --- a/plugin/builtin/transformer/metadata_test.go +++ b/operator/builtin/transformer/metadata_test.go @@ -8,8 +8,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -19,7 +19,7 @@ func TestMetadata(t *testing.T) { defer os.Unsetenv("TEST_METADATA_PLUGIN_ENV") baseConfig := func() *MetadataOperatorConfig { - cfg := NewMetadataOperatorConfig("test_plugin_id") + cfg := NewMetadataOperatorConfig("test_operator_id") cfg.OutputIDs = []string{"output1"} return cfg } @@ -139,7 +139,7 @@ func TestMetadata(t *testing.T) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = metadataOperator.SetOutputs([]plugin.Operator{mockOutput}) + err = metadataOperator.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) err = metadataOperator.Process(context.Background(), tc.input) diff --git a/plugin/builtin/transformer/noop.go b/operator/builtin/transformer/noop.go similarity index 53% rename from plugin/builtin/transformer/noop.go rename to operator/builtin/transformer/noop.go index e477cc4f1..4221937ca 100644 --- a/plugin/builtin/transformer/noop.go +++ b/operator/builtin/transformer/noop.go @@ -4,27 +4,27 @@ import ( "context" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("noop", func() plugin.Builder { return NewNoopOperatorConfig("") }) + operator.Register("noop", func() operator.Builder { return NewNoopOperatorConfig("") }) } -func NewNoopOperatorConfig(pluginID string) *NoopOperatorConfig { +func NewNoopOperatorConfig(operatorID string) *NoopOperatorConfig { return &NoopOperatorConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "noop"), + TransformerConfig: helper.NewTransformerConfig(operatorID, "noop"), } } -// NoopOperatorConfig is the configuration of a noop plugin. +// NoopOperatorConfig is the configuration of a noop operator. type NoopOperatorConfig struct { helper.TransformerConfig `yaml:",inline"` } -// Build will build a noop plugin. -func (c NoopOperatorConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a noop operator. +func (c NoopOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err @@ -37,7 +37,7 @@ func (c NoopOperatorConfig) Build(context plugin.BuildContext) (plugin.Operator, return noopOperator, nil } -// NoopOperator is a plugin that performs no operations on an entry. +// NoopOperator is a operator that performs no operations on an entry. type NoopOperator struct { helper.TransformerOperator } diff --git a/plugin/builtin/transformer/noop_test.go b/operator/builtin/transformer/noop_test.go similarity index 85% rename from plugin/builtin/transformer/noop_test.go rename to operator/builtin/transformer/noop_test.go index cb56c07ba..ba61bff60 100644 --- a/plugin/builtin/transformer/noop_test.go +++ b/operator/builtin/transformer/noop_test.go @@ -8,7 +8,7 @@ import ( ) func TestNoopOperatorBuild(t *testing.T) { - cfg := NewNoopOperatorConfig("test_plugin_id") + cfg := NewNoopOperatorConfig("test_operator_id") cfg.OutputIDs = []string{"output"} _, err := cfg.Build(testutil.NewBuildContext(t)) diff --git a/plugin/builtin/transformer/rate_limit.go b/operator/builtin/transformer/rate_limit.go similarity index 65% rename from plugin/builtin/transformer/rate_limit.go rename to operator/builtin/transformer/rate_limit.go index 7c9737f1d..657b352d1 100644 --- a/plugin/builtin/transformer/rate_limit.go +++ b/operator/builtin/transformer/rate_limit.go @@ -6,31 +6,31 @@ import ( "time" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("rate_limit", func() plugin.Builder { return NewRateLimitConfig("") }) + operator.Register("rate_limit", func() operator.Builder { return NewRateLimitConfig("") }) } -func NewRateLimitConfig(pluginID string) *RateLimitConfig { +func NewRateLimitConfig(operatorID string) *RateLimitConfig { return &RateLimitConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "rate_limit"), + TransformerConfig: helper.NewTransformerConfig(operatorID, "rate_limit"), } } -// RateLimitConfig is the configuration of a rate filter plugin. +// RateLimitConfig is the configuration of a rate filter operator. type RateLimitConfig struct { helper.TransformerConfig `yaml:",inline"` - Rate float64 `json:"rate,omitempty" yaml:"rate,omitempty"` - Interval plugin.Duration `json:"interval,omitempty" yaml:"interval,omitempty"` - Burst uint `json:"burst,omitempty" yaml:"burst,omitempty"` + Rate float64 `json:"rate,omitempty" yaml:"rate,omitempty"` + Interval operator.Duration `json:"interval,omitempty" yaml:"interval,omitempty"` + Burst uint `json:"burst,omitempty" yaml:"burst,omitempty"` } -// Build will build a rate limit plugin. -func (c RateLimitConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a rate limit operator. +func (c RateLimitConfig) Build(context operator.BuildContext) (operator.Operator, error) { transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err @@ -57,7 +57,7 @@ func (c RateLimitConfig) Build(context plugin.BuildContext) (plugin.Operator, er return rateLimitOperator, nil } -// RateLimitOperator is a plugin that limits the rate of log consumption between plugins. +// RateLimitOperator is a operator that limits the rate of log consumption between operators. type RateLimitOperator struct { helper.TransformerOperator @@ -74,7 +74,7 @@ func (p *RateLimitOperator) Process(ctx context.Context, entry *entry.Entry) err return nil } -// Start will start the rate limit plugin. +// Start will start the rate limit operator. func (p *RateLimitOperator) Start() error { p.isReady = make(chan struct{}, p.burst) ticker := time.NewTicker(p.interval) @@ -99,7 +99,7 @@ func (p *RateLimitOperator) Start() error { return nil } -// Stop will stop the rate limit plugin. +// Stop will stop the rate limit operator. func (p *RateLimitOperator) Stop() error { p.cancel() return nil diff --git a/plugin/builtin/transformer/rate_limit_test.go b/operator/builtin/transformer/rate_limit_test.go similarity index 92% rename from plugin/builtin/transformer/rate_limit_test.go rename to operator/builtin/transformer/rate_limit_test.go index 6ac994850..859dd75d6 100644 --- a/plugin/builtin/transformer/rate_limit_test.go +++ b/operator/builtin/transformer/rate_limit_test.go @@ -8,7 +8,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -30,7 +30,7 @@ func TestRateLimit(t *testing.T) { receivedLog <- struct{}{} }) - err = rateLimit.SetOutputs([]plugin.Operator{mockOutput}) + err = rateLimit.SetOutputs([]operator.Operator{mockOutput}) require.NoError(t, err) err = rateLimit.Start() diff --git a/plugin/builtin/transformer/restructure.go b/operator/builtin/transformer/restructure.go similarity index 94% rename from plugin/builtin/transformer/restructure.go rename to operator/builtin/transformer/restructure.go index 3605b2d3b..c6f603f9d 100644 --- a/plugin/builtin/transformer/restructure.go +++ b/operator/builtin/transformer/restructure.go @@ -9,29 +9,29 @@ import ( "github.com/antonmedv/expr/vm" "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" ) func init() { - plugin.Register("restructure", func() plugin.Builder { return NewRestructureOperatorConfig("") }) + operator.Register("restructure", func() operator.Builder { return NewRestructureOperatorConfig("") }) } -func NewRestructureOperatorConfig(pluginID string) *RestructureOperatorConfig { +func NewRestructureOperatorConfig(operatorID string) *RestructureOperatorConfig { return &RestructureOperatorConfig{ - TransformerConfig: helper.NewTransformerConfig(pluginID, "restructure"), + TransformerConfig: helper.NewTransformerConfig(operatorID, "restructure"), } } -// RestructureOperatorConfig is the configuration of a restructure plugin +// RestructureOperatorConfig is the configuration of a restructure operator type RestructureOperatorConfig struct { helper.TransformerConfig `yaml:",inline"` Ops []Op `json:"ops" yaml:"ops"` } -// Build will build a restructure plugin from the supplied configuration -func (c RestructureOperatorConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a restructure operator from the supplied configuration +func (c RestructureOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return nil, err @@ -45,7 +45,7 @@ func (c RestructureOperatorConfig) Build(context plugin.BuildContext) (plugin.Op return restructureOperator, nil } -// RestructureOperator is a plugin that can restructure incoming entries using operations +// RestructureOperator is a operator that can restructure incoming entries using operations type RestructureOperator struct { helper.TransformerOperator ops []Op diff --git a/plugin/builtin/transformer/restructure_test.go b/operator/builtin/transformer/restructure_test.go similarity index 95% rename from plugin/builtin/transformer/restructure_test.go rename to operator/builtin/transformer/restructure_test.go index 0e30e34ca..78656b80d 100644 --- a/plugin/builtin/transformer/restructure_test.go +++ b/operator/builtin/transformer/restructure_test.go @@ -11,8 +11,8 @@ import ( "github.com/antonmedv/expr/vm" "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -30,7 +30,7 @@ func NewFakeRestructureOperator() (*RestructureOperator, *testutil.Operator) { OperatorType: "restructure", SugaredLogger: logger.Sugar(), }, - OutputOperators: []plugin.Operator{&mock}, + OutputOperators: []operator.Operator{&mock}, }, }, }, &mock @@ -202,14 +202,14 @@ func TestRestructureOperator(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - plugin, mockOutput := NewFakeRestructureOperator() - plugin.ops = tc.ops + operator, mockOutput := NewFakeRestructureOperator() + operator.ops = tc.ops mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { require.Equal(t, tc.output, args[1].(*entry.Entry)) }).Return(nil) - err := plugin.Process(context.Background(), tc.input) + err := operator.Process(context.Background(), tc.input) require.NoError(t, err) }) } @@ -342,7 +342,7 @@ ops: }] }` - expected := plugin.Config(plugin.Config{ + expected := operator.Config(operator.Config{ Builder: &RestructureOperatorConfig{ TransformerConfig: helper.TransformerConfig{ WriterConfig: helper.WriterConfig{ @@ -392,12 +392,12 @@ ops: }, }) - var unmarshalledYAML plugin.Config + var unmarshalledYAML operator.Config err := yaml.UnmarshalStrict([]byte(configYAML), &unmarshalledYAML) require.NoError(t, err) require.Equal(t, expected, unmarshalledYAML) - var unmarshalledJSON plugin.Config + var unmarshalledJSON operator.Config err = json.Unmarshal([]byte(configJSON), &unmarshalledJSON) require.NoError(t, err) require.Equal(t, expected, unmarshalledJSON) diff --git a/plugin/builtin/transformer/router.go b/operator/builtin/transformer/router.go similarity index 59% rename from plugin/builtin/transformer/router.go rename to operator/builtin/transformer/router.go index eecd0ffe7..3b4205775 100644 --- a/plugin/builtin/transformer/router.go +++ b/operator/builtin/transformer/router.go @@ -7,35 +7,35 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "go.uber.org/zap" ) func init() { - plugin.Register("router", func() plugin.Builder { return NewRouterOperatorConfig("") }) + operator.Register("router", func() operator.Builder { return NewRouterOperatorConfig("") }) } -func NewRouterOperatorConfig(pluginID string) *RouterOperatorConfig { +func NewRouterOperatorConfig(operatorID string) *RouterOperatorConfig { return &RouterOperatorConfig{ - BasicConfig: helper.NewBasicConfig(pluginID, "router"), + BasicConfig: helper.NewBasicConfig(operatorID, "router"), } } -// RouterOperatorConfig is the configuration of a router plugin +// RouterOperatorConfig is the configuration of a router operator type RouterOperatorConfig struct { helper.BasicConfig `yaml:",inline"` Routes []*RouterOperatorRouteConfig `json:"routes" yaml:"routes"` } -// RouterOperatorRouteConfig is the configuration of a route on a router plugin +// RouterOperatorRouteConfig is the configuration of a route on a router operator type RouterOperatorRouteConfig struct { Expression string `json:"expr" yaml:"expr"` OutputIDs helper.OutputIDs `json:"output" yaml:"output"` } -// Build will build a router plugin from the supplied configuration -func (c RouterOperatorConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { +// Build will build a router operator from the supplied configuration +func (c RouterOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { basicOperator, err := c.BasicConfig.Build(context) if err != nil { return nil, err @@ -62,7 +62,7 @@ func (c RouterOperatorConfig) Build(context plugin.BuildContext) (plugin.Operato return routerOperator, nil } -// SetNamespace will namespace the router plugin and the outputs contained in its routes +// SetNamespace will namespace the router operator and the outputs contained in its routes func (c *RouterOperatorConfig) SetNamespace(namespace string, exclusions ...string) { c.BasicConfig.SetNamespace(namespace, exclusions...) for _, route := range c.Routes { @@ -74,20 +74,20 @@ func (c *RouterOperatorConfig) SetNamespace(namespace string, exclusions ...stri } } -// RouterOperator is a plugin that routes entries based on matching expressions +// RouterOperator is a operator that routes entries based on matching expressions type RouterOperator struct { helper.BasicOperator routes []*RouterOperatorRoute } -// RouterOperatorRoute is a route on a router plugin +// RouterOperatorRoute is a route on a router operator type RouterOperatorRoute struct { Expression *vm.Program OutputIDs helper.OutputIDs - OutputOperators []plugin.Operator + OutputOperators []operator.Operator } -// CanProcess will always return true for a router plugin +// CanProcess will always return true for a router operator func (p *RouterOperator) CanProcess() bool { return true } @@ -116,24 +116,24 @@ func (p *RouterOperator) Process(ctx context.Context, entry *entry.Entry) error return nil } -// CanOutput will always return true for a router plugin +// CanOutput will always return true for a router operator func (p *RouterOperator) CanOutput() bool { return true } -// Outputs will return all connected plugins. -func (p *RouterOperator) Outputs() []plugin.Operator { - outputs := make([]plugin.Operator, 0, len(p.routes)) +// Outputs will return all connected operators. +func (p *RouterOperator) Outputs() []operator.Operator { + outputs := make([]operator.Operator, 0, len(p.routes)) for _, route := range p.routes { outputs = append(outputs, route.OutputOperators...) } return outputs } -// SetOutputs will set the outputs of the router plugin. -func (p *RouterOperator) SetOutputs(plugins []plugin.Operator) error { +// SetOutputs will set the outputs of the router operator. +func (p *RouterOperator) SetOutputs(operators []operator.Operator) error { for _, route := range p.routes { - outputOperators, err := p.findOperators(plugins, route.OutputIDs) + outputOperators, err := p.findOperators(operators, route.OutputIDs) if err != nil { return fmt.Errorf("failed to set outputs on route: %s", err) } @@ -142,25 +142,25 @@ func (p *RouterOperator) SetOutputs(plugins []plugin.Operator) error { return nil } -// findOperators will find a subset of plugins from a collection. -func (p *RouterOperator) findOperators(plugins []plugin.Operator, pluginIDs []string) ([]plugin.Operator, error) { - result := make([]plugin.Operator, 0) - for _, pluginID := range pluginIDs { - plugin, err := p.findOperator(plugins, pluginID) +// findOperators will find a subset of operators from a collection. +func (p *RouterOperator) findOperators(operators []operator.Operator, operatorIDs []string) ([]operator.Operator, error) { + result := make([]operator.Operator, 0) + for _, operatorID := range operatorIDs { + operator, err := p.findOperator(operators, operatorID) if err != nil { return nil, err } - result = append(result, plugin) + result = append(result, operator) } return result, nil } -// findOperator will find a plugin from a collection. -func (p *RouterOperator) findOperator(plugins []plugin.Operator, pluginID string) (plugin.Operator, error) { - for _, plugin := range plugins { - if plugin.ID() == pluginID { - return plugin, nil +// findOperator will find a operator from a collection. +func (p *RouterOperator) findOperator(operators []operator.Operator, operatorID string) (operator.Operator, error) { + for _, operator := range operators { + if operator.ID() == operatorID { + return operator, nil } } - return nil, fmt.Errorf("plugin %s does not exist", pluginID) + return nil, fmt.Errorf("operator %s does not exist", operatorID) } diff --git a/plugin/builtin/transformer/router_test.go b/operator/builtin/transformer/router_test.go similarity index 92% rename from plugin/builtin/transformer/router_test.go rename to operator/builtin/transformer/router_test.go index ca142517c..a0f0f4abb 100644 --- a/plugin/builtin/transformer/router_test.go +++ b/operator/builtin/transformer/router_test.go @@ -7,8 +7,8 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -20,7 +20,7 @@ func TestRouterOperator(t *testing.T) { basicConfig := func() *RouterOperatorConfig { return &RouterOperatorConfig{ BasicConfig: helper.BasicConfig{ - OperatorID: "test_plugin_id", + OperatorID: "test_operator_id", OperatorType: "router", }, } @@ -115,7 +115,7 @@ func TestRouterOperator(t *testing.T) { }) routerOperator := newOperator.(*RouterOperator) - err = routerOperator.SetOutputs([]plugin.Operator{mock1, mock2}) + err = routerOperator.SetOutputs([]operator.Operator{mock1, mock2}) require.NoError(t, err) err = routerOperator.Process(context.Background(), tc.input) diff --git a/plugin/config.go b/operator/config.go similarity index 99% rename from plugin/config.go rename to operator/config.go index 8515005cd..114207f98 100644 --- a/plugin/config.go +++ b/operator/config.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "encoding/json" diff --git a/plugin/config_test.go b/operator/config_test.go similarity index 99% rename from plugin/config_test.go rename to operator/config_test.go index 7f0e76db7..00848e0a7 100644 --- a/plugin/config_test.go +++ b/operator/config_test.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "encoding/json" diff --git a/plugin/custom.go b/operator/custom.go similarity index 99% rename from plugin/custom.go rename to operator/custom.go index bd336dec1..25aa5d760 100644 --- a/plugin/custom.go +++ b/operator/custom.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "bytes" diff --git a/plugin/custom_test.go b/operator/custom_test.go similarity index 99% rename from plugin/custom_test.go rename to operator/custom_test.go index 6472553ae..5532b909d 100644 --- a/plugin/custom_test.go +++ b/operator/custom_test.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "io/ioutil" diff --git a/plugin/duration.go b/operator/duration.go similarity index 98% rename from plugin/duration.go rename to operator/duration.go index c22f34625..e53d8dbbc 100644 --- a/plugin/duration.go +++ b/operator/duration.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "encoding/json" diff --git a/plugin/duration_test.go b/operator/duration_test.go similarity index 98% rename from plugin/duration_test.go rename to operator/duration_test.go index d4711691e..8c5c85d3d 100644 --- a/plugin/duration_test.go +++ b/operator/duration_test.go @@ -1,4 +1,4 @@ -package plugin +package operator import ( "encoding/json" diff --git a/plugin/helper/expr_string.go b/operator/helper/expr_string.go similarity index 100% rename from plugin/helper/expr_string.go rename to operator/helper/expr_string.go diff --git a/plugin/helper/expr_string_test.go b/operator/helper/expr_string_test.go similarity index 100% rename from plugin/helper/expr_string_test.go rename to operator/helper/expr_string_test.go diff --git a/plugin/helper/input.go b/operator/helper/input.go similarity index 65% rename from plugin/helper/input.go rename to operator/helper/input.go index 50826de03..70ed69eee 100644 --- a/plugin/helper/input.go +++ b/operator/helper/input.go @@ -5,28 +5,28 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "go.uber.org/zap" ) -func NewInputConfig(pluginID, pluginType string) InputConfig { +func NewInputConfig(operatorID, operatorType string) InputConfig { return InputConfig{ - WriterConfig: NewWriterConfig(pluginID, pluginType), + WriterConfig: NewWriterConfig(operatorID, operatorType), WriteTo: entry.NewRecordField(), } } -// InputConfig provides a basic implementation of an input plugin config. +// InputConfig provides a basic implementation of an input operator config. type InputConfig struct { WriterConfig `yaml:",inline"` WriteTo entry.Field `json:"write_to" yaml:"write_to"` } // Build will build a base producer. -func (c InputConfig) Build(context plugin.BuildContext) (InputOperator, error) { +func (c InputConfig) Build(context operator.BuildContext) (InputOperator, error) { writerOperator, err := c.WriterConfig.Build(context) if err != nil { - return InputOperator{}, errors.WithDetails(err, "plugin_id", c.ID()) + return InputOperator{}, errors.WithDetails(err, "operator_id", c.ID()) } inputOperator := InputOperator{ @@ -37,7 +37,7 @@ func (c InputConfig) Build(context plugin.BuildContext) (InputOperator, error) { return inputOperator, nil } -// InputOperator provides a basic implementation of an input plugin. +// InputOperator provides a basic implementation of an input operator. type InputOperator struct { WriterOperator WriteTo entry.Field @@ -50,7 +50,7 @@ func (i *InputOperator) NewEntry(value interface{}) *entry.Entry { return entry } -// CanProcess will always return false for an input plugin. +// CanProcess will always return false for an input operator. func (i *InputOperator) CanProcess() bool { return false } @@ -60,6 +60,6 @@ func (i *InputOperator) Process(ctx context.Context, entry *entry.Entry) error { i.Errorw("Operator received an entry, but can not process", zap.Any("entry", entry)) return errors.NewError( "Operator can not process logs.", - "Ensure that plugin is not configured to receive logs from other plugins", + "Ensure that operator is not configured to receive logs from other operators", ) } diff --git a/plugin/helper/input_test.go b/operator/helper/input_test.go similarity index 100% rename from plugin/helper/input_test.go rename to operator/helper/input_test.go diff --git a/plugin/helper/namespace.go b/operator/helper/namespace.go similarity index 100% rename from plugin/helper/namespace.go rename to operator/helper/namespace.go diff --git a/plugin/helper/output.go b/operator/helper/output.go similarity index 51% rename from plugin/helper/output.go rename to operator/helper/output.go index b35c6dacb..b76558f02 100644 --- a/plugin/helper/output.go +++ b/operator/helper/output.go @@ -2,22 +2,22 @@ package helper import ( "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" ) -func NewOutputConfig(pluginID, pluginType string) OutputConfig { +func NewOutputConfig(operatorID, operatorType string) OutputConfig { return OutputConfig{ - BasicConfig: NewBasicConfig(pluginID, pluginType), + BasicConfig: NewBasicConfig(operatorID, operatorType), } } -// OutputConfig provides a basic implementation of an output plugin config. +// OutputConfig provides a basic implementation of an output operator config. type OutputConfig struct { BasicConfig `mapstructure:",squash" yaml:",inline"` } -// Build will build an output plugin. -func (c OutputConfig) Build(context plugin.BuildContext) (OutputOperator, error) { +// Build will build an output operator. +func (c OutputConfig) Build(context operator.BuildContext) (OutputOperator, error) { basicOperator, err := c.BasicConfig.Build(context) if err != nil { return OutputOperator{}, err @@ -30,35 +30,35 @@ func (c OutputConfig) Build(context plugin.BuildContext) (OutputOperator, error) return outputOperator, nil } -// SetNamespace will namespace the id and output of the plugin config. +// SetNamespace will namespace the id and output of the operator config. func (c *OutputConfig) SetNamespace(namespace string, exclusions ...string) { if CanNamespace(c.ID(), exclusions) { c.OperatorID = AddNamespace(c.ID(), namespace) } } -// OutputOperator provides a basic implementation of an output plugin. +// OutputOperator provides a basic implementation of an output operator. type OutputOperator struct { BasicOperator } -// CanProcess will always return true for an output plugin. +// CanProcess will always return true for an output operator. func (o *OutputOperator) CanProcess() bool { return true } -// CanOutput will always return false for an output plugin. +// CanOutput will always return false for an output operator. func (o *OutputOperator) CanOutput() bool { return false } -// Outputs will always return an empty array for an output plugin. -func (o *OutputOperator) Outputs() []plugin.Operator { - return []plugin.Operator{} +// Outputs will always return an empty array for an output operator. +func (o *OutputOperator) Outputs() []operator.Operator { + return []operator.Operator{} } // SetOutputs will return an error if called. -func (o *OutputOperator) SetOutputs(plugins []plugin.Operator) error { +func (o *OutputOperator) SetOutputs(operators []operator.Operator) error { return errors.NewError( "Operator can not output, but is attempting to set an output.", "This is an unexpected internal error. Please submit a bug/issue.", diff --git a/plugin/helper/output_test.go b/operator/helper/output_test.go similarity index 93% rename from plugin/helper/output_test.go rename to operator/helper/output_test.go index 1e240f9a6..96b9c871e 100644 --- a/plugin/helper/output_test.go +++ b/operator/helper/output_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/require" ) @@ -72,7 +72,7 @@ func TestOutputOperatorOutputs(t *testing.T) { SugaredLogger: buildContext.Logger, }, } - require.Equal(t, []plugin.Operator{}, output.Outputs()) + require.Equal(t, []operator.Operator{}, output.Outputs()) } func TestOutputOperatorSetOutputs(t *testing.T) { @@ -85,7 +85,7 @@ func TestOutputOperatorSetOutputs(t *testing.T) { }, } - err := output.SetOutputs([]plugin.Operator{}) + err := output.SetOutputs([]operator.Operator{}) require.Error(t, err) require.Contains(t, err.Error(), "Operator can not output") } diff --git a/plugin/helper/parser.go b/operator/helper/parser.go similarity index 89% rename from plugin/helper/parser.go rename to operator/helper/parser.go index af1ef39c0..98dd1bc94 100644 --- a/plugin/helper/parser.go +++ b/operator/helper/parser.go @@ -5,12 +5,12 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" ) -func NewParserConfig(pluginID, pluginType string) ParserConfig { +func NewParserConfig(operatorID, operatorType string) ParserConfig { return ParserConfig{ - TransformerConfig: NewTransformerConfig(pluginID, pluginType), + TransformerConfig: NewTransformerConfig(operatorID, operatorType), ParseFrom: entry.NewRecordField(), ParseTo: entry.NewRecordField(), Preserve: false, @@ -28,8 +28,8 @@ type ParserConfig struct { SeverityParserConfig *SeverityParserConfig `json:"severity,omitempty" yaml:"severity,omitempty"` } -// Build will build a parser plugin. -func (c ParserConfig) Build(context plugin.BuildContext) (ParserOperator, error) { +// Build will build a parser operator. +func (c ParserConfig) Build(context operator.BuildContext) (ParserOperator, error) { transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { return ParserOperator{}, err @@ -39,7 +39,7 @@ func (c ParserConfig) Build(context plugin.BuildContext) (ParserOperator, error) transformerOperator.Warnw( "preserve is true, but parse_to is set to the same field as parse_from, "+ "which will cause the original value to be overwritten", - "plugin_id", c.ID(), + "operator_id", c.ID(), ) } @@ -68,7 +68,7 @@ func (c ParserConfig) Build(context plugin.BuildContext) (ParserOperator, error) return parserOperator, nil } -// ParserOperator provides a basic implementation of a parser plugin. +// ParserOperator provides a basic implementation of a parser operator. type ParserOperator struct { TransformerOperator ParseFrom entry.Field diff --git a/plugin/helper/parser_test.go b/operator/helper/parser_test.go similarity index 98% rename from plugin/helper/parser_test.go rename to operator/helper/parser_test.go index 98a3a7023..9dbc4fd30 100644 --- a/plugin/helper/parser_test.go +++ b/operator/helper/parser_test.go @@ -8,7 +8,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" @@ -259,7 +259,7 @@ func TestParserOutput(t *testing.T) { OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputOperators: []plugin.Operator{output}, + OutputOperators: []operator.Operator{output}, }, }, ParseFrom: entry.NewRecordField(), @@ -289,7 +289,7 @@ func TestParserWithPreserve(t *testing.T) { OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputOperators: []plugin.Operator{output}, + OutputOperators: []operator.Operator{output}, }, }, ParseFrom: entry.NewRecordField("parse_from"), @@ -328,7 +328,7 @@ func TestParserWithoutPreserve(t *testing.T) { OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputOperators: []plugin.Operator{output}, + OutputOperators: []operator.Operator{output}, }, }, ParseFrom: entry.NewRecordField("parse_from"), diff --git a/plugin/helper/persister.go b/operator/helper/persister.go similarity index 93% rename from plugin/helper/persister.go rename to operator/helper/persister.go index 4c19d9071..fea266f72 100644 --- a/plugin/helper/persister.go +++ b/operator/helper/persister.go @@ -3,7 +3,7 @@ package helper import ( "sync" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "go.etcd.io/bbolt" ) @@ -18,13 +18,13 @@ type Persister interface { // ScopedBBoltPersister is a persister that uses a database for the backend type ScopedBBoltPersister struct { scope []byte - db plugin.Database + db operator.Database cache map[string][]byte cacheMux sync.Mutex } // NewScopedDBPersister returns a new ScopedBBoltPersister -func NewScopedDBPersister(db plugin.Database, scope string) *ScopedBBoltPersister { +func NewScopedDBPersister(db operator.Database, scope string) *ScopedBBoltPersister { return &ScopedBBoltPersister{ scope: []byte(scope), db: db, diff --git a/plugin/helper/plugin.go b/operator/helper/plugin.go similarity index 56% rename from plugin/helper/plugin.go rename to operator/helper/plugin.go index b11239b94..4d1333242 100644 --- a/plugin/helper/plugin.go +++ b/operator/helper/plugin.go @@ -2,24 +2,24 @@ package helper import ( "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "go.uber.org/zap" ) -func NewBasicConfig(pluginID, pluginType string) BasicConfig { +func NewBasicConfig(operatorID, operatorType string) BasicConfig { return BasicConfig{ - OperatorID: pluginID, - OperatorType: pluginType, + OperatorID: operatorID, + OperatorType: operatorType, } } -// BasicConfig provides a basic implemention for a plugin config. +// BasicConfig provides a basic implemention for a operator config. type BasicConfig struct { OperatorID string `json:"id" yaml:"id"` OperatorType string `json:"type" yaml:"type"` } -// ID will return the plugin id. +// ID will return the operator id. func (c BasicConfig) ID() string { if c.OperatorID == "" { return c.OperatorType @@ -27,54 +27,54 @@ func (c BasicConfig) ID() string { return c.OperatorID } -// Type will return the plugin type. +// Type will return the operator type. func (c BasicConfig) Type() string { return c.OperatorType } -// Build will build a basic plugin. -func (c BasicConfig) Build(context plugin.BuildContext) (BasicOperator, error) { +// Build will build a basic operator. +func (c BasicConfig) Build(context operator.BuildContext) (BasicOperator, error) { if c.OperatorType == "" { return BasicOperator{}, errors.NewError( "missing required `type` field.", - "ensure that all plugins have a uniquely defined `type` field.", - "plugin_id", c.ID(), + "ensure that all operators have a uniquely defined `type` field.", + "operator_id", c.ID(), ) } if context.Logger == nil { return BasicOperator{}, errors.NewError( - "plugin build context is missing a logger.", + "operator build context is missing a logger.", "this is an unexpected internal error", - "plugin_id", c.ID(), - "plugin_type", c.Type(), + "operator_id", c.ID(), + "operator_type", c.Type(), ) } - plugin := BasicOperator{ + operator := BasicOperator{ OperatorID: c.ID(), OperatorType: c.Type(), - SugaredLogger: context.Logger.With("plugin_id", c.ID(), "plugin_type", c.Type()), + SugaredLogger: context.Logger.With("operator_id", c.ID(), "operator_type", c.Type()), } - return plugin, nil + return operator, nil } -// SetNamespace will namespace the plugin id. +// SetNamespace will namespace the operator id. func (c *BasicConfig) SetNamespace(namespace string, exclusions ...string) { if CanNamespace(c.ID(), exclusions) { c.OperatorID = AddNamespace(c.ID(), namespace) } } -// BasicOperator provides a basic implementation of a plugin. +// BasicOperator provides a basic implementation of a operator. type BasicOperator struct { OperatorID string OperatorType string *zap.SugaredLogger } -// ID will return the plugin id. +// ID will return the operator id. func (p *BasicOperator) ID() string { if p.OperatorID == "" { return p.OperatorType @@ -82,22 +82,22 @@ func (p *BasicOperator) ID() string { return p.OperatorID } -// Type will return the plugin type. +// Type will return the operator type. func (p *BasicOperator) Type() string { return p.OperatorType } -// Logger returns the plugin's scoped logger. +// Logger returns the operator's scoped logger. func (p *BasicOperator) Logger() *zap.SugaredLogger { return p.SugaredLogger } -// Start will start the plugin. +// Start will start the operator. func (p *BasicOperator) Start() error { return nil } -// Stop will stop the plugin. +// Stop will stop the operator. func (p *BasicOperator) Stop() error { return nil } diff --git a/plugin/helper/plugin_test.go b/operator/helper/plugin_test.go similarity index 74% rename from plugin/helper/plugin_test.go rename to operator/helper/plugin_test.go index d1b352821..715384122 100644 --- a/plugin/helper/plugin_test.go +++ b/operator/helper/plugin_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -38,7 +38,7 @@ func TestBasicConfigBuildWithoutType(t *testing.T) { config := BasicConfig{ OperatorID: "test-id", } - context := plugin.BuildContext{} + context := operator.BuildContext{} _, err := config.Build(context) require.Error(t, err) require.Contains(t, err.Error(), "missing required `type` field.") @@ -49,10 +49,10 @@ func TestBasicConfigBuildMissingLogger(t *testing.T) { OperatorID: "test-id", OperatorType: "test-type", } - context := plugin.BuildContext{} + context := operator.BuildContext{} _, err := config.Build(context) require.Error(t, err) - require.Contains(t, err.Error(), "plugin build context is missing a logger.") + require.Contains(t, err.Error(), "operator build context is missing a logger.") } func TestBasicConfigBuildValid(t *testing.T) { @@ -61,52 +61,52 @@ func TestBasicConfigBuildValid(t *testing.T) { OperatorType: "test-type", } context := testutil.NewBuildContext(t) - plugin, err := config.Build(context) + operator, err := config.Build(context) require.NoError(t, err) - require.Equal(t, "test-id", plugin.OperatorID) - require.Equal(t, "test-type", plugin.OperatorType) + require.Equal(t, "test-id", operator.OperatorID) + require.Equal(t, "test-type", operator.OperatorType) } func TestBasicOperatorID(t *testing.T) { - plugin := BasicOperator{ + operator := BasicOperator{ OperatorID: "test-id", OperatorType: "test-type", } - require.Equal(t, "test-id", plugin.ID()) + require.Equal(t, "test-id", operator.ID()) } func TestBasicOperatorType(t *testing.T) { - plugin := BasicOperator{ + operator := BasicOperator{ OperatorID: "test-id", OperatorType: "test-type", } - require.Equal(t, "test-type", plugin.Type()) + require.Equal(t, "test-type", operator.Type()) } func TestBasicOperatorLogger(t *testing.T) { logger := &zap.SugaredLogger{} - plugin := BasicOperator{ + operator := BasicOperator{ OperatorID: "test-id", OperatorType: "test-type", SugaredLogger: logger, } - require.Equal(t, logger, plugin.Logger()) + require.Equal(t, logger, operator.Logger()) } func TestBasicOperatorStart(t *testing.T) { - plugin := BasicOperator{ + operator := BasicOperator{ OperatorID: "test-id", OperatorType: "test-type", } - err := plugin.Start() + err := operator.Start() require.NoError(t, err) } func TestBasicOperatorStop(t *testing.T) { - plugin := BasicOperator{ + operator := BasicOperator{ OperatorID: "test-id", OperatorType: "test-type", } - err := plugin.Stop() + err := operator.Stop() require.NoError(t, err) } diff --git a/plugin/helper/severity.go b/operator/helper/severity.go similarity index 100% rename from plugin/helper/severity.go rename to operator/helper/severity.go diff --git a/plugin/helper/severity_builder.go b/operator/helper/severity_builder.go similarity index 94% rename from plugin/helper/severity_builder.go rename to operator/helper/severity_builder.go index 228b5e4ba..295700183 100644 --- a/plugin/helper/severity_builder.go +++ b/operator/helper/severity_builder.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" ) const minSeverity = 0 @@ -73,8 +73,8 @@ type SeverityParserConfig struct { } // Build builds a SeverityParser from a SeverityParserConfig -func (c *SeverityParserConfig) Build(context plugin.BuildContext) (SeverityParser, error) { - pluginMapping := getBuiltinMapping(c.Preset) +func (c *SeverityParserConfig) Build(context operator.BuildContext) (SeverityParser, error) { + operatorMapping := getBuiltinMapping(c.Preset) for severity, unknown := range c.Mapping { sev, err := validateSeverity(severity) @@ -89,14 +89,14 @@ func (c *SeverityParserConfig) Build(context plugin.BuildContext) (SeverityParse if err != nil { return SeverityParser{}, err } - pluginMapping.add(sev, v...) + operatorMapping.add(sev, v...) } case interface{}: v, err := parseableValues(u) if err != nil { return SeverityParser{}, err } - pluginMapping.add(sev, v...) + operatorMapping.add(sev, v...) } } @@ -107,7 +107,7 @@ func (c *SeverityParserConfig) Build(context plugin.BuildContext) (SeverityParse p := SeverityParser{ ParseFrom: *c.ParseFrom, Preserve: c.Preserve, - Mapping: pluginMapping, + Mapping: operatorMapping, } return p, nil diff --git a/plugin/helper/time.go b/operator/helper/time.go similarity index 97% rename from plugin/helper/time.go rename to operator/helper/time.go index a6c662878..66f525745 100644 --- a/plugin/helper/time.go +++ b/operator/helper/time.go @@ -11,7 +11,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" ) // StrptimeKey is literally "strptime", and is the default layout type @@ -24,7 +24,7 @@ const GotimeKey = "gotime" const EpochKey = "epoch" // NativeKey is literally "native" and refers to Golang's native time.Time -const NativeKey = "native" // provided for plugin development +const NativeKey = "native" // provided for operator development func NewTimeParser() TimeParser { return TimeParser{ @@ -46,7 +46,7 @@ func (t *TimeParser) IsZero() bool { } // Validate validates a TimeParser, and reconfigures it if necessary -func (t *TimeParser) Validate(context plugin.BuildContext) error { +func (t *TimeParser) Validate(context operator.BuildContext) error { if t.ParseFrom == nil { return fmt.Errorf("missing required parameter 'parse_from'") } diff --git a/plugin/helper/time_test.go b/operator/helper/time_test.go similarity index 100% rename from plugin/helper/time_test.go rename to operator/helper/time_test.go diff --git a/plugin/helper/transformer.go b/operator/helper/transformer.go similarity index 79% rename from plugin/helper/transformer.go rename to operator/helper/transformer.go index 5713ac11c..df2b96fdf 100644 --- a/plugin/helper/transformer.go +++ b/operator/helper/transformer.go @@ -5,13 +5,13 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "go.uber.org/zap" ) -func NewTransformerConfig(pluginID, pluginType string) TransformerConfig { +func NewTransformerConfig(operatorID, operatorType string) TransformerConfig { return TransformerConfig{ - WriterConfig: NewWriterConfig(pluginID, pluginType), + WriterConfig: NewWriterConfig(operatorID, operatorType), OnError: SendOnError, } } @@ -22,18 +22,18 @@ type TransformerConfig struct { OnError string `json:"on_error" yaml:"on_error"` } -// Build will build a transformer plugin. -func (c TransformerConfig) Build(context plugin.BuildContext) (TransformerOperator, error) { +// Build will build a transformer operator. +func (c TransformerConfig) Build(context operator.BuildContext) (TransformerOperator, error) { writerOperator, err := c.WriterConfig.Build(context) if err != nil { - return TransformerOperator{}, errors.WithDetails(err, "plugin_id", c.ID()) + return TransformerOperator{}, errors.WithDetails(err, "operator_id", c.ID()) } switch c.OnError { case SendOnError, DropOnError: default: return TransformerOperator{}, errors.NewError( - "plugin config has an invalid `on_error` field.", + "operator config has an invalid `on_error` field.", "ensure that the `on_error` field is set to either `send` or `drop`.", "on_error", c.OnError, ) @@ -47,13 +47,13 @@ func (c TransformerConfig) Build(context plugin.BuildContext) (TransformerOperat return transformerOperator, nil } -// TransformerOperator provides a basic implementation of a transformer plugin. +// TransformerOperator provides a basic implementation of a transformer operator. type TransformerOperator struct { WriterOperator OnError string } -// CanProcess will always return true for a transformer plugin. +// CanProcess will always return true for a transformer operator. func (t *TransformerOperator) CanProcess() bool { return true } diff --git a/plugin/helper/transformer_test.go b/operator/helper/transformer_test.go similarity index 94% rename from plugin/helper/transformer_test.go rename to operator/helper/transformer_test.go index ccf628a30..f9cbcb5b0 100644 --- a/plugin/helper/transformer_test.go +++ b/operator/helper/transformer_test.go @@ -7,7 +7,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -45,7 +45,7 @@ func TestTransformerOnErrorInvalid(t *testing.T) { cfg.OnError = "invalid" _, err := cfg.Build(testutil.NewBuildContext(t)) require.Error(t, err) - require.Contains(t, err.Error(), "plugin config has an invalid `on_error` field.") + require.Contains(t, err.Error(), "operator config has an invalid `on_error` field.") } func TestTransformerConfigSetNamespace(t *testing.T) { @@ -76,7 +76,7 @@ func TestTransformerDropOnError(t *testing.T) { OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputOperators: []plugin.Operator{output}, + OutputOperators: []operator.Operator{output}, OutputIDs: []string{"test-output"}, }, } @@ -104,7 +104,7 @@ func TestTransformerSendOnError(t *testing.T) { OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputOperators: []plugin.Operator{output}, + OutputOperators: []operator.Operator{output}, OutputIDs: []string{"test-output"}, }, } @@ -132,7 +132,7 @@ func TestTransformerProcessWithValid(t *testing.T) { OperatorType: "test-type", SugaredLogger: buildContext.Logger, }, - OutputOperators: []plugin.Operator{output}, + OutputOperators: []operator.Operator{output}, OutputIDs: []string{"test-output"}, }, } diff --git a/plugin/helper/writer.go b/operator/helper/writer.go similarity index 58% rename from plugin/helper/writer.go rename to operator/helper/writer.go index a79488cd6..e5be7b0c0 100644 --- a/plugin/helper/writer.go +++ b/operator/helper/writer.go @@ -6,23 +6,23 @@ import ( "fmt" "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" ) -func NewWriterConfig(pluginID, pluginType string) WriterConfig { +func NewWriterConfig(operatorID, operatorType string) WriterConfig { return WriterConfig{ - BasicConfig: NewBasicConfig(pluginID, pluginType), + BasicConfig: NewBasicConfig(operatorID, operatorType), } } -// WriterConfig is the configuration of a writer plugin. +// WriterConfig is the configuration of a writer operator. type WriterConfig struct { BasicConfig `yaml:",inline"` OutputIDs OutputIDs `json:"output" yaml:"output"` } -// Build will build a writer plugin from the config. -func (c WriterConfig) Build(context plugin.BuildContext) (WriterOperator, error) { +// Build will build a writer operator from the config. +func (c WriterConfig) Build(context operator.BuildContext) (WriterOperator, error) { basicOperator, err := c.BasicConfig.Build(context) if err != nil { return WriterOperator{}, err @@ -45,70 +45,70 @@ func (c *WriterConfig) SetNamespace(namespace string, exclusions ...string) { } } -// WriterOperator is a plugin that can write to other plugins. +// WriterOperator is a operator that can write to other operators. type WriterOperator struct { BasicOperator OutputIDs OutputIDs - OutputOperators []plugin.Operator + OutputOperators []operator.Operator } -// Write will write an entry to the outputs of the plugin. +// Write will write an entry to the outputs of the operator. func (w *WriterOperator) Write(ctx context.Context, e *entry.Entry) { - for i, plugin := range w.OutputOperators { + for i, operator := range w.OutputOperators { if i == len(w.OutputOperators)-1 { - _ = plugin.Process(ctx, e) + _ = operator.Process(ctx, e) return } - plugin.Process(ctx, e.Copy()) + operator.Process(ctx, e.Copy()) } } -// CanOutput always returns true for a writer plugin. +// CanOutput always returns true for a writer operator. func (w *WriterOperator) CanOutput() bool { return true } -// Outputs returns the outputs of the writer plugin. -func (w *WriterOperator) Outputs() []plugin.Operator { +// Outputs returns the outputs of the writer operator. +func (w *WriterOperator) Outputs() []operator.Operator { return w.OutputOperators } -// SetOutputs will set the outputs of the plugin. -func (w *WriterOperator) SetOutputs(plugins []plugin.Operator) error { - outputOperators := make([]plugin.Operator, 0) +// SetOutputs will set the outputs of the operator. +func (w *WriterOperator) SetOutputs(operators []operator.Operator) error { + outputOperators := make([]operator.Operator, 0) - for _, pluginID := range w.OutputIDs { - plugin, ok := w.findOperator(plugins, pluginID) + for _, operatorID := range w.OutputIDs { + operator, ok := w.findOperator(operators, operatorID) if !ok { - return fmt.Errorf("plugin '%s' does not exist", pluginID) + return fmt.Errorf("operator '%s' does not exist", operatorID) } - if !plugin.CanProcess() { - return fmt.Errorf("plugin '%s' can not process entries", pluginID) + if !operator.CanProcess() { + return fmt.Errorf("operator '%s' can not process entries", operatorID) } - outputOperators = append(outputOperators, plugin) + outputOperators = append(outputOperators, operator) } - // No outputs have been set, so use the next configured plugin + // No outputs have been set, so use the next configured operator if len(w.OutputIDs) == 0 { currentOperatorIndex := -1 - for i, plugin := range plugins { - if plugin.ID() == w.ID() { + for i, operator := range operators { + if operator.ID() == w.ID() { currentOperatorIndex = i break } } if currentOperatorIndex == -1 { - return fmt.Errorf("unexpectedly could not find self in array of plugins") + return fmt.Errorf("unexpectedly could not find self in array of operators") } nextOperatorIndex := currentOperatorIndex + 1 - if nextOperatorIndex == len(plugins) { - return fmt.Errorf("cannot omit output for the last plugin in the pipeline") + if nextOperatorIndex == len(operators) { + return fmt.Errorf("cannot omit output for the last operator in the pipeline") } - nextOperator := plugins[nextOperatorIndex] + nextOperator := operators[nextOperatorIndex] if !nextOperator.CanProcess() { - return fmt.Errorf("plugin '%s' cannot process entries, but it was selected as a receiver because 'output' was omitted", nextOperator.ID()) + return fmt.Errorf("operator '%s' cannot process entries, but it was selected as a receiver because 'output' was omitted", nextOperator.ID()) } outputOperators = append(outputOperators, nextOperator) } @@ -117,17 +117,17 @@ func (w *WriterOperator) SetOutputs(plugins []plugin.Operator) error { return nil } -// FindOperator will find a plugin matching the supplied id. -func (w *WriterOperator) findOperator(plugins []plugin.Operator, pluginID string) (plugin.Operator, bool) { - for _, plugin := range plugins { - if plugin.ID() == pluginID { - return plugin, true +// FindOperator will find a operator matching the supplied id. +func (w *WriterOperator) findOperator(operators []operator.Operator, operatorID string) (operator.Operator, bool) { + for _, operator := range operators { + if operator.ID() == operatorID { + return operator, true } } return nil, false } -// OutputIDs is a collection of plugin IDs used as outputs. +// OutputIDs is a collection of operator IDs used as outputs. type OutputIDs []string // UnmarshalJSON will unmarshal a string or array of strings to OutputIDs. diff --git a/plugin/helper/writer_test.go b/operator/helper/writer_test.go similarity index 93% rename from plugin/helper/writer_test.go rename to operator/helper/writer_test.go index a0a405909..1f5476198 100644 --- a/plugin/helper/writer_test.go +++ b/operator/helper/writer_test.go @@ -7,7 +7,7 @@ import ( "github.com/observiq/carbon/entry" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" yaml "gopkg.in/yaml.v2" @@ -50,7 +50,7 @@ func TestWriterOperatorWrite(t *testing.T) { output2 := &testutil.Operator{} output2.On("Process", mock.Anything, mock.Anything).Return(nil) writer := WriterOperator{ - OutputOperators: []plugin.Operator{output1, output2}, + OutputOperators: []operator.Operator{output1, output2}, } ctx := context.Background() @@ -72,7 +72,7 @@ func TestWriterOperatorOutputs(t *testing.T) { output2 := &testutil.Operator{} output2.On("Process", mock.Anything, mock.Anything).Return(nil) writer := WriterOperator{ - OutputOperators: []plugin.Operator{output1, output2}, + OutputOperators: []operator.Operator{output1, output2}, } ctx := context.Background() @@ -90,7 +90,7 @@ func TestWriterSetOutputsMissing(t *testing.T) { OutputIDs: OutputIDs{"output2"}, } - err := writer.SetOutputs([]plugin.Operator{output1}) + err := writer.SetOutputs([]operator.Operator{output1}) require.Error(t, err) require.Contains(t, err.Error(), "does not exist") } @@ -103,7 +103,7 @@ func TestWriterSetOutputsInvalid(t *testing.T) { OutputIDs: OutputIDs{"output1"}, } - err := writer.SetOutputs([]plugin.Operator{output1}) + err := writer.SetOutputs([]operator.Operator{output1}) require.Error(t, err) require.Contains(t, err.Error(), "can not process entries") } @@ -119,9 +119,9 @@ func TestWriterSetOutputsValid(t *testing.T) { OutputIDs: OutputIDs{"output1", "output2"}, } - err := writer.SetOutputs([]plugin.Operator{output1, output2}) + err := writer.SetOutputs([]operator.Operator{output1, output2}) require.NoError(t, err) - require.Equal(t, []plugin.Operator{output1, output2}, writer.Outputs()) + require.Equal(t, []operator.Operator{output1, output2}, writer.Outputs()) } func TestUnmarshalJSONString(t *testing.T) { diff --git a/plugin/plugin.go b/operator/operator.go similarity index 98% rename from plugin/plugin.go rename to operator/operator.go index f1acdb9af..22fc0f28a 100644 --- a/plugin/plugin.go +++ b/operator/operator.go @@ -1,6 +1,6 @@ //go:generate mockery -name=^(Operator)$ -output=../internal/testutil -outpkg=testutil -case=snake -package plugin +package operator import ( "context" diff --git a/pipeline/config.go b/pipeline/config.go index d94099a32..7c6eafdc4 100644 --- a/pipeline/config.go +++ b/pipeline/config.go @@ -5,8 +5,8 @@ import ( "strings" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" + "github.com/observiq/carbon/operator" + "github.com/observiq/carbon/operator/helper" yaml "gopkg.in/yaml.v2" ) @@ -14,18 +14,18 @@ import ( type Config []Params // BuildPipeline will build a pipeline from the config. -func (c Config) BuildPipeline(context plugin.BuildContext) (*Pipeline, error) { - pluginConfigs, err := c.buildOperatorConfigs(context.CustomRegistry) +func (c Config) BuildPipeline(context operator.BuildContext) (*Pipeline, error) { + operatorConfigs, err := c.buildOperatorConfigs(context.CustomRegistry) if err != nil { return nil, err } - plugins, err := c.buildOperators(pluginConfigs, context) + operators, err := c.buildOperators(operatorConfigs, context) if err != nil { return nil, err } - pipeline, err := NewPipeline(plugins) + pipeline, err := NewPipeline(operators) if err != nil { return nil, err } @@ -33,26 +33,26 @@ func (c Config) BuildPipeline(context plugin.BuildContext) (*Pipeline, error) { return pipeline, nil } -func (c Config) buildOperators(pluginConfigs []plugin.Config, context plugin.BuildContext) ([]plugin.Operator, error) { - plugins := make([]plugin.Operator, 0, len(pluginConfigs)) - for _, pluginConfig := range pluginConfigs { - plugin, err := pluginConfig.Build(context) +func (c Config) buildOperators(operatorConfigs []operator.Config, context operator.BuildContext) ([]operator.Operator, error) { + operators := make([]operator.Operator, 0, len(operatorConfigs)) + for _, operatorConfig := range operatorConfigs { + operator, err := operatorConfig.Build(context) if err != nil { return nil, errors.WithDetails(err, - "plugin_id", pluginConfig.ID(), - "plugin_type", pluginConfig.Type(), + "operator_id", operatorConfig.ID(), + "operator_type", operatorConfig.Type(), ) } - plugins = append(plugins, plugin) + operators = append(operators, operator) } - return plugins, nil + return operators, nil } -func (c Config) buildOperatorConfigs(customRegistry plugin.CustomRegistry) ([]plugin.Config, error) { - pluginConfigs := make([]plugin.Config, 0, len(c)) +func (c Config) buildOperatorConfigs(customRegistry operator.CustomRegistry) ([]operator.Config, error) { + operatorConfigs := make([]operator.Config, 0, len(c)) for _, params := range c { if err := params.Validate(); err != nil { @@ -61,15 +61,15 @@ func (c Config) buildOperatorConfigs(customRegistry plugin.CustomRegistry) ([]pl configs, err := params.BuildConfigs(customRegistry, "$") if err != nil { - return nil, errors.Wrap(err, "build plugin configs") + return nil, errors.Wrap(err, "build operator configs") } - pluginConfigs = append(pluginConfigs, configs...) + operatorConfigs = append(operatorConfigs, configs...) } - return pluginConfigs, nil + return operatorConfigs, nil } -// Params is a raw params map that can be converted into a plugin config. +// Params is a raw params map that can be converted into a operator config. type Params map[string]interface{} // ID returns the id field in the params map. @@ -124,12 +124,12 @@ func (p Params) NamespaceExclusions(namespace string) []string { return exclusions } -// Validate will validate the basic fields required to make a plugin config. +// Validate will validate the basic fields required to make a operator config. func (p Params) Validate() error { if p.Type() == "" { return errors.NewError( - "missing required `type` field for plugin config", - "ensure that all plugin configs have a defined type field", + "missing required `type` field for operator config", + "ensure that all operator configs have a defined type field", "id", p.ID(), ) } @@ -177,9 +177,9 @@ func (p Params) getStringArray(key string) []string { } } -// BuildConfigs will build plugin configs from a params map. -func (p Params) BuildConfigs(customRegistry plugin.CustomRegistry, namespace string) ([]plugin.Config, error) { - if plugin.IsDefined(p.Type()) { +// BuildConfigs will build operator configs from a params map. +func (p Params) BuildConfigs(customRegistry operator.CustomRegistry, namespace string) ([]operator.Config, error) { + if operator.IsDefined(p.Type()) { return p.buildAsBuiltin(namespace) } @@ -188,15 +188,15 @@ func (p Params) BuildConfigs(customRegistry plugin.CustomRegistry, namespace str } return nil, errors.NewError( - "unsupported `type` for plugin config", - "ensure that all plugins have a supported builtin or custom type", + "unsupported `type` for operator config", + "ensure that all operators have a supported builtin or custom type", "type", p.Type(), "id", p.ID(), ) } // buildAsBuiltin will build a builtin config from a params map. -func (p Params) buildAsBuiltin(namespace string) ([]plugin.Config, error) { +func (p Params) buildAsBuiltin(namespace string) ([]operator.Config, error) { bytes, err := yaml.Marshal(p) if err != nil { return nil, errors.NewError( @@ -206,17 +206,17 @@ func (p Params) buildAsBuiltin(namespace string) ([]plugin.Config, error) { ) } - var config plugin.Config + var config operator.Config if err := yaml.UnmarshalStrict(bytes, &config); err != nil { return nil, err } config.SetNamespace(namespace) - return []plugin.Config{config}, nil + return []operator.Config{config}, nil } // buildAsCustom will build a custom config from a params map. -func (p Params) buildAsCustom(customRegistry plugin.CustomRegistry, namespace string) ([]plugin.Config, error) { +func (p Params) buildAsCustom(customRegistry operator.CustomRegistry, namespace string) ([]operator.Config, error) { templateParams := map[string]interface{}{} for key, value := range p { templateParams[key] = value @@ -231,9 +231,9 @@ func (p Params) buildAsCustom(customRegistry plugin.CustomRegistry, namespace st } exclusions := p.NamespaceExclusions(namespace) - for _, pluginConfig := range config.Pipeline { + for _, operatorConfig := range config.Pipeline { innerNamespace := p.NamespacedID(namespace) - pluginConfig.SetNamespace(innerNamespace, exclusions...) + operatorConfig.SetNamespace(innerNamespace, exclusions...) } return config.Pipeline, nil diff --git a/pipeline/config_test.go b/pipeline/config_test.go index 88aa6668a..498bbc962 100644 --- a/pipeline/config_test.go +++ b/pipeline/config_test.go @@ -5,9 +5,9 @@ import ( "fmt" "testing" - "github.com/observiq/carbon/plugin" - _ "github.com/observiq/carbon/plugin/builtin" - "github.com/observiq/carbon/plugin/builtin/transformer" + "github.com/observiq/carbon/operator" + _ "github.com/observiq/carbon/operator/builtin" + "github.com/observiq/carbon/operator/builtin/transformer" "github.com/stretchr/testify/require" "go.uber.org/zap" yaml "gopkg.in/yaml.v2" @@ -188,7 +188,7 @@ func TestBuildBuiltinFromParamsWithUnsupportedYaml(t *testing.T) { "output": "test", "field": invalidMarshaller{}, } - _, err := params.BuildConfigs(plugin.CustomRegistry{}, "test_namespace") + _, err := params.BuildConfigs(operator.CustomRegistry{}, "test_namespace") require.Error(t, err) require.Contains(t, err.Error(), "failed to parse config map as yaml") } @@ -200,7 +200,7 @@ func TestBuildBuiltinFromParamsWithUnknownField(t *testing.T) { "unknown": true, "output": "test_output", } - _, err := params.BuildConfigs(plugin.CustomRegistry{}, "test_namespace") + _, err := params.BuildConfigs(operator.CustomRegistry{}, "test_namespace") require.Error(t, err) } @@ -210,7 +210,7 @@ func TestBuildBuiltinFromValidParams(t *testing.T) { "type": "noop", "output": "test_output", } - configs, err := params.BuildConfigs(plugin.CustomRegistry{}, "test_namespace") + configs, err := params.BuildConfigs(operator.CustomRegistry{}, "test_namespace") require.NoError(t, err) require.Equal(t, 1, len(configs)) @@ -219,19 +219,19 @@ func TestBuildBuiltinFromValidParams(t *testing.T) { } func TestBuildCustomFromValidParams(t *testing.T) { - registry := plugin.CustomRegistry{} + registry := operator.CustomRegistry{} customTemplate := ` pipeline: - id: custom_noop type: noop output: {{.output}} ` - err := registry.Add("custom_plugin", customTemplate) + err := registry.Add("custom_operator", customTemplate) require.NoError(t, err) params := Params{ - "id": "custom_plugin", - "type": "custom_plugin", + "id": "custom_operator", + "type": "custom_operator", "output": "test_output", } @@ -239,11 +239,11 @@ pipeline: require.NoError(t, err) require.Equal(t, 1, len(configs)) require.IsType(t, &transformer.NoopOperatorConfig{}, configs[0].Builder) - require.Equal(t, "test_namespace.custom_plugin.custom_noop", configs[0].ID()) + require.Equal(t, "test_namespace.custom_operator.custom_noop", configs[0].ID()) } func TestBuildValidPipeline(t *testing.T) { - registry := plugin.CustomRegistry{} + registry := operator.CustomRegistry{} customTemplate := ` pipeline: - id: custom_generate @@ -254,22 +254,22 @@ pipeline: message: test output: {{.output}} ` - err := registry.Add("custom_plugin", customTemplate) + err := registry.Add("custom_operator", customTemplate) require.NoError(t, err) logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) - context := plugin.BuildContext{ + context := operator.BuildContext{ CustomRegistry: registry, Logger: logger.Sugar(), } pipelineConfig := Config{ Params{ - "id": "custom_plugin", - "type": "custom_plugin", + "id": "custom_operator", + "type": "custom_operator", "output": "drop_output", }, Params{ @@ -283,20 +283,20 @@ pipeline: } func TestBuildInvalidPipelineInvalidType(t *testing.T) { - registry := plugin.CustomRegistry{} + registry := operator.CustomRegistry{} logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) - context := plugin.BuildContext{ + context := operator.BuildContext{ CustomRegistry: registry, Logger: logger.Sugar(), } pipelineConfig := Config{ Params{ - "id": "custom_plugin", - "type": "custom_plugin", + "id": "custom_operator", + "type": "custom_operator", "output": "drop_output", }, Params{ @@ -307,11 +307,11 @@ func TestBuildInvalidPipelineInvalidType(t *testing.T) { _, err = pipelineConfig.BuildPipeline(context) require.Error(t, err) - require.Contains(t, err.Error(), "unsupported `type` for plugin config") + require.Contains(t, err.Error(), "unsupported `type` for operator config") } func TestBuildInvalidPipelineInvalidParam(t *testing.T) { - registry := plugin.CustomRegistry{} + registry := operator.CustomRegistry{} customTemplate := ` pipeline: - id: custom_generate @@ -321,22 +321,22 @@ pipeline: message: test output: {{.output}} ` - err := registry.Add("custom_plugin", customTemplate) + err := registry.Add("custom_operator", customTemplate) require.NoError(t, err) logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) - context := plugin.BuildContext{ + context := operator.BuildContext{ CustomRegistry: registry, Logger: logger.Sugar(), } pipelineConfig := Config{ Params{ - "id": "custom_plugin", - "type": "custom_plugin", + "id": "custom_operator", + "type": "custom_operator", "output": "drop_output", }, Params{ @@ -347,16 +347,16 @@ pipeline: _, err = pipelineConfig.BuildPipeline(context) require.Error(t, err) - require.Contains(t, err.Error(), "build plugin configs") + require.Contains(t, err.Error(), "build operator configs") } func TestBuildInvalidPipelineInvalidOperator(t *testing.T) { - registry := plugin.CustomRegistry{} + registry := operator.CustomRegistry{} logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) - context := plugin.BuildContext{ + context := operator.BuildContext{ CustomRegistry: registry, Logger: logger.Sugar(), } @@ -379,12 +379,12 @@ func TestBuildInvalidPipelineInvalidOperator(t *testing.T) { } func TestBuildInvalidPipelineInvalidGraph(t *testing.T) { - registry := plugin.CustomRegistry{} + registry := operator.CustomRegistry{} logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) - context := plugin.BuildContext{ + context := operator.BuildContext{ CustomRegistry: registry, Logger: logger.Sugar(), } diff --git a/pipeline/node.go b/pipeline/node.go index f364bb437..4dc0e4a36 100644 --- a/pipeline/node.go +++ b/pipeline/node.go @@ -3,19 +3,19 @@ package pipeline import ( "hash/fnv" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" ) -// OperatorNode is a basic node that represents a plugin in a pipeline. +// OperatorNode is a basic node that represents a operator in a pipeline. type OperatorNode struct { - plugin plugin.Operator + operator operator.Operator id int64 outputIDs map[string]int64 } -// Operator returns the plugin of the node. -func (b OperatorNode) Operator() plugin.Operator { - return b.plugin +// Operator returns the operator of the node. +func (b OperatorNode) Operator() operator.Operator { + return b.operator } // ID returns the node id. @@ -25,29 +25,29 @@ func (b OperatorNode) ID() int64 { // DOTID returns the id used to represent this node in a dot graph. func (b OperatorNode) DOTID() string { - return b.plugin.ID() + return b.operator.ID() } -// OutputIDs returns a map of output plugin ids to node ids. +// OutputIDs returns a map of output operator ids to node ids. func (b OperatorNode) OutputIDs() map[string]int64 { return b.outputIDs } -// createOperatorNode will create a plugin node. -func createOperatorNode(plugin plugin.Operator) OperatorNode { - id := createNodeID(plugin.ID()) +// createOperatorNode will create a operator node. +func createOperatorNode(operator operator.Operator) OperatorNode { + id := createNodeID(operator.ID()) outputIDs := make(map[string]int64) - if plugin.CanOutput() { - for _, output := range plugin.Outputs() { + if operator.CanOutput() { + for _, output := range operator.Outputs() { outputIDs[output.ID()] = createNodeID(output.ID()) } } - return OperatorNode{plugin, id, outputIDs} + return OperatorNode{operator, id, outputIDs} } -// createNodeID generates a node id from a plugin id. -func createNodeID(pluginID string) int64 { +// createNodeID generates a node id from a operator id. +func createNodeID(operatorID string) int64 { hash := fnv.New64a() - _, _ = hash.Write([]byte(pluginID)) + _, _ = hash.Write([]byte(operatorID)) return int64(hash.Sum64()) } diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index b66a2d8c2..162444d26 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -5,19 +5,19 @@ import ( "strings" "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "gonum.org/v1/gonum/graph/encoding/dot" "gonum.org/v1/gonum/graph/simple" "gonum.org/v1/gonum/graph/topo" ) -// Pipeline is a directed graph of connected plugins. +// Pipeline is a directed graph of connected operators. type Pipeline struct { Graph *simple.DirectedGraph running bool } -// Start will start the plugins in a pipeline in reverse topological order. +// Start will start the operators in a pipeline in reverse topological order. func (p *Pipeline) Start() error { if p.running { return nil @@ -25,19 +25,19 @@ func (p *Pipeline) Start() error { sortedNodes, _ := topo.Sort(p.Graph) for i := len(sortedNodes) - 1; i >= 0; i-- { - plugin := sortedNodes[i].(OperatorNode).Operator() - plugin.Logger().Debug("Starting plugin") - if err := plugin.Start(); err != nil { + operator := sortedNodes[i].(OperatorNode).Operator() + operator.Logger().Debug("Starting operator") + if err := operator.Start(); err != nil { return err } - plugin.Logger().Debug("Started plugin") + operator.Logger().Debug("Started operator") } p.running = true return nil } -// Stop will stop the plugins in a pipeline in topological order. +// Stop will stop the operators in a pipeline in topological order. func (p *Pipeline) Stop() { if !p.running { return @@ -45,10 +45,10 @@ func (p *Pipeline) Stop() { sortedNodes, _ := topo.Sort(p.Graph) for _, node := range sortedNodes { - plugin := node.(OperatorNode).Operator() - plugin.Logger().Debug("Stopping plugin") - _ = plugin.Stop() - plugin.Logger().Debug("Stopped plugin") + operator := node.(OperatorNode).Operator() + operator.Logger().Debug("Stopping operator") + _ = operator.Stop() + operator.Logger().Debug("Stopped operator") } p.running = false @@ -59,18 +59,18 @@ func (p *Pipeline) MarshalDot() ([]byte, error) { return dot.Marshal(p.Graph, "G", "", " ") } -// addNodes will add plugins as nodes to the supplied graph. -func addNodes(graph *simple.DirectedGraph, plugins []plugin.Operator) error { - for _, plugin := range plugins { - pluginNode := createOperatorNode(plugin) - if graph.Node(pluginNode.ID()) != nil { +// addNodes will add operators as nodes to the supplied graph. +func addNodes(graph *simple.DirectedGraph, operators []operator.Operator) error { + for _, operator := range operators { + operatorNode := createOperatorNode(operator) + if graph.Node(operatorNode.ID()) != nil { return errors.NewError( - fmt.Sprintf("plugin with id '%s' already exists in pipeline", pluginNode.Operator().ID()), - "ensure that each plugin has a unique `type` or `id`", + fmt.Sprintf("operator with id '%s' already exists in pipeline", operatorNode.Operator().ID()), + "ensure that each operator has a unique `type` or `id`", ) } - graph.AddNode(pluginNode) + graph.AddNode(operatorNode) } return nil } @@ -88,7 +88,7 @@ func connectNodes(graph *simple.DirectedGraph) error { if _, err := topo.Sort(graph); err != nil { return errors.NewError( "pipeline has a circular dependency", - "ensure that all plugins are connected in a straight, acyclic line", + "ensure that all operators are connected in a straight, acyclic line", "cycles", unorderableToCycles(err.(topo.Unorderable)), ) } @@ -101,29 +101,29 @@ func connectNode(graph *simple.DirectedGraph, inputNode OperatorNode) error { for outputOperatorID, outputNodeID := range inputNode.OutputIDs() { if graph.Node(outputNodeID) == nil { return errors.NewError( - "plugins cannot be connected, because the output does not exist in the pipeline", - "ensure that the output plugin is defined", - "input_plugin", inputNode.Operator().ID(), - "output_plugin", outputOperatorID, + "operators cannot be connected, because the output does not exist in the pipeline", + "ensure that the output operator is defined", + "input_operator", inputNode.Operator().ID(), + "output_operator", outputOperatorID, ) } outputNode := graph.Node(outputNodeID).(OperatorNode) if !outputNode.Operator().CanProcess() { return errors.NewError( - "plugins cannot be connected, because the output plugin can not process logs", - "ensure that the output plugin can process logs (like a parser or destination)", - "input_plugin", inputNode.Operator().ID(), - "output_plugin", outputOperatorID, + "operators cannot be connected, because the output operator can not process logs", + "ensure that the output operator can process logs (like a parser or destination)", + "input_operator", inputNode.Operator().ID(), + "output_operator", outputOperatorID, ) } if graph.HasEdgeFromTo(inputNode.ID(), outputNodeID) { return errors.NewError( - "plugins cannot be connected, because a connection already exists", - "ensure that only a single connection exists between the two plugins", - "input_plugin", inputNode.Operator().ID(), - "output_plugin", outputOperatorID, + "operators cannot be connected, because a connection already exists", + "ensure that only a single connection exists between the two operators", + "input_operator", inputNode.Operator().ID(), + "output_operator", outputOperatorID, ) } @@ -134,28 +134,28 @@ func connectNode(graph *simple.DirectedGraph, inputNode OperatorNode) error { return nil } -// setOperatorOutputs will set the outputs on plugins that can output. -func setOperatorOutputs(plugins []plugin.Operator) error { - for _, plugin := range plugins { - if !plugin.CanOutput() { +// setOperatorOutputs will set the outputs on operators that can output. +func setOperatorOutputs(operators []operator.Operator) error { + for _, operator := range operators { + if !operator.CanOutput() { continue } - if err := plugin.SetOutputs(plugins); err != nil { - return errors.WithDetails(err, "plugin_id", plugin.ID()) + if err := operator.SetOutputs(operators); err != nil { + return errors.WithDetails(err, "operator_id", operator.ID()) } } return nil } -// NewPipeline creates a new pipeline of connected plugins. -func NewPipeline(plugins []plugin.Operator) (*Pipeline, error) { - if err := setOperatorOutputs(plugins); err != nil { +// NewPipeline creates a new pipeline of connected operators. +func NewPipeline(operators []operator.Operator) (*Pipeline, error) { + if err := setOperatorOutputs(operators); err != nil { return nil, err } graph := simple.NewDirectedGraph() - if err := addNodes(graph, plugins); err != nil { + if err := addNodes(graph, operators); err != nil { return nil, err } @@ -174,10 +174,10 @@ func unorderableToCycles(err topo.Unorderable) string { } cycles.WriteByte('(') for _, node := range cycle { - cycles.WriteString(node.(OperatorNode).plugin.ID()) + cycles.WriteString(node.(OperatorNode).operator.ID()) cycles.Write([]byte(` -> `)) } - cycles.WriteString(cycle[0].(OperatorNode).plugin.ID()) + cycles.WriteString(cycle[0].(OperatorNode).operator.ID()) cycles.WriteByte(')') } return cycles.String() diff --git a/pipeline/pipeline_test.go b/pipeline/pipeline_test.go index b8e34deeb..edd26abc3 100644 --- a/pipeline/pipeline_test.go +++ b/pipeline/pipeline_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/observiq/carbon/internal/testutil" - "github.com/observiq/carbon/plugin" + "github.com/observiq/carbon/operator" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "gonum.org/v1/gonum/graph" @@ -14,12 +14,12 @@ import ( func TestUnorderableToCycles(t *testing.T) { t.Run("SingleCycle", func(t *testing.T) { - mockOperator1 := testutil.NewMockOperator("plugin1") - mockOperator2 := testutil.NewMockOperator("plugin2") - mockOperator3 := testutil.NewMockOperator("plugin3") - mockOperator1.On("Outputs").Return([]plugin.Operator{mockOperator2}) - mockOperator2.On("Outputs").Return([]plugin.Operator{mockOperator3}) - mockOperator3.On("Outputs").Return([]plugin.Operator{mockOperator1}) + mockOperator1 := testutil.NewMockOperator("operator1") + mockOperator2 := testutil.NewMockOperator("operator2") + mockOperator3 := testutil.NewMockOperator("operator3") + mockOperator1.On("Outputs").Return([]operator.Operator{mockOperator2}) + mockOperator2.On("Outputs").Return([]operator.Operator{mockOperator3}) + mockOperator3.On("Outputs").Return([]operator.Operator{mockOperator1}) err := topo.Unorderable([][]graph.Node{{ createOperatorNode(mockOperator1), @@ -28,25 +28,25 @@ func TestUnorderableToCycles(t *testing.T) { }}) output := unorderableToCycles(err) - expected := `(plugin1 -> plugin2 -> plugin3 -> plugin1)` + expected := `(operator1 -> operator2 -> operator3 -> operator1)` require.Equal(t, expected, output) }) t.Run("MultipleCycles", func(t *testing.T) { - mockOperator1 := testutil.NewMockOperator("plugin1") - mockOperator2 := testutil.NewMockOperator("plugin2") - mockOperator3 := testutil.NewMockOperator("plugin3") - mockOperator1.On("Outputs").Return([]plugin.Operator{mockOperator2}) - mockOperator2.On("Outputs").Return([]plugin.Operator{mockOperator3}) - mockOperator3.On("Outputs").Return([]plugin.Operator{mockOperator1}) - - mockOperator4 := testutil.NewMockOperator("plugin4") - mockOperator5 := testutil.NewMockOperator("plugin5") - mockOperator6 := testutil.NewMockOperator("plugin6") - mockOperator4.On("Outputs").Return([]plugin.Operator{mockOperator5}) - mockOperator5.On("Outputs").Return([]plugin.Operator{mockOperator6}) - mockOperator6.On("Outputs").Return([]plugin.Operator{mockOperator4}) + mockOperator1 := testutil.NewMockOperator("operator1") + mockOperator2 := testutil.NewMockOperator("operator2") + mockOperator3 := testutil.NewMockOperator("operator3") + mockOperator1.On("Outputs").Return([]operator.Operator{mockOperator2}) + mockOperator2.On("Outputs").Return([]operator.Operator{mockOperator3}) + mockOperator3.On("Outputs").Return([]operator.Operator{mockOperator1}) + + mockOperator4 := testutil.NewMockOperator("operator4") + mockOperator5 := testutil.NewMockOperator("operator5") + mockOperator6 := testutil.NewMockOperator("operator6") + mockOperator4.On("Outputs").Return([]operator.Operator{mockOperator5}) + mockOperator5.On("Outputs").Return([]operator.Operator{mockOperator6}) + mockOperator6.On("Outputs").Return([]operator.Operator{mockOperator4}) err := topo.Unorderable([][]graph.Node{{ createOperatorNode(mockOperator1), @@ -59,7 +59,7 @@ func TestUnorderableToCycles(t *testing.T) { }}) output := unorderableToCycles(err) - expected := `(plugin1 -> plugin2 -> plugin3 -> plugin1),(plugin4 -> plugin5 -> plugin6 -> plugin4)` + expected := `(operator1 -> operator2 -> operator3 -> operator1),(operator4 -> operator5 -> operator6 -> operator4)` require.Equal(t, expected, output) }) @@ -67,7 +67,7 @@ func TestUnorderableToCycles(t *testing.T) { func TestPipeline(t *testing.T) { t.Run("MultipleStart", func(t *testing.T) { - pipeline, err := NewPipeline([]plugin.Operator{}) + pipeline, err := NewPipeline([]operator.Operator{}) require.NoError(t, err) err = pipeline.Start() @@ -80,7 +80,7 @@ func TestPipeline(t *testing.T) { }) t.Run("MultipleStop", func(t *testing.T) { - pipeline, err := NewPipeline([]plugin.Operator{}) + pipeline, err := NewPipeline([]operator.Operator{}) require.NoError(t, err) err = pipeline.Start() @@ -91,60 +91,60 @@ func TestPipeline(t *testing.T) { }) t.Run("DuplicateNodeIDs", func(t *testing.T) { - plugin1 := testutil.NewMockOperator("plugin1") - plugin1.On("SetOutputs", mock.Anything).Return(nil) - plugin1.On("Outputs").Return(nil) - plugin2 := testutil.NewMockOperator("plugin1") - plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return(nil) - - _, err := NewPipeline([]plugin.Operator{plugin1, plugin2}) + operator1 := testutil.NewMockOperator("operator1") + operator1.On("SetOutputs", mock.Anything).Return(nil) + operator1.On("Outputs").Return(nil) + operator2 := testutil.NewMockOperator("operator1") + operator2.On("SetOutputs", mock.Anything).Return(nil) + operator2.On("Outputs").Return(nil) + + _, err := NewPipeline([]operator.Operator{operator1, operator2}) require.Error(t, err) require.Contains(t, err.Error(), "already exists") }) t.Run("OutputNotExist", func(t *testing.T) { - plugin1 := testutil.NewMockOperator("plugin1") - plugin1.On("SetOutputs", mock.Anything).Return(nil) - plugin1.On("Outputs").Return() + operator1 := testutil.NewMockOperator("operator1") + operator1.On("SetOutputs", mock.Anything).Return(nil) + operator1.On("Outputs").Return() - plugin2 := testutil.NewMockOperator("plugin2") - plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return([]plugin.Operator{plugin1}) + operator2 := testutil.NewMockOperator("operator2") + operator2.On("SetOutputs", mock.Anything).Return(nil) + operator2.On("Outputs").Return([]operator.Operator{operator1}) - _, err := NewPipeline([]plugin.Operator{plugin2}) + _, err := NewPipeline([]operator.Operator{operator2}) require.Error(t, err) require.Contains(t, err.Error(), "does not exist") }) t.Run("OutputNotProcessor", func(t *testing.T) { - plugin1 := &testutil.Operator{} - plugin1.On("ID").Return("plugin1") - plugin1.On("CanProcess").Return(false) - plugin1.On("CanOutput").Return(true) - plugin1.On("SetOutputs", mock.Anything).Return(nil) - plugin1.On("Outputs").Return(nil) - - plugin2 := testutil.NewMockOperator("plugin2") - plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return([]plugin.Operator{plugin1}) - - _, err := NewPipeline([]plugin.Operator{plugin1, plugin2}) + operator1 := &testutil.Operator{} + operator1.On("ID").Return("operator1") + operator1.On("CanProcess").Return(false) + operator1.On("CanOutput").Return(true) + operator1.On("SetOutputs", mock.Anything).Return(nil) + operator1.On("Outputs").Return(nil) + + operator2 := testutil.NewMockOperator("operator2") + operator2.On("SetOutputs", mock.Anything).Return(nil) + operator2.On("Outputs").Return([]operator.Operator{operator1}) + + _, err := NewPipeline([]operator.Operator{operator1, operator2}) require.Error(t, err) require.Contains(t, err.Error(), "can not process") }) t.Run("DuplicateEdges", func(t *testing.T) { - plugin1 := testutil.NewMockOperator("plugin1") - plugin1.On("SetOutputs", mock.Anything).Return(nil) - plugin1.On("Outputs").Return(nil) + operator1 := testutil.NewMockOperator("operator1") + operator1.On("SetOutputs", mock.Anything).Return(nil) + operator1.On("Outputs").Return(nil) - plugin2 := testutil.NewMockOperator("plugin2") - plugin2.On("SetOutputs", mock.Anything).Return(nil) - plugin2.On("Outputs").Return([]plugin.Operator{plugin1, plugin1}) + operator2 := testutil.NewMockOperator("operator2") + operator2.On("SetOutputs", mock.Anything).Return(nil) + operator2.On("Outputs").Return([]operator.Operator{operator1, operator1}) - node1 := createOperatorNode(plugin1) - node2 := createOperatorNode(plugin2) + node1 := createOperatorNode(operator1) + node2 := createOperatorNode(operator2) graph := simple.NewDirectedGraph() graph.AddNode(node1) @@ -158,17 +158,17 @@ func TestPipeline(t *testing.T) { }) t.Run("Cyclical", func(t *testing.T) { - mockOperator1 := testutil.NewMockOperator("plugin1") - mockOperator2 := testutil.NewMockOperator("plugin2") - mockOperator3 := testutil.NewMockOperator("plugin3") - mockOperator1.On("Outputs").Return([]plugin.Operator{mockOperator2}) + mockOperator1 := testutil.NewMockOperator("operator1") + mockOperator2 := testutil.NewMockOperator("operator2") + mockOperator3 := testutil.NewMockOperator("operator3") + mockOperator1.On("Outputs").Return([]operator.Operator{mockOperator2}) mockOperator1.On("SetOutputs", mock.Anything).Return(nil) - mockOperator2.On("Outputs").Return([]plugin.Operator{mockOperator3}) + mockOperator2.On("Outputs").Return([]operator.Operator{mockOperator3}) mockOperator2.On("SetOutputs", mock.Anything).Return(nil) - mockOperator3.On("Outputs").Return([]plugin.Operator{mockOperator1}) + mockOperator3.On("Outputs").Return([]operator.Operator{mockOperator1}) mockOperator3.On("SetOutputs", mock.Anything).Return(nil) - _, err := NewPipeline([]plugin.Operator{mockOperator1, mockOperator2, mockOperator3}) + _, err := NewPipeline([]operator.Operator{mockOperator1, mockOperator2, mockOperator3}) require.Error(t, err) require.Contains(t, err.Error(), "circular dependency") }) diff --git a/plugin/buffer/buffer.go b/plugin/buffer/buffer.go deleted file mode 100644 index 11c731234..000000000 --- a/plugin/buffer/buffer.go +++ /dev/null @@ -1,76 +0,0 @@ -package buffer - -import ( - "context" - "fmt" - "time" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/errors" - "github.com/observiq/carbon/plugin" -) - -// Buffer is an entity that buffers log entries to a plugin -type Buffer interface { - Flush(context.Context) error - Add(interface{}, int) error - AddWait(context.Context, interface{}, int) error - SetHandler(BundleHandler) - Process(context.Context, *entry.Entry) error -} - -func NewConfig() Config { - return Config{ - BufferType: "memory", - DelayThreshold: plugin.Duration{Duration: time.Second}, - BundleCountThreshold: 10_000, - BundleByteThreshold: 4 * 1024 * 1024 * 1024, // 4MB - BundleByteLimit: 4 * 1024 * 1024 * 1024, // 4MB - BufferedByteLimit: 500 * 1024 * 1024 * 1024, // 500MB - HandlerLimit: 32, - Retry: NewRetryConfig(), - } -} - -// Config is the configuration of a buffer -type Config struct { - BufferType string `json:"type,omitempty" yaml:"type,omitempty"` - DelayThreshold plugin.Duration `json:"delay_threshold,omitempty" yaml:"delay_threshold,omitempty"` - BundleCountThreshold int `json:"bundle_count_threshold,omitempty" yaml:"buffer_count_threshold,omitempty"` - BundleByteThreshold int `json:"bundle_byte_threshold,omitempty" yaml:"bundle_byte_threshold,omitempty"` - BundleByteLimit int `json:"bundle_byte_limit,omitempty" yaml:"bundle_byte_limit,omitempty"` - BufferedByteLimit int `json:"buffered_byte_limit,omitempty" yaml:"buffered_byte_limit,omitempty"` - HandlerLimit int `json:"handler_limit,omitempty" yaml:"handler_limit,omitempty"` - Retry RetryConfig `json:"retry,omitempty" yaml:"retry,omitempty"` -} - -// Build will build a buffer from the supplied configuration -func (config *Config) Build() (Buffer, error) { - switch config.BufferType { - case "memory", "": - return NewMemoryBuffer(config), nil - default: - return nil, errors.NewError( - fmt.Sprintf("Invalid buffer type %s", config.BufferType), - "The only supported buffer type is 'memory'", - ) - } -} - -func NewRetryConfig() RetryConfig { - return RetryConfig{ - InitialInterval: plugin.Duration{Duration: 500 * time.Millisecond}, - RandomizationFactor: 0.5, - Multiplier: 1.5, - MaxInterval: plugin.Duration{Duration: 15 * time.Minute}, - } -} - -// RetryConfig is the configuration of an entity that will retry processing after an error -type RetryConfig struct { - InitialInterval plugin.Duration `json:"initial_interval,omitempty" yaml:"initial_interval,omitempty"` - RandomizationFactor float64 `json:"randomization_factor,omitempty" yaml:"randomization_factor,omitempty"` - Multiplier float64 `json:"multiplier,omitempty" yaml:"multiplier,omitempty"` - MaxInterval plugin.Duration `json:"max_interval,omitempty" yaml:"max_interval,omitempty"` - MaxElapsedTime plugin.Duration `json:"max_elapsed_time,omitempty" yaml:"max_elapsed_time,omitempty"` -} diff --git a/plugin/builtin/builtin.go b/plugin/builtin/builtin.go deleted file mode 100644 index 694c54e92..000000000 --- a/plugin/builtin/builtin.go +++ /dev/null @@ -1,9 +0,0 @@ -package builtin - -import ( - // Load embedded packages when importing builtin plugins - _ "github.com/observiq/carbon/plugin/builtin/input" - _ "github.com/observiq/carbon/plugin/builtin/output" - _ "github.com/observiq/carbon/plugin/builtin/parser" - _ "github.com/observiq/carbon/plugin/builtin/transformer" -) diff --git a/plugin/builtin/input/input.go b/plugin/builtin/input/input.go deleted file mode 100644 index 6e7256552..000000000 --- a/plugin/builtin/input/input.go +++ /dev/null @@ -1,6 +0,0 @@ -package input - -import ( - // Load file package when importing input plugins - _ "github.com/observiq/carbon/plugin/builtin/input/file" -) diff --git a/plugin/builtin/output/drop.go b/plugin/builtin/output/drop.go deleted file mode 100644 index 2f45c5d8d..000000000 --- a/plugin/builtin/output/drop.go +++ /dev/null @@ -1,48 +0,0 @@ -package output - -import ( - "context" - - "github.com/observiq/carbon/entry" - "github.com/observiq/carbon/plugin" - "github.com/observiq/carbon/plugin/helper" -) - -func init() { - plugin.Register("drop_output", func() plugin.Builder { return NewDropOutputConfig("") }) -} - -func NewDropOutputConfig(pluginID string) *DropOutputConfig { - return &DropOutputConfig{ - OutputConfig: helper.NewOutputConfig(pluginID, "drop_output"), - } -} - -// DropOutputConfig is the configuration of a drop output plugin. -type DropOutputConfig struct { - helper.OutputConfig `yaml:",inline"` -} - -// Build will build a drop output plugin. -func (c DropOutputConfig) Build(context plugin.BuildContext) (plugin.Operator, error) { - outputOperator, err := c.OutputConfig.Build(context) - if err != nil { - return nil, err - } - - dropOutput := &DropOutput{ - OutputOperator: outputOperator, - } - - return dropOutput, nil -} - -// DropOutput is a plugin that consumes and ignores incoming entries. -type DropOutput struct { - helper.OutputOperator -} - -// Process will drop the incoming entry. -func (p *DropOutput) Process(ctx context.Context, entry *entry.Entry) error { - return nil -} From 22255eed0b9201dca0d441b0674a7a087c81bdf5 Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Wed, 22 Jul 2020 12:12:41 -0400 Subject: [PATCH 3/7] Renamed plugins to operators and custom to plugins --- CONTRIBUTING.md | 2 +- README.md | 2 +- agent/agent.go | 6 +- commands/graph.go | 6 +- internal/testutil/{plugin.go => operator.go} | 0 operator/buffer/buffer.go | 2 +- operator/builtin/input/file/file.go | 2 +- operator/builtin/input/generate.go | 2 +- operator/builtin/input/generate_test.go | 6 +- operator/builtin/input/journald.go | 2 +- operator/builtin/input/tcp.go | 2 +- operator/builtin/input/udp.go | 2 +- operator/builtin/output/drop.go | 2 +- operator/builtin/output/elastic.go | 2 +- operator/builtin/output/file.go | 2 +- operator/builtin/output/google_cloud.go | 2 +- operator/builtin/output/stdout.go | 2 +- operator/builtin/parser/json.go | 2 +- operator/builtin/parser/regex.go | 2 +- operator/builtin/parser/severity.go | 2 +- operator/builtin/parser/syslog.go | 2 +- operator/builtin/parser/time.go | 2 +- .../transformer/k8s_metadata_decorator.go | 2 +- operator/builtin/transformer/metadata.go | 6 +- operator/builtin/transformer/noop.go | 2 +- operator/builtin/transformer/rate_limit.go | 2 +- operator/builtin/transformer/restructure.go | 2 +- operator/builtin/transformer/router.go | 4 +- operator/config.go | 8 +-- operator/config_test.go | 30 ++++----- operator/helper/{plugin.go => operator.go} | 4 +- .../{plugin_test.go => operator_test.go} | 0 operator/helper/writer.go | 4 +- operator/operator.go | 2 +- operator/{custom.go => plugin.go} | 64 +++++++++--------- operator/{custom_test.go => plugin_test.go} | 36 +++++----- pipeline/config.go | 26 ++++---- pipeline/config_test.go | 66 +++++++++---------- pipeline/node.go | 6 +- 39 files changed, 159 insertions(+), 159 deletions(-) rename internal/testutil/{plugin.go => operator.go} (100%) rename operator/helper/{plugin.go => operator.go} (94%) rename operator/helper/{plugin_test.go => operator_test.go} (100%) rename operator/{custom.go => plugin.go} (53%) rename operator/{custom_test.go => plugin_test.go} (88%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0504430d9..416378cf9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,7 +111,7 @@ func init() { ## Any tips for building operators? We highly recommend that developers take advantage of [helpers](operator/helper) when building their operators. Helpers are structs that help satisfy common behavior shared across many operators. By embedding these structs, you can skip having to satisfy certain aspects of the `operator` and `config` interfaces. -For example, almost all operators should embed the [BasicOperator](operator/helper/basic_operator.go) helper, as it provides simple functionality for returning a operator id and operator type. +For example, almost all operators should embed the [BasicOperator](operator/helper/basic_operator.go) helper, as it provides simple functionality for returning an operator id and operator type. ```go // ExampleOperator is a basic operator, with a basic lifecycle, that consumes diff --git a/README.md b/README.md index 88e927988..e908d9429 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ carbon # Supported flags: --config The location of the agent config file (default: ./config.yaml) ---plugin_dir The location of the custom plugins directory (default: ./plugins) +--plugin_dir The location of the plugins directory (default: ./plugins) --database The location of the offsets database file. If this is not specified, offsets will not be maintained across agent restarts --log_file The location of the agent log file. If not specified, carbon will log to `stderr` --debug Enables debug logging diff --git a/agent/agent.go b/agent/agent.go index 2b9e15b13..7da8a0eb2 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -39,13 +39,13 @@ func (a *LogAgent) Start() error { } a.database = database - registry, err := operator.NewCustomRegistry(a.PluginDir) + registry, err := operator.NewPluginRegistry(a.PluginDir) if err != nil { - a.Errorw("Failed to load custom operator registry", zap.Any("error", err)) + a.Errorw("Failed to load plugin registry", zap.Any("error", err)) } buildContext := operator.BuildContext{ - CustomRegistry: registry, + PluginRegistry: registry, Logger: a.SugaredLogger, Database: a.database, } diff --git a/commands/graph.go b/commands/graph.go index 4e2845101..2cc68f952 100644 --- a/commands/graph.go +++ b/commands/graph.go @@ -43,13 +43,13 @@ func runGraph(_ *cobra.Command, _ []string, flags *RootFlags) { os.Exit(1) } - customRegistry, err := operator.NewCustomRegistry(flags.PluginDir) + pluginRegistry, err := operator.NewPluginRegistry(flags.PluginDir) if err != nil { - logger.Errorw("Failed to load custom operator registry", zap.Any("error", err)) + logger.Errorw("Failed to load plugin registry", zap.Any("error", err)) } buildContext := pg.BuildContext{ - CustomRegistry: customRegistry, + PluginRegistry: pluginRegistry, Logger: logger, } diff --git a/internal/testutil/plugin.go b/internal/testutil/operator.go similarity index 100% rename from internal/testutil/plugin.go rename to internal/testutil/operator.go diff --git a/operator/buffer/buffer.go b/operator/buffer/buffer.go index 28d1926bb..cb7c6913d 100644 --- a/operator/buffer/buffer.go +++ b/operator/buffer/buffer.go @@ -10,7 +10,7 @@ import ( "github.com/observiq/carbon/operator" ) -// Buffer is an entity that buffers log entries to a operator +// Buffer is an entity that buffers log entries to an operator type Buffer interface { Flush(context.Context) error Add(interface{}, int) error diff --git a/operator/builtin/input/file/file.go b/operator/builtin/input/file/file.go index 946fdce1d..d04c42780 100644 --- a/operator/builtin/input/file/file.go +++ b/operator/builtin/input/file/file.go @@ -186,7 +186,7 @@ func (c InputConfig) getSplitFunc(encoding encoding.Encoding) (bufio.SplitFunc, return splitFunc, nil } -// InputOperator is a operator that monitors files for entries +// InputOperator is an operator that monitors files for entries type InputOperator struct { helper.InputOperator diff --git a/operator/builtin/input/generate.go b/operator/builtin/input/generate.go index fbdc19ba2..b85d36eac 100644 --- a/operator/builtin/input/generate.go +++ b/operator/builtin/input/generate.go @@ -47,7 +47,7 @@ func (c *GenerateInputConfig) Build(context operator.BuildContext) (operator.Ope return generateInput, nil } -// GenerateInput is a operator that generates log entries. +// GenerateInput is an operator that generates log entries. type GenerateInput struct { helper.InputOperator entry entry.Entry diff --git a/operator/builtin/input/generate_test.go b/operator/builtin/input/generate_test.go index 5ebf03d48..668a7bc0f 100644 --- a/operator/builtin/input/generate_test.go +++ b/operator/builtin/input/generate_test.go @@ -53,7 +53,7 @@ func TestInputGenerate(t *testing.T) { } } -func TestRenderFromCustom(t *testing.T) { +func TestRenderFromPluginTemplate(t *testing.T) { templateText := ` pipeline: - id: my_generator @@ -66,7 +66,7 @@ pipeline: tmpl, err := template.New("my_generator").Parse(templateText) require.NoError(t, err) - registry := operator.CustomRegistry{ + registry := operator.PluginRegistry{ "sample": tmpl, } @@ -76,7 +76,7 @@ pipeline: config, err := registry.Render("sample", params) require.NoError(t, err) - expectedConfig := operator.CustomConfig{ + expectedConfig := operator.PluginConfig{ Pipeline: []operator.Config{ { Builder: &GenerateInputConfig{ diff --git a/operator/builtin/input/journald.go b/operator/builtin/input/journald.go index fdf0fedc3..a1f387cbf 100644 --- a/operator/builtin/input/journald.go +++ b/operator/builtin/input/journald.go @@ -79,7 +79,7 @@ func (c JournaldInputConfig) Build(buildContext operator.BuildContext) (operator return journaldInput, nil } -// JournaldInput is a operator that process logs using journald +// JournaldInput is an operator that process logs using journald type JournaldInput struct { helper.InputPlugin diff --git a/operator/builtin/input/tcp.go b/operator/builtin/input/tcp.go index cd555e3c1..215ff1e93 100644 --- a/operator/builtin/input/tcp.go +++ b/operator/builtin/input/tcp.go @@ -52,7 +52,7 @@ func (c TCPInputConfig) Build(context operator.BuildContext) (operator.Operator, return tcpInput, nil } -// TCPInput is a operator that listens for log entries over tcp. +// TCPInput is an operator that listens for log entries over tcp. type TCPInput struct { helper.InputOperator address *net.TCPAddr diff --git a/operator/builtin/input/udp.go b/operator/builtin/input/udp.go index 1331e0d13..7080cd917 100644 --- a/operator/builtin/input/udp.go +++ b/operator/builtin/input/udp.go @@ -51,7 +51,7 @@ func (c UDPInputConfig) Build(context operator.BuildContext) (operator.Operator, return udpInput, nil } -// UDPInput is a operator that listens to a socket for log entries. +// UDPInput is an operator that listens to a socket for log entries. type UDPInput struct { helper.InputOperator address *net.UDPAddr diff --git a/operator/builtin/output/drop.go b/operator/builtin/output/drop.go index 9d0d650ad..51ce22983 100644 --- a/operator/builtin/output/drop.go +++ b/operator/builtin/output/drop.go @@ -37,7 +37,7 @@ func (c DropOutputConfig) Build(context operator.BuildContext) (operator.Operato return dropOutput, nil } -// DropOutput is a operator that consumes and ignores incoming entries. +// DropOutput is an operator that consumes and ignores incoming entries. type DropOutput struct { helper.OutputOperator } diff --git a/operator/builtin/output/elastic.go b/operator/builtin/output/elastic.go index e2c37d6ef..91e8a21d1 100644 --- a/operator/builtin/output/elastic.go +++ b/operator/builtin/output/elastic.go @@ -84,7 +84,7 @@ func (c ElasticOutputConfig) Build(context operator.BuildContext) (operator.Oper return elasticOutput, nil } -// ElasticOutput is a operator that sends entries to elasticsearch. +// ElasticOutput is an operator that sends entries to elasticsearch. type ElasticOutput struct { helper.OutputOperator buffer.Buffer diff --git a/operator/builtin/output/file.go b/operator/builtin/output/file.go index 00ba693c5..13ad7ce2d 100644 --- a/operator/builtin/output/file.go +++ b/operator/builtin/output/file.go @@ -59,7 +59,7 @@ func (c FileOutputConfig) Build(context operator.BuildContext) (operator.Operato return fileOutput, nil } -// FileOutput is a operator that writes logs to a file. +// FileOutput is an operator that writes logs to a file. type FileOutput struct { helper.OutputOperator diff --git a/operator/builtin/output/google_cloud.go b/operator/builtin/output/google_cloud.go index 38a691d76..5c8530988 100644 --- a/operator/builtin/output/google_cloud.go +++ b/operator/builtin/output/google_cloud.go @@ -79,7 +79,7 @@ func (c GoogleCloudOutputConfig) Build(buildContext operator.BuildContext) (oper return googleCloudOutput, nil } -// GoogleCloudOutput is a operator that sends logs to google cloud logging. +// GoogleCloudOutput is an operator that sends logs to google cloud logging. type GoogleCloudOutput struct { helper.OutputOperator buffer.Buffer diff --git a/operator/builtin/output/stdout.go b/operator/builtin/output/stdout.go index a8be92ce8..537e850e2 100644 --- a/operator/builtin/output/stdout.go +++ b/operator/builtin/output/stdout.go @@ -43,7 +43,7 @@ func (c StdoutConfig) Build(context operator.BuildContext) (operator.Operator, e }, nil } -// StdoutOperator is a operator that logs entries using stdout. +// StdoutOperator is an operator that logs entries using stdout. type StdoutOperator struct { helper.OutputOperator encoder *json.Encoder diff --git a/operator/builtin/parser/json.go b/operator/builtin/parser/json.go index 3daac0000..5cc4a028e 100644 --- a/operator/builtin/parser/json.go +++ b/operator/builtin/parser/json.go @@ -40,7 +40,7 @@ func (c JSONParserConfig) Build(context operator.BuildContext) (operator.Operato return jsonParser, nil } -// JSONParser is a operator that parses JSON. +// JSONParser is an operator that parses JSON. type JSONParser struct { helper.ParserOperator json jsoniter.API diff --git a/operator/builtin/parser/regex.go b/operator/builtin/parser/regex.go index 8392dd196..8e41a3e4d 100644 --- a/operator/builtin/parser/regex.go +++ b/operator/builtin/parser/regex.go @@ -65,7 +65,7 @@ func (c RegexParserConfig) Build(context operator.BuildContext) (operator.Operat return regexParser, nil } -// RegexParser is a operator that parses regex in an entry. +// RegexParser is an operator that parses regex in an entry. type RegexParser struct { helper.ParserOperator regexp *regexp.Regexp diff --git a/operator/builtin/parser/severity.go b/operator/builtin/parser/severity.go index 8a7bc2d5d..28c701a6a 100644 --- a/operator/builtin/parser/severity.go +++ b/operator/builtin/parser/severity.go @@ -46,7 +46,7 @@ func (c SeverityParserConfig) Build(context operator.BuildContext) (operator.Ope return severityOperator, nil } -// SeverityParserOperator is a operator that parses time from a field to an entry. +// SeverityParserOperator is an operator that parses time from a field to an entry. type SeverityParserOperator struct { helper.TransformerOperator helper.SeverityParser diff --git a/operator/builtin/parser/syslog.go b/operator/builtin/parser/syslog.go index 87b2c390e..e6ac61e70 100644 --- a/operator/builtin/parser/syslog.go +++ b/operator/builtin/parser/syslog.go @@ -73,7 +73,7 @@ func buildMachine(protocol string) (syslog.Machine, error) { } } -// SyslogParser is a operator that parses syslog. +// SyslogParser is an operator that parses syslog. type SyslogParser struct { helper.ParserOperator machine syslog.Machine diff --git a/operator/builtin/parser/time.go b/operator/builtin/parser/time.go index 7453d45bf..9776f9af2 100644 --- a/operator/builtin/parser/time.go +++ b/operator/builtin/parser/time.go @@ -45,7 +45,7 @@ func (c TimeParserConfig) Build(context operator.BuildContext) (operator.Operato return timeParser, nil } -// TimeParserOperator is a operator that parses time from a field to an entry. +// TimeParserOperator is an operator that parses time from a field to an entry. type TimeParserOperator struct { helper.TransformerOperator helper.TimeParser diff --git a/operator/builtin/transformer/k8s_metadata_decorator.go b/operator/builtin/transformer/k8s_metadata_decorator.go index dc1d6a5d7..5a7fdf9aa 100644 --- a/operator/builtin/transformer/k8s_metadata_decorator.go +++ b/operator/builtin/transformer/k8s_metadata_decorator.go @@ -50,7 +50,7 @@ func (c K8sMetadataDecoratorConfig) Build(context operator.BuildContext) (operat }, nil } -// K8sMetadataDecorator is a operator for decorating entries with kubernetes metadata +// K8sMetadataDecorator is an operator for decorating entries with kubernetes metadata type K8sMetadataDecorator struct { helper.TransformerOperator podNameField entry.Field diff --git a/operator/builtin/transformer/metadata.go b/operator/builtin/transformer/metadata.go index 9aaab64be..0c8a06692 100644 --- a/operator/builtin/transformer/metadata.go +++ b/operator/builtin/transformer/metadata.go @@ -19,7 +19,7 @@ func NewMetadataOperatorConfig(operatorID string) *MetadataOperatorConfig { } } -// MetadataOperatorConfig is the configuration of a metadata operator +// MetadataOperatorConfig is the configuration of a metadatan operator type MetadataOperatorConfig struct { helper.TransformerConfig `yaml:",inline"` @@ -27,7 +27,7 @@ type MetadataOperatorConfig struct { Tags []helper.ExprStringConfig `json:"tags" yaml:"tags"` } -// Build will build a metadata operator from the supplied configuration +// Build will build a metadatan operator from the supplied configuration func (c MetadataOperatorConfig) Build(context operator.BuildContext) (operator.Operator, error) { transformerOperator, err := c.TransformerConfig.Build(context) if err != nil { @@ -53,7 +53,7 @@ func (c MetadataOperatorConfig) Build(context operator.BuildContext) (operator.O return restructureOperator, nil } -// MetadataOperator is a operator that can add metadata to incoming entries +// MetadataOperator is an operator that can add metadata to incoming entries type MetadataOperator struct { helper.TransformerOperator labeler *labeler diff --git a/operator/builtin/transformer/noop.go b/operator/builtin/transformer/noop.go index 4221937ca..77e75322a 100644 --- a/operator/builtin/transformer/noop.go +++ b/operator/builtin/transformer/noop.go @@ -37,7 +37,7 @@ func (c NoopOperatorConfig) Build(context operator.BuildContext) (operator.Opera return noopOperator, nil } -// NoopOperator is a operator that performs no operations on an entry. +// NoopOperator is an operator that performs no operations on an entry. type NoopOperator struct { helper.TransformerOperator } diff --git a/operator/builtin/transformer/rate_limit.go b/operator/builtin/transformer/rate_limit.go index 657b352d1..2e97dbde1 100644 --- a/operator/builtin/transformer/rate_limit.go +++ b/operator/builtin/transformer/rate_limit.go @@ -57,7 +57,7 @@ func (c RateLimitConfig) Build(context operator.BuildContext) (operator.Operator return rateLimitOperator, nil } -// RateLimitOperator is a operator that limits the rate of log consumption between operators. +// RateLimitOperator is an operator that limits the rate of log consumption between operators. type RateLimitOperator struct { helper.TransformerOperator diff --git a/operator/builtin/transformer/restructure.go b/operator/builtin/transformer/restructure.go index c6f603f9d..2f1e847ca 100644 --- a/operator/builtin/transformer/restructure.go +++ b/operator/builtin/transformer/restructure.go @@ -45,7 +45,7 @@ func (c RestructureOperatorConfig) Build(context operator.BuildContext) (operato return restructureOperator, nil } -// RestructureOperator is a operator that can restructure incoming entries using operations +// RestructureOperator is an operator that can restructure incoming entries using operations type RestructureOperator struct { helper.TransformerOperator ops []Op diff --git a/operator/builtin/transformer/router.go b/operator/builtin/transformer/router.go index 3b4205775..93730887b 100644 --- a/operator/builtin/transformer/router.go +++ b/operator/builtin/transformer/router.go @@ -74,7 +74,7 @@ func (c *RouterOperatorConfig) SetNamespace(namespace string, exclusions ...stri } } -// RouterOperator is a operator that routes entries based on matching expressions +// RouterOperator is an operator that routes entries based on matching expressions type RouterOperator struct { helper.BasicOperator routes []*RouterOperatorRoute @@ -155,7 +155,7 @@ func (p *RouterOperator) findOperators(operators []operator.Operator, operatorID return result, nil } -// findOperator will find a operator from a collection. +// findOperator will find an operator from a collection. func (p *RouterOperator) findOperator(operators []operator.Operator, operatorID string) (operator.Operator, error) { for _, operator := range operators { if operator.ID() == operatorID { diff --git a/operator/config.go b/operator/config.go index 114207f98..31d2cc399 100644 --- a/operator/config.go +++ b/operator/config.go @@ -8,7 +8,7 @@ import ( "go.uber.org/zap" ) -// Config is the configuration of a operator +// Config is the configuration of an operator type Config struct { Builder } @@ -23,7 +23,7 @@ type Builder interface { // BuildContext supplies contextual resources when building an operator. type BuildContext struct { - CustomRegistry CustomRegistry + PluginRegistry PluginRegistry Database Database Logger *zap.SugaredLogger } @@ -61,13 +61,13 @@ func NewStubDatabase() *StubDatabase { // registry is a global registry of operator types to operator builders. var registry = make(map[string]func() Builder) -// Register will register a function to a operator type. +// Register will register a function to an operator type. // This function will return a builder for the supplied type. func Register(operatorType string, newBuilder func() Builder) { registry[operatorType] = newBuilder } -// IsDefined will return a boolean indicating if a operator type is registered and defined. +// IsDefined will return a boolean indicating if an operator type is registered and defined. func IsDefined(operatorType string) bool { _, ok := registry[operatorType] return ok diff --git a/operator/config_test.go b/operator/config_test.go index 00848e0a7..d8930b3d0 100644 --- a/operator/config_test.go +++ b/operator/config_test.go @@ -32,8 +32,8 @@ type FakeBuilder struct { func (f *FakeBuilder) SetNamespace(s string, e ...string) {} func (f *FakeBuilder) Build(context BuildContext) (Operator, error) { return nil, nil } -func (f *FakeBuilder) ID() string { return "custom" } -func (f *FakeBuilder) Type() string { return "custom" } +func (f *FakeBuilder) ID() string { return "plugin" } +func (f *FakeBuilder) Type() string { return "plugin" } func TestUnmarshalJSONErrors(t *testing.T) { t.Run("InvalidJSON", func(t *testing.T) { @@ -61,8 +61,8 @@ func TestUnmarshalJSONErrors(t *testing.T) { }) t.Run("TypeSpecificUnmarshal", func(t *testing.T) { - raw := `{"id":"custom","type":"custom","array":"non-array-value"}` - Register("custom", func() Builder { return &FakeBuilder{} }) + raw := `{"id":"plugin","type":"plugin","array":"non-array-value"}` + Register("plugin", func() Builder { return &FakeBuilder{} }) var cfg Config err := json.Unmarshal([]byte(raw), &cfg) require.Error(t, err) @@ -73,14 +73,14 @@ func TestUnmarshalJSONErrors(t *testing.T) { func TestMarshalJSON(t *testing.T) { cfg := Config{ Builder: &FakeBuilder{ - OperatorID: "custom", - OperatorType: "custom", + OperatorID: "plugin", + OperatorType: "plugin", Array: []string{"test"}, }, } out, err := json.Marshal(cfg) require.NoError(t, err) - expected := `{"id":"custom","type":"custom","array":["test"]}` + expected := `{"id":"plugin","type":"plugin","array":["test"]}` require.Equal(t, expected, string(out)) } @@ -94,7 +94,7 @@ func TestUnmarshalYAMLErrors(t *testing.T) { }) t.Run("MissingType", func(t *testing.T) { - raw := "id: custom\n" + raw := "id: plugin\n" var cfg Config err := yaml.Unmarshal([]byte(raw), &cfg) require.Error(t, err) @@ -102,7 +102,7 @@ func TestUnmarshalYAMLErrors(t *testing.T) { }) t.Run("NonStringType", func(t *testing.T) { - raw := "id: custom\ntype: 123" + raw := "id: plugin\ntype: 123" var cfg Config err := yaml.Unmarshal([]byte(raw), &cfg) require.Error(t, err) @@ -110,7 +110,7 @@ func TestUnmarshalYAMLErrors(t *testing.T) { }) t.Run("UnknownType", func(t *testing.T) { - raw := "id: custom\ntype: unknown\n" + raw := "id: plugin\ntype: unknown\n" var cfg Config err := yaml.Unmarshal([]byte(raw), &cfg) require.Error(t, err) @@ -118,8 +118,8 @@ func TestUnmarshalYAMLErrors(t *testing.T) { }) t.Run("TypeSpecificUnmarshal", func(t *testing.T) { - raw := "id: custom\ntype: custom\narray: nonarray" - Register("custom", func() Builder { return &FakeBuilder{} }) + raw := "id: plugin\ntype: plugin\narray: nonarray" + Register("plugin", func() Builder { return &FakeBuilder{} }) var cfg Config err := yaml.Unmarshal([]byte(raw), &cfg) require.Error(t, err) @@ -130,13 +130,13 @@ func TestUnmarshalYAMLErrors(t *testing.T) { func TestMarshalYAML(t *testing.T) { cfg := Config{ Builder: &FakeBuilder{ - OperatorID: "custom", - OperatorType: "custom", + OperatorID: "plugin", + OperatorType: "plugin", Array: []string{"test"}, }, } out, err := yaml.Marshal(cfg) require.NoError(t, err) - expected := "id: custom\ntype: custom\narray:\n- test\n" + expected := "id: plugin\ntype: plugin\narray:\n- test\n" require.Equal(t, expected, string(out)) } diff --git a/operator/helper/plugin.go b/operator/helper/operator.go similarity index 94% rename from operator/helper/plugin.go rename to operator/helper/operator.go index 4d1333242..bb1838719 100644 --- a/operator/helper/plugin.go +++ b/operator/helper/operator.go @@ -13,7 +13,7 @@ func NewBasicConfig(operatorID, operatorType string) BasicConfig { } } -// BasicConfig provides a basic implemention for a operator config. +// BasicConfig provides a basic implemention for an operator config. type BasicConfig struct { OperatorID string `json:"id" yaml:"id"` OperatorType string `json:"type" yaml:"type"` @@ -67,7 +67,7 @@ func (c *BasicConfig) SetNamespace(namespace string, exclusions ...string) { } } -// BasicOperator provides a basic implementation of a operator. +// BasicOperator provides a basic implementation of an operator. type BasicOperator struct { OperatorID string OperatorType string diff --git a/operator/helper/plugin_test.go b/operator/helper/operator_test.go similarity index 100% rename from operator/helper/plugin_test.go rename to operator/helper/operator_test.go diff --git a/operator/helper/writer.go b/operator/helper/writer.go index e5be7b0c0..2c3b8b5f6 100644 --- a/operator/helper/writer.go +++ b/operator/helper/writer.go @@ -45,7 +45,7 @@ func (c *WriterConfig) SetNamespace(namespace string, exclusions ...string) { } } -// WriterOperator is a operator that can write to other operators. +// WriterOperator is an operator that can write to other operators. type WriterOperator struct { BasicOperator OutputIDs OutputIDs @@ -117,7 +117,7 @@ func (w *WriterOperator) SetOutputs(operators []operator.Operator) error { return nil } -// FindOperator will find a operator matching the supplied id. +// FindOperator will find an operator matching the supplied id. func (w *WriterOperator) findOperator(operators []operator.Operator, operatorID string) (operator.Operator, bool) { for _, operator := range operators { if operator.ID() == operatorID { diff --git a/operator/operator.go b/operator/operator.go index 22fc0f28a..7c1215054 100644 --- a/operator/operator.go +++ b/operator/operator.go @@ -30,7 +30,7 @@ type Operator interface { // CanProcess indicates if the operator will process entries from other operators. CanProcess() bool - // Process will process an entry from a operator. + // Process will process an entry from an operator. Process(context.Context, *entry.Entry) error // Logger returns the operator's logger Logger() *zap.SugaredLogger diff --git a/operator/custom.go b/operator/plugin.go similarity index 53% rename from operator/custom.go rename to operator/plugin.go index 25aa5d760..6e54f27d6 100644 --- a/operator/custom.go +++ b/operator/plugin.go @@ -12,31 +12,31 @@ import ( yaml "gopkg.in/yaml.v2" ) -// CustomConfig is the rendered config of a custom plugin. -type CustomConfig struct { +// PluginConfig is the rendered config of a plugin. +type PluginConfig struct { Version string Title string Description string - Parameters map[string]CustomParameter + Parameters map[string]PluginParameter Pipeline []Config } -// CustomParameter is a basic description of a custom plugin's parameter. -type CustomParameter struct { +// PluginParameter is a basic description of a plugin's parameter. +type PluginParameter struct { Label string Description string Type string } -// CustomRegistry is a registry of custom plugin templates. -type CustomRegistry map[string]*template.Template +// PluginRegistry is a registry of plugin templates. +type PluginRegistry map[string]*template.Template -// Render will render a custom config using the params and plugin type. -func (r CustomRegistry) Render(pluginType string, params map[string]interface{}) (CustomConfig, error) { +// Render will render a plugin config using the params and plugin type. +func (r PluginRegistry) Render(pluginType string, params map[string]interface{}) (PluginConfig, error) { template, ok := r[pluginType] if !ok { - return CustomConfig{}, errors.NewError( - "custom plugin type does not exist", + return PluginConfig{}, errors.NewError( + "plugin type does not exist", "ensure that all plugins are defined with a registered type", "plugin_type", pluginType, ) @@ -44,19 +44,19 @@ func (r CustomRegistry) Render(pluginType string, params map[string]interface{}) var writer bytes.Buffer if err := template.Execute(&writer, params); err != nil { - return CustomConfig{}, errors.NewError( - "failed to render template for custom plugin", - "ensure that all parameters are valid for the custom plugin", + return PluginConfig{}, errors.NewError( + "failed to render template for plugin", + "ensure that all parameters are valid for the plugin", "plugin_type", pluginType, "error_message", err.Error(), ) } - var config CustomConfig + var config PluginConfig if err := yaml.UnmarshalStrict(writer.Bytes(), &config); err != nil { - return CustomConfig{}, errors.NewError( - "failed to unmarshal custom template to custom config", - "ensure that the custom template renders a valid pipeline", + return PluginConfig{}, errors.NewError( + "failed to unmarshal plugin template to plugin config", + "ensure that the plugin template renders a valid pipeline", "plugin_type", pluginType, "rendered_config", writer.String(), "error_message", err.Error(), @@ -66,19 +66,19 @@ func (r CustomRegistry) Render(pluginType string, params map[string]interface{}) return config, nil } -// IsDefined returns a boolean indicating if a custom plugin is defined and registered. -func (r CustomRegistry) IsDefined(pluginType string) bool { +// IsDefined returns a boolean indicating if a plugin is defined and registered. +func (r PluginRegistry) IsDefined(pluginType string) bool { _, ok := r[pluginType] return ok } -// LoadAll will load all custom plugin templates contained in a directory. -func (r CustomRegistry) LoadAll(dir string, pattern string) error { +// LoadAll will load all plugin templates contained in a directory. +func (r PluginRegistry) LoadAll(dir string, pattern string) error { glob := filepath.Join(dir, pattern) filePaths, err := filepath.Glob(glob) if err != nil { return errors.NewError( - "failed to find custom plugins with glob pattern", + "failed to find plugins with glob pattern", "ensure that the plugin directory and file pattern are valid", "glob_pattern", glob, ) @@ -93,7 +93,7 @@ func (r CustomRegistry) LoadAll(dir string, pattern string) error { if len(failures) > 0 { return errors.NewError( - "failed to load all custom plugins", + "failed to load all plugins", "review the list of failures for more information", "failures", strings.Join(failures, ", "), ) @@ -102,8 +102,8 @@ func (r CustomRegistry) LoadAll(dir string, pattern string) error { return nil } -// Load will load a custom plugin template from a file path. -func (r CustomRegistry) Load(path string) error { +// Load will load a plugin template from a file path. +func (r PluginRegistry) Load(path string) error { fileName := filepath.Base(path) pluginType := strings.TrimSuffix(fileName, filepath.Ext(fileName)) @@ -115,24 +115,24 @@ func (r CustomRegistry) Load(path string) error { return r.Add(pluginType, string(fileContents)) } -// Add will add a custom plugin to the registry. -func (r CustomRegistry) Add(pluginType string, contents string) error { +// Add will add a plugin to the registry. +func (r PluginRegistry) Add(pluginType string, contents string) error { if IsDefined(pluginType) { return fmt.Errorf("plugin type %s already exists as a builtin plugin", pluginType) } pluginTemplate, err := template.New(pluginType).Parse(contents) if err != nil { - return fmt.Errorf("failed to parse %s as a custom template: %s", pluginType, err) + return fmt.Errorf("failed to parse %s as a plugin template: %s", pluginType, err) } r[pluginType] = pluginTemplate return nil } -// NewCustomRegistry creates a new custom plugin registry from a plugin directory. -func NewCustomRegistry(dir string) (CustomRegistry, error) { - registry := CustomRegistry{} +// NewPluginRegistry creates a new plugin registry from a plugin directory. +func NewPluginRegistry(dir string) (PluginRegistry, error) { + registry := PluginRegistry{} if err := registry.LoadAll(dir, "*.yaml"); err != nil { return registry, err } diff --git a/operator/custom_test.go b/operator/plugin_test.go similarity index 88% rename from operator/custom_test.go rename to operator/plugin_test.go index 5532b909d..43d9b421e 100644 --- a/operator/custom_test.go +++ b/operator/plugin_test.go @@ -23,7 +23,7 @@ func NewTempDir(t *testing.T) string { return tempDir } -func TestCustomRegistry_LoadAll(t *testing.T) { +func TestPluginRegistry_LoadAll(t *testing.T) { tempDir, err := ioutil.TempDir("", "") require.NoError(t, err) t.Cleanup(func() { @@ -51,48 +51,48 @@ record: err = ioutil.WriteFile(filepath.Join(tempDir, "test2.yaml"), test2, 0666) require.NoError(t, err) - customRegistry := CustomRegistry{} - err = customRegistry.LoadAll(tempDir, "*.yaml") + pluginRegistry := PluginRegistry{} + err = pluginRegistry.LoadAll(tempDir, "*.yaml") require.NoError(t, err) - require.Equal(t, 2, len(customRegistry)) + require.Equal(t, 2, len(pluginRegistry)) } -func TestCustomRegistryRender(t *testing.T) { +func TestPluginRegistryRender(t *testing.T) { t.Run("ErrorTypeDoesNotExist", func(t *testing.T) { - reg := CustomRegistry{} + reg := PluginRegistry{} _, err := reg.Render("unknown", map[string]interface{}{}) require.Error(t, err) require.Contains(t, err.Error(), "does not exist") }) t.Run("ErrorExecFailure", func(t *testing.T) { - tmpl, err := template.New("customtype").Parse(`{{ .panicker }}`) + tmpl, err := template.New("plugintype").Parse(`{{ .panicker }}`) require.NoError(t, err) - reg := CustomRegistry{ - "customtype": tmpl, + reg := PluginRegistry{ + "plugintype": tmpl, } params := map[string]interface{}{ "panicker": func() { panic("testpanic") }, } - _, err = reg.Render("customtype", params) + _, err = reg.Render("plugintype", params) require.Contains(t, err.Error(), "failed to render") }) } -func TestCustomRegistryLoad(t *testing.T) { +func TestPluginRegistryLoad(t *testing.T) { t.Run("LoadAllBadGlob", func(t *testing.T) { - reg := CustomRegistry{} + reg := PluginRegistry{} err := reg.LoadAll("", `[]`) require.Error(t, err) require.Contains(t, err.Error(), "with glob pattern") }) t.Run("AddDuplicate", func(t *testing.T) { - reg := CustomRegistry{} + reg := PluginRegistry{} Register("copy", func() Builder { return nil }) err := reg.Add("copy", "pipeline:\n") require.Error(t, err) @@ -100,10 +100,10 @@ func TestCustomRegistryLoad(t *testing.T) { }) t.Run("AddBadTemplate", func(t *testing.T) { - reg := CustomRegistry{} + reg := PluginRegistry{} err := reg.Add("new", "{{ nofunc }") require.Error(t, err) - require.Contains(t, err.Error(), "as a custom template") + require.Contains(t, err.Error(), "as a plugin template") }) t.Run("LoadAllWithFailures", func(t *testing.T) { @@ -112,13 +112,13 @@ func TestCustomRegistryLoad(t *testing.T) { err := ioutil.WriteFile(pluginPath, []byte("pipeline:\n"), 0755) require.NoError(t, err) - reg := CustomRegistry{} + reg := PluginRegistry{} err = reg.LoadAll(tempDir, "*.yaml") require.Error(t, err) }) } -func TestCustomPluginMetadata(t *testing.T) { +func TestPluginMetadata(t *testing.T) { testCases := []struct { name string @@ -280,7 +280,7 @@ pipeline: for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - reg := CustomRegistry{} + reg := PluginRegistry{} err := reg.Add(tc.name, tc.template) require.NoError(t, err) _, err = reg.Render(tc.name, map[string]interface{}{}) diff --git a/pipeline/config.go b/pipeline/config.go index 7c6eafdc4..35867887e 100644 --- a/pipeline/config.go +++ b/pipeline/config.go @@ -15,7 +15,7 @@ type Config []Params // BuildPipeline will build a pipeline from the config. func (c Config) BuildPipeline(context operator.BuildContext) (*Pipeline, error) { - operatorConfigs, err := c.buildOperatorConfigs(context.CustomRegistry) + operatorConfigs, err := c.buildOperatorConfigs(context.PluginRegistry) if err != nil { return nil, err } @@ -51,7 +51,7 @@ func (c Config) buildOperators(operatorConfigs []operator.Config, context operat return operators, nil } -func (c Config) buildOperatorConfigs(customRegistry operator.CustomRegistry) ([]operator.Config, error) { +func (c Config) buildOperatorConfigs(pluginRegistry operator.PluginRegistry) ([]operator.Config, error) { operatorConfigs := make([]operator.Config, 0, len(c)) for _, params := range c { @@ -59,7 +59,7 @@ func (c Config) buildOperatorConfigs(customRegistry operator.CustomRegistry) ([] return nil, errors.Wrap(err, "validate config params") } - configs, err := params.BuildConfigs(customRegistry, "$") + configs, err := params.BuildConfigs(pluginRegistry, "$") if err != nil { return nil, errors.Wrap(err, "build operator configs") } @@ -69,7 +69,7 @@ func (c Config) buildOperatorConfigs(customRegistry operator.CustomRegistry) ([] return operatorConfigs, nil } -// Params is a raw params map that can be converted into a operator config. +// Params is a raw params map that can be converted into an operator config. type Params map[string]interface{} // ID returns the id field in the params map. @@ -124,7 +124,7 @@ func (p Params) NamespaceExclusions(namespace string) []string { return exclusions } -// Validate will validate the basic fields required to make a operator config. +// Validate will validate the basic fields required to make an operator config. func (p Params) Validate() error { if p.Type() == "" { return errors.NewError( @@ -178,18 +178,18 @@ func (p Params) getStringArray(key string) []string { } // BuildConfigs will build operator configs from a params map. -func (p Params) BuildConfigs(customRegistry operator.CustomRegistry, namespace string) ([]operator.Config, error) { +func (p Params) BuildConfigs(pluginRegistry operator.PluginRegistry, namespace string) ([]operator.Config, error) { if operator.IsDefined(p.Type()) { return p.buildAsBuiltin(namespace) } - if customRegistry.IsDefined(p.Type()) { - return p.buildAsCustom(customRegistry, namespace) + if pluginRegistry.IsDefined(p.Type()) { + return p.buildPlugin(pluginRegistry, namespace) } return nil, errors.NewError( "unsupported `type` for operator config", - "ensure that all operators have a supported builtin or custom type", + "ensure that all operators have a supported builtin or plugin type", "type", p.Type(), "id", p.ID(), ) @@ -215,8 +215,8 @@ func (p Params) buildAsBuiltin(namespace string) ([]operator.Config, error) { return []operator.Config{config}, nil } -// buildAsCustom will build a custom config from a params map. -func (p Params) buildAsCustom(customRegistry operator.CustomRegistry, namespace string) ([]operator.Config, error) { +// buildPlugin will build a plugin config from a params map. +func (p Params) buildPlugin(pluginRegistry operator.PluginRegistry, namespace string) ([]operator.Config, error) { templateParams := map[string]interface{}{} for key, value := range p { templateParams[key] = value @@ -225,9 +225,9 @@ func (p Params) buildAsCustom(customRegistry operator.CustomRegistry, namespace templateParams["input"] = p.TemplateInput(namespace) templateParams["output"] = p.TemplateOutput(namespace) - config, err := customRegistry.Render(p.Type(), templateParams) + config, err := pluginRegistry.Render(p.Type(), templateParams) if err != nil { - return nil, errors.Wrap(err, "failed to render custom config") + return nil, errors.Wrap(err, "failed to render plugin config") } exclusions := p.NamespaceExclusions(namespace) diff --git a/pipeline/config_test.go b/pipeline/config_test.go index 498bbc962..d244d7af7 100644 --- a/pipeline/config_test.go +++ b/pipeline/config_test.go @@ -188,7 +188,7 @@ func TestBuildBuiltinFromParamsWithUnsupportedYaml(t *testing.T) { "output": "test", "field": invalidMarshaller{}, } - _, err := params.BuildConfigs(operator.CustomRegistry{}, "test_namespace") + _, err := params.BuildConfigs(operator.PluginRegistry{}, "test_namespace") require.Error(t, err) require.Contains(t, err.Error(), "failed to parse config map as yaml") } @@ -200,7 +200,7 @@ func TestBuildBuiltinFromParamsWithUnknownField(t *testing.T) { "unknown": true, "output": "test_output", } - _, err := params.BuildConfigs(operator.CustomRegistry{}, "test_namespace") + _, err := params.BuildConfigs(operator.PluginRegistry{}, "test_namespace") require.Error(t, err) } @@ -210,7 +210,7 @@ func TestBuildBuiltinFromValidParams(t *testing.T) { "type": "noop", "output": "test_output", } - configs, err := params.BuildConfigs(operator.CustomRegistry{}, "test_namespace") + configs, err := params.BuildConfigs(operator.PluginRegistry{}, "test_namespace") require.NoError(t, err) require.Equal(t, 1, len(configs)) @@ -218,20 +218,20 @@ func TestBuildBuiltinFromValidParams(t *testing.T) { require.Equal(t, "test_namespace.noop", configs[0].ID()) } -func TestBuildCustomFromValidParams(t *testing.T) { - registry := operator.CustomRegistry{} - customTemplate := ` +func TestBuildPluginFromValidParams(t *testing.T) { + registry := operator.PluginRegistry{} + pluginTemplate := ` pipeline: - - id: custom_noop + - id: plugin_noop type: noop output: {{.output}} ` - err := registry.Add("custom_operator", customTemplate) + err := registry.Add("plugin", pluginTemplate) require.NoError(t, err) params := Params{ - "id": "custom_operator", - "type": "custom_operator", + "id": "plugin", + "type": "plugin", "output": "test_output", } @@ -239,14 +239,14 @@ pipeline: require.NoError(t, err) require.Equal(t, 1, len(configs)) require.IsType(t, &transformer.NoopOperatorConfig{}, configs[0].Builder) - require.Equal(t, "test_namespace.custom_operator.custom_noop", configs[0].ID()) + require.Equal(t, "test_namespace.plugin.plugin_noop", configs[0].ID()) } func TestBuildValidPipeline(t *testing.T) { - registry := operator.CustomRegistry{} - customTemplate := ` + registry := operator.PluginRegistry{} + pluginTemplate := ` pipeline: - - id: custom_generate + - id: plugin_generate type: generate_input count: 1 entry: @@ -254,7 +254,7 @@ pipeline: message: test output: {{.output}} ` - err := registry.Add("custom_operator", customTemplate) + err := registry.Add("plugin", pluginTemplate) require.NoError(t, err) logCfg := zap.NewProductionConfig() @@ -262,14 +262,14 @@ pipeline: require.NoError(t, err) context := operator.BuildContext{ - CustomRegistry: registry, + PluginRegistry: registry, Logger: logger.Sugar(), } pipelineConfig := Config{ Params{ - "id": "custom_operator", - "type": "custom_operator", + "id": "plugin", + "type": "plugin", "output": "drop_output", }, Params{ @@ -283,20 +283,20 @@ pipeline: } func TestBuildInvalidPipelineInvalidType(t *testing.T) { - registry := operator.CustomRegistry{} + registry := operator.PluginRegistry{} logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) context := operator.BuildContext{ - CustomRegistry: registry, + PluginRegistry: registry, Logger: logger.Sugar(), } pipelineConfig := Config{ Params{ - "id": "custom_operator", - "type": "custom_operator", + "id": "plugin", + "type": "plugin", "output": "drop_output", }, Params{ @@ -311,17 +311,17 @@ func TestBuildInvalidPipelineInvalidType(t *testing.T) { } func TestBuildInvalidPipelineInvalidParam(t *testing.T) { - registry := operator.CustomRegistry{} - customTemplate := ` + registry := operator.PluginRegistry{} + pluginTemplate := ` pipeline: - - id: custom_generate + - id: plugin_generate type: generate_input count: invalid_value record: message: test output: {{.output}} ` - err := registry.Add("custom_operator", customTemplate) + err := registry.Add("plugin", pluginTemplate) require.NoError(t, err) logCfg := zap.NewProductionConfig() @@ -329,14 +329,14 @@ pipeline: require.NoError(t, err) context := operator.BuildContext{ - CustomRegistry: registry, + PluginRegistry: registry, Logger: logger.Sugar(), } pipelineConfig := Config{ Params{ - "id": "custom_operator", - "type": "custom_operator", + "id": "plugin", + "type": "plugin", "output": "drop_output", }, Params{ @@ -351,13 +351,13 @@ pipeline: } func TestBuildInvalidPipelineInvalidOperator(t *testing.T) { - registry := operator.CustomRegistry{} + registry := operator.PluginRegistry{} logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) context := operator.BuildContext{ - CustomRegistry: registry, + PluginRegistry: registry, Logger: logger.Sugar(), } @@ -379,13 +379,13 @@ func TestBuildInvalidPipelineInvalidOperator(t *testing.T) { } func TestBuildInvalidPipelineInvalidGraph(t *testing.T) { - registry := operator.CustomRegistry{} + registry := operator.PluginRegistry{} logCfg := zap.NewProductionConfig() logger, err := logCfg.Build() require.NoError(t, err) context := operator.BuildContext{ - CustomRegistry: registry, + PluginRegistry: registry, Logger: logger.Sugar(), } diff --git a/pipeline/node.go b/pipeline/node.go index 4dc0e4a36..316a0822e 100644 --- a/pipeline/node.go +++ b/pipeline/node.go @@ -6,7 +6,7 @@ import ( "github.com/observiq/carbon/operator" ) -// OperatorNode is a basic node that represents a operator in a pipeline. +// OperatorNode is a basic node that represents an operator in a pipeline. type OperatorNode struct { operator operator.Operator id int64 @@ -33,7 +33,7 @@ func (b OperatorNode) OutputIDs() map[string]int64 { return b.outputIDs } -// createOperatorNode will create a operator node. +// createOperatorNode will create an operator node. func createOperatorNode(operator operator.Operator) OperatorNode { id := createNodeID(operator.ID()) outputIDs := make(map[string]int64) @@ -45,7 +45,7 @@ func createOperatorNode(operator operator.Operator) OperatorNode { return OperatorNode{operator, id, outputIDs} } -// createNodeID generates a node id from a operator id. +// createNodeID generates a node id from an operator id. func createNodeID(operatorID string) int64 { hash := fnv.New64a() _, _ = hash.Write([]byte(operatorID)) From 5a8772214f836f6b8ef5bea58ebca321141b9f6e Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Wed, 22 Jul 2020 12:26:27 -0400 Subject: [PATCH 4/7] Fix typo in contribution.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 416378cf9..bf88d316e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ A PR is considered to be **ready to merge** when: ## Design Choices -Best practices for developing a builtin operators are documented below, but for changes to +Best practices for developing a builtin operator are documented below, but for changes to the core agent, we are happy to discuss proposals in the issue tracker. ### Builtin Operator Development From 5424e8621b46082d1e93331d60622fbb65674356 Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Wed, 22 Jul 2020 12:42:35 -0400 Subject: [PATCH 5/7] Try using GOPROXY=direct for benchmark builds --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ef60ad1b..9030d8e16 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -177,7 +177,7 @@ jobs: command: mkdir {bin,out,tmp} - run: name: Build Carbon Agent - command: go get -v -t -d ./... && go build -v -o ./bin/carbon ./ + command: GOPROXY=direct go get -v -t -d ./... && go build -v -o ./bin/carbon ./ - run: name: Build Log Bench command: GOPROXY=direct go get github.com/observiq/amazon-log-agent-benchmark-tool/cmd/logbench/ && From 30b6fe4152eb7de7b9210aa213216d4e089c1ea9 Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Wed, 22 Jul 2020 12:48:45 -0400 Subject: [PATCH 6/7] Try running go build without go get --- .circleci/config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9030d8e16..97e87ad26 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,9 +116,7 @@ jobs: command: make test test-windows: - executor: - name: win/default - size: large + executor: win/default steps: - checkout @@ -177,7 +175,7 @@ jobs: command: mkdir {bin,out,tmp} - run: name: Build Carbon Agent - command: GOPROXY=direct go get -v -t -d ./... && go build -v -o ./bin/carbon ./ + command: GOPROXY=direct go build -v -o ./bin/carbon ./ - run: name: Build Log Bench command: GOPROXY=direct go get github.com/observiq/amazon-log-agent-benchmark-tool/cmd/logbench/ && From f802544c89188c111fa289e5ad43bfbdd76dc2cd Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Wed, 22 Jul 2020 13:26:54 -0400 Subject: [PATCH 7/7] Migrated journald to new terminology --- go.mod | 3 +- go.sum | 236 ----------------------------- operator/builtin/input/journald.go | 10 +- 3 files changed, 6 insertions(+), 243 deletions(-) diff --git a/go.mod b/go.mod index 13dd206bc..e50328dcb 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/observiq/ctimefmt v1.0.0 github.com/pkg/errors v0.9.1 // indirect github.com/spf13/cobra v1.0.0 - github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.5.1 go.etcd.io/bbolt v1.3.4 go.uber.org/zap v1.15.0 @@ -34,7 +33,7 @@ require ( google.golang.org/grpc v1.27.1 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.3.0 - honnef.co/go/tools v0.0.1-2020.1.3 + honnef.co/go/tools v0.0.1-2020.1.3 // indirect k8s.io/apimachinery v0.18.4 k8s.io/client-go v0.18.4 ) diff --git a/go.sum b/go.sum index 57893a328..e48f4a3a4 100644 --- a/go.sum +++ b/go.sum @@ -1,141 +1,66 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= cloud.google.com/go v0.43.0 h1:banaiRPAM8kUVYneOSkhgcDsLzEvL25FinuiSZaH/2w= cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go/logging v1.0.0 h1:kaunpnoEh9L4hu6JUsBa8Y20LBfKnCuDhKUgdZp7oK8= cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= -contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= -contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= -contrib.go.opencensus.io/exporter/stackdriver v0.11.0/go.mod h1:hA7rlmtavV03FGxzWXAPBUnZeZBhWN/QYQAuMtxc9Bk= -contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= -contrib.go.opencensus.io/resource v0.0.0-20190131005048-21591786a5e0/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= -github.com/Azure/azure-amqp-common-go v1.1.3/go.mod h1:FhZtXirFANw40UXI2ntweO+VOkfaw8s6vZxUiRhLYW8= -github.com/Azure/azure-amqp-common-go v1.1.4/go.mod h1:FhZtXirFANw40UXI2ntweO+VOkfaw8s6vZxUiRhLYW8= -github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-sdk-for-go v21.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v27.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-service-bus-go v0.4.1/go.mod h1:d9ho9e/06euiTwGpKxmlbpPhFUsfCsq6a4tZ68r51qI= -github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= -github.com/Azure/go-autorest v11.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v11.1.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69/go.mod h1:L1AbZdiDllfyYH5l5OkAaZtk7VkWe89bPJFmnDBNHxg= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= -github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= -github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190418212003-6ac0b49e7197/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo= github.com/Mottl/ctimefmt v0.0.0-20190803144728-fd2ac23a585a/go.mod h1:eyj2WSIdoPMPs2eNTLpSmM6Nzqo4V80/d6jHpnJ1SAI= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= -github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= -github.com/alecthomas/chroma v0.7.2-0.20200305040604-4f3623dce67a/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s= -github.com/alecthomas/chroma v0.7.3/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM= -github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= -github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= -github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= -github.com/alecthomas/kong v0.2.4/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= -github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA= -github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= -github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antonmedv/expr v1.8.2 h1:BfkVHGudYqq7jp3Ji33kTn+qZ9D19t/Mndg0ag/Ycq4= github.com/antonmedv/expr v1.8.2/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.18.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.19.16/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= -github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY= -github.com/bep/golibsass v0.6.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA= -github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= -github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= -github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= -github.com/disintegration/gift v1.2.1/go.mod h1:Jh2i7f7Q2BM7Ezno3PhfezbR1xpUg9dUg3/RlKGr4HI= -github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/elastic/go-elasticsearch v0.0.0 h1:Pd5fqOuBxKxv83b0+xOAJDAkziWYwFinWnBO0y+TZaA= -github.com/elastic/go-elasticsearch v0.0.0/go.mod h1:TkBSJBuTyFdBnrNqoPc54FN0vKf5c04IdM4zuStJ7xg= github.com/elastic/go-elasticsearch/v7 v7.7.0 h1:oQBx/S3RiaH0/kiP0scYSay9xgSmVAYJpuqEf+e9GZg= github.com/elastic/go-elasticsearch/v7 v7.7.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= -github.com/elastic/go-elasticsearch/v8 v8.0.0-20200429055951-e9eb76d55d12 h1:MzWVDDyyAsT9yXAkuCQovqdDiZ+cAQTiXqkHdX7wzwo= -github.com/elastic/go-elasticsearch/v8 v8.0.0-20200429055951-e9eb76d55d12/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanw/esbuild v0.6.2/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= -github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= -github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= -github.com/getkin/kin-openapi v0.14.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -144,18 +69,11 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gohugoio/hugo v0.74.2 h1:F1DGnxBo0hG8/JnODhCMNVnrkPn6HCbBtvveudzwiwU= -github.com/gohugoio/hugo v0.74.2/go.mod h1:2YCeGiVVSSdH/8lQ24W+G5mQTcdI0M10RlT7at1EACQ= -github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95/go.mod h1:bOlVlCa1/RajcHpXkrUXPSHB/Re1UnlXxD1Qp8SKOd8= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -172,30 +90,22 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93 h1:VvBteXw2zOXEgm0o3PgONTWf+bhUGsCaiNn3pbkU9LA= -github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.2.2/go.mod h1:7FHVg6mFpFQrjeUZrm+BaD50N5jnDKm50uVPTpyYOmU= github.com/googleapis/gax-go v1.0.3 h1:9dMLqhaibYONnDRcnHdUs9P8Mw64jLlZTYlDe3leBtQ= github.com/googleapis/gax-go v1.0.3/go.mod h1:QyXYajJFdARxGzjwUfbDFIse7Spkw81SJ4LrBJXtlQ8= -github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= -github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.2/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= @@ -203,19 +113,10 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= -github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -228,21 +129,14 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/go-syslog v1.0.1 h1:a/ARpnCDr/sX/hVH7dyQVi+COXlEzM4bNIoolOfw99Y= github.com/influxdata/go-syslog/v3 v3.0.0 h1:jichmjSZlYK0VMmlz+k4WeOQd7z745YLsvGMqwtYt4I= github.com/influxdata/go-syslog/v3 v3.0.0/go.mod h1:tulsOp+CecTAYC27u9miMgq21GqXRW6VdKbOG+QSP4Q= -github.com/jdkato/prose v1.1.1/go.mod h1:jkF0lkxaX5PFSlk9l4Gh9Y+T57TqUZziWT7uZbW5ADg= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kardianos/service v1.0.0 h1:HgQS3mFfOlyntWX8Oke98JcJLqt1DBcHR4kxShpYef0= @@ -258,31 +152,15 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/markbates/inflect v1.0.0/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88= -github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/mmark v1.3.6/go.mod h1:w7r9mkTvpS55jlfyn22qJ618itLryxXBhA7Jp3FIlkw= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= @@ -293,37 +171,19 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/muesli/smartcrop v0.3.0/go.mod h1:i2fCI/UorTfgEpPPLWiFBv4pye+YAG78RwcQLUkocpI= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= -github.com/niklasfasching/go-org v1.3.0 h1:X8Ob7WOF61iP4KnisZo1eVVei8/GzmNz3ymJrYLnIic= -github.com/niklasfasching/go-org v1.3.0/go.mod h1:AsLD6X7djzRIz4/RFZu8vwRL0VGjUvGZCCH1Nz0VdrU= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= -github.com/observiq/ctimefmt v0.0.0-20200611210846-e39bd83b5771 h1:vvoch82jTZ2EbgdjMQ437TBFKwzyeiDdwuGvMqG0XeQ= -github.com/observiq/ctimefmt v0.0.0-20200611210846-e39bd83b5771/go.mod h1:jfs3wpVShz6PBYh3nouK+OwrJNxXXZESIKm6xyKzzBY= -github.com/observiq/ctimefmt v0.0.0-20200612160717-3e07deba22cf h1:WTafs6lnKha7XqeguHvaboRwWSHdRwVBVrFQlIEYkvI= -github.com/observiq/ctimefmt v0.0.0-20200612160717-3e07deba22cf/go.mod h1:jfs3wpVShz6PBYh3nouK+OwrJNxXXZESIKm6xyKzzBY= github.com/observiq/ctimefmt v1.0.0 h1:r7vTJ+Slkrt9fZ67mkf+mA6zAdR5nGIJRMTzkUyvilk= github.com/observiq/ctimefmt v1.0.0/go.mod h1:mxi62//WbSpG/roCO1c6MqZ7zQTvjVtYheqHN3eOjvc= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= -github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -332,58 +192,37 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.5.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday v1.5.3-0.20200218234912-41c5fccfd6f6/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/fsync v0.9.0/go.mod h1:fNtJEfG3HiltN3y4cPOz6MLjos9+2pIEqLIgszqhp/0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= -github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -393,39 +232,17 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tdewolff/minify/v2 v2.6.2/go.mod h1:BkDSm8aMMT0ALGmpt7j3Ra7nLUgZL0qhyrAHXwxcy5w= -github.com/tdewolff/parse/v2 v2.4.2/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= -github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= -github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= -github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.31/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691/go.mod h1:YLF3kDffRfUH/bTxOxHhV6lxwIB3Vfj91rEwNMS9MXo= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -437,13 +254,9 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -gocloud.dev v0.15.0/go.mod h1:ShXCyJaGrJu9y/7a6+DSCyBb9MFGZ1P5wwPa0Wu6w34= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= @@ -459,7 +272,6 @@ golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522 h1:OeRHuibLsmZkFj773W4LcfAGs golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20191214001246-9130b4cfad52/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -478,33 +290,23 @@ golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190322120337-addf6b3196f6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190420063019-afa5a82059c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190523182746-aaccbc9213b0/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -518,36 +320,26 @@ golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200513112337-417ce2331b5c h1:kISX68E8gSkNYAFRFiDU8rl5RIn1sJYKYb/r2vMLDrU= golang.org/x/sys v0.0.0-20200513112337-417ce2331b5c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -565,8 +357,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -580,7 +370,6 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97 h1:DAuln/hGp+aJiHpID1Y1hYzMEPP5WLwtZHPb50mN0OE= golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -591,36 +380,27 @@ gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200304201815-d429ff31ee6c h1:Mm69MraVZ+yh1vw8pQOUW4uJkkSEQbbTr076A94lvqs= google.golang.org/genproto v0.0.0-20200304201815-d429ff31ee6c/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -630,7 +410,6 @@ google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -638,15 +417,11 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= -gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -662,27 +437,16 @@ k8s.io/api v0.18.4 h1:8x49nBRxuXGUlDlwlWd3RMY1SayZrzFfxea3UZSkFw4= k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4= k8s.io/apimachinery v0.18.4 h1:ST2beySjhqwJoIFk6p7Hp5v5O0hYY6Gngq/gUYXTPIA= k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apimachinery v0.18.5-rc.0 h1:V04jBUrFtuyH8bXfV/UJUfG7DvleLJsC6yZwAkLiiGE= -k8s.io/apimachinery v0.18.5-rc.0/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/client-go v0.18.4 h1:un55V1Q/B3JO3A76eS0kUSywgGK/WR3BQ8fHQjNa6Zc= k8s.io/client-go v0.18.4/go.mod h1:f5sXwL4yAZRkAtzOxRWUhA/N8XzGCb+nPZI8PfobZ9g= -k8s.io/client-go v1.5.1 h1:XaX/lo2/u3/pmFau8HN+sB5C/b4dc4Dmm2eXjBH4p1E= -k8s.io/client-go v1.5.1/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= -k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= -k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19 h1:7Nu2dTj82c6IaWvL7hImJzcXoTPz1MsSCH7r+0m6rfo= -k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -pack.ag/amqp v0.8.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= -pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/operator/builtin/input/journald.go b/operator/builtin/input/journald.go index a1f387cbf..41f4d80c1 100644 --- a/operator/builtin/input/journald.go +++ b/operator/builtin/input/journald.go @@ -39,8 +39,8 @@ type JournaldInputConfig struct { } // Build will build a journald input operator from the supplied configuration -func (c JournaldInputConfig) Build(buildContext operator.BuildContext) (operator.Plugin, error) { - inputPlugin, err := c.InputConfig.Build(buildContext) +func (c JournaldInputConfig) Build(buildContext operator.BuildContext) (operator.Operator, error) { + inputOperator, err := c.InputConfig.Build(buildContext) if err != nil { return nil, err } @@ -66,8 +66,8 @@ func (c JournaldInputConfig) Build(buildContext operator.BuildContext) (operator } journaldInput := &JournaldInput{ - InputPlugin: inputPlugin, - persist: helper.NewScopedDBPersister(buildContext.Database, c.ID()), + InputOperator: inputOperator, + persist: helper.NewScopedDBPersister(buildContext.Database, c.ID()), newCmd: func(ctx context.Context, cursor []byte) cmd { if cursor != nil { args = append(args, "--after-cursor", string(cursor)) @@ -81,7 +81,7 @@ func (c JournaldInputConfig) Build(buildContext operator.BuildContext) (operator // JournaldInput is an operator that process logs using journald type JournaldInput struct { - helper.InputPlugin + helper.InputOperator newCmd func(ctx context.Context, cursor []byte) cmd