From 366ffb2da66f2b0f0a5b9af980c325d7e8b20bc4 Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Mon, 5 Apr 2021 16:40:54 -0400 Subject: [PATCH 1/4] Remove database, pass persistor to Start methods --- agent/agent.go | 12 +- agent/agent_test.go | 41 ++---- agent/builder.go | 29 +---- agent/builder_test.go | 45 ------- database/database.go | 77 ------------ database/database_test.go | 102 --------------- go.mod | 1 + go.sum | 59 +++++---- operator/build_context.go | 6 +- operator/builtin/input/file/config.go | 1 - operator/builtin/input/file/file.go | 12 +- operator/builtin/input/file/file_test.go | 61 +++++---- operator/builtin/input/generate/generate.go | 2 +- .../builtin/input/generate/generate_test.go | 2 +- operator/builtin/input/journald/journald.go | 45 ++----- .../builtin/input/journald/journald_test.go | 2 +- operator/builtin/input/k8sevent/k8s_event.go | 2 +- operator/builtin/input/stanza/stanza.go | 2 +- operator/builtin/input/stanza/stanza_test.go | 2 +- operator/builtin/input/stdin/stdin.go | 2 +- operator/builtin/input/stdin/stdin_test.go | 2 +- operator/builtin/input/syslog/syslog_test.go | 2 +- operator/builtin/input/tcp/tcp.go | 2 +- operator/builtin/input/tcp/tcp_test.go | 6 +- operator/builtin/input/udp/udp.go | 2 +- operator/builtin/input/udp/udp_test.go | 4 +- operator/builtin/input/windows/operator.go | 22 ++-- operator/builtin/output/file/file.go | 2 +- .../transformer/recombine/recombine.go | 2 +- operator/helper/operator.go | 2 +- operator/helper/operator_test.go | 2 +- operator/helper/persister.go | 117 ------------------ operator/helper/persister_test.go | 51 -------- operator/operator.go | 2 +- operator/persister.go | 22 ++++ pipeline/directed.go | 4 +- pipeline/directed_test.go | 24 ++-- pipeline/pipeline.go | 6 +- plugin/config_test.go | 2 +- testutil/database.go | 70 ----------- testutil/mocks.go | 2 +- testutil/operator.go | 10 +- testutil/operator_builder.go | 8 +- testutil/pipeline.go | 10 +- testutil/util.go | 33 ++++- 45 files changed, 222 insertions(+), 692 deletions(-) delete mode 100644 database/database.go delete mode 100644 database/database_test.go delete mode 100644 operator/helper/persister.go delete mode 100644 operator/helper/persister_test.go create mode 100644 operator/persister.go delete mode 100644 testutil/database.go diff --git a/agent/agent.go b/agent/agent.go index 21db1d53..46bac1e6 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -19,13 +19,12 @@ import ( "go.uber.org/zap" - "github.com/open-telemetry/opentelemetry-log-collection/database" + "github.com/open-telemetry/opentelemetry-log-collection/operator" "github.com/open-telemetry/opentelemetry-log-collection/pipeline" ) // LogAgent is an entity that handles log monitoring. type LogAgent struct { - database database.Database pipeline pipeline.Pipeline startOnce sync.Once @@ -35,9 +34,9 @@ type LogAgent struct { } // Start will start the log monitoring process -func (a *LogAgent) Start() (err error) { +func (a *LogAgent) Start(persister operator.Persister) (err error) { a.startOnce.Do(func() { - err = a.pipeline.Start() + err = a.pipeline.Start(persister) if err != nil { return } @@ -52,11 +51,6 @@ func (a *LogAgent) Stop() (err error) { if err != nil { return } - - err = a.database.Close() - if err != nil { - return - } }) return } diff --git a/agent/agent_test.go b/agent/agent_test.go index eb0130ac..d2559c0c 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -27,48 +27,46 @@ import ( func TestStartAgentSuccess(t *testing.T) { logger := zap.NewNop().Sugar() pipeline := &testutil.Pipeline{} - pipeline.On("Start").Return(nil) + persister := testutil.NewMockPersister() + pipeline.On("Start", persister).Return(nil) agent := LogAgent{ SugaredLogger: logger, pipeline: pipeline, } - err := agent.Start() + err := agent.Start(persister) require.NoError(t, err) - pipeline.AssertCalled(t, "Start") + pipeline.AssertCalled(t, "Start", persister) } func TestStartAgentFailure(t *testing.T) { logger := zap.NewNop().Sugar() pipeline := &testutil.Pipeline{} + persister := testutil.NewMockPersister() failure := fmt.Errorf("failed to start pipeline") - pipeline.On("Start").Return(failure) + pipeline.On("Start", persister).Return(failure) agent := LogAgent{ SugaredLogger: logger, pipeline: pipeline, } - err := agent.Start() + err := agent.Start(persister) require.Error(t, err, failure) - pipeline.AssertCalled(t, "Start") + pipeline.AssertCalled(t, "Start", persister) } func TestStopAgentSuccess(t *testing.T) { logger := zap.NewNop().Sugar() pipeline := &testutil.Pipeline{} pipeline.On("Stop").Return(nil) - database := &testutil.Database{} - database.On("Close").Return(nil) agent := LogAgent{ SugaredLogger: logger, pipeline: pipeline, - database: database, } err := agent.Stop() require.NoError(t, err) pipeline.AssertCalled(t, "Stop") - database.AssertCalled(t, "Close") } func TestStopAgentPipelineFailure(t *testing.T) { @@ -76,35 +74,12 @@ func TestStopAgentPipelineFailure(t *testing.T) { pipeline := &testutil.Pipeline{} failure := fmt.Errorf("failed to start pipeline") pipeline.On("Stop").Return(failure) - database := &testutil.Database{} - database.On("Close").Return(nil) agent := LogAgent{ SugaredLogger: logger, pipeline: pipeline, - database: database, } err := agent.Stop() require.Error(t, err, failure) pipeline.AssertCalled(t, "Stop") - database.AssertNotCalled(t, "Close") -} - -func TestStopAgentDatabaseFailure(t *testing.T) { - logger := zap.NewNop().Sugar() - pipeline := &testutil.Pipeline{} - pipeline.On("Stop").Return(nil) - database := &testutil.Database{} - failure := fmt.Errorf("failed to close database") - database.On("Close").Return(failure) - - agent := LogAgent{ - SugaredLogger: logger, - pipeline: pipeline, - database: database, - } - err := agent.Stop() - require.Error(t, err, failure) - pipeline.AssertCalled(t, "Stop") - database.AssertCalled(t, "Close") } diff --git a/agent/builder.go b/agent/builder.go index a48ef100..e596b34f 100644 --- a/agent/builder.go +++ b/agent/builder.go @@ -15,13 +15,11 @@ package agent import ( - "fmt" "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "github.com/open-telemetry/opentelemetry-log-collection/database" "github.com/open-telemetry/opentelemetry-log-collection/errors" "github.com/open-telemetry/opentelemetry-log-collection/operator" "github.com/open-telemetry/opentelemetry-log-collection/plugin" @@ -33,8 +31,6 @@ type LogAgentBuilder struct { config *Config logger *zap.SugaredLogger pluginDir string - namespace string - databaseFile string defaultOutput operator.Operator } @@ -63,13 +59,6 @@ func (b *LogAgentBuilder) WithConfig(cfg *Config) *LogAgentBuilder { return b } -// WithDatabaseFile adds the specified database file when building a log agent -func (b *LogAgentBuilder) WithDatabaseFile(databaseFile, namespace string) *LogAgentBuilder { - b.databaseFile = databaseFile - b.namespace = namespace - return b -} - // WithDefaultOutput adds a default output when building a log agent func (b *LogAgentBuilder) WithDefaultOutput(defaultOutput operator.Operator) *LogAgentBuilder { b.defaultOutput = defaultOutput @@ -78,15 +67,6 @@ func (b *LogAgentBuilder) WithDefaultOutput(defaultOutput operator.Operator) *Lo // Build will build a new log agent using the values defined on the builder func (b *LogAgentBuilder) Build() (*LogAgent, error) { - if b.databaseFile != "" && b.namespace == "" { - return nil, fmt.Errorf("use of database requires namespace") - } - - db, err := database.OpenDatabase(b.databaseFile) - if err != nil { - return nil, errors.Wrap(err, "open database") - } - if b.pluginDir != "" { if errs := plugin.RegisterPlugins(b.pluginDir, operator.DefaultRegistry); len(errs) != 0 { b.logger.Errorw("Got errors parsing plugins", "errors", errs) @@ -100,10 +80,11 @@ func (b *LogAgentBuilder) Build() (*LogAgent, error) { return nil, errors.NewError("agent cannot be built without WithConfig or WithConfigFiles", "") } if len(b.configFiles) > 0 { - b.config, err = NewConfigFromGlobs(b.configFiles) + cfgs, err := NewConfigFromGlobs(b.configFiles) if err != nil { return nil, errors.Wrap(err, "read configs from globs") } + b.config = cfgs } sampledLogger := b.logger.Desugar().WithOptions( @@ -112,10 +93,7 @@ func (b *LogAgentBuilder) Build() (*LogAgent, error) { }), ).Sugar() - buildContext := operator.NewBuildContext(db, sampledLogger) - if b.namespace != "" { - buildContext = buildContext.WithSubNamespace(b.namespace) - } + buildContext := operator.NewBuildContext(sampledLogger) pipeline, err := b.config.Pipeline.BuildPipeline(buildContext, b.defaultOutput) if err != nil { @@ -124,7 +102,6 @@ func (b *LogAgentBuilder) Build() (*LogAgent, error) { return &LogAgent{ pipeline: pipeline, - database: db, SugaredLogger: b.logger, }, nil } diff --git a/agent/builder_test.go b/agent/builder_test.go index 6381d58d..685c96e6 100644 --- a/agent/builder_test.go +++ b/agent/builder_test.go @@ -15,8 +15,6 @@ package agent import ( - "io/ioutil" - "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -29,60 +27,17 @@ func TestBuildAgentSuccess(t *testing.T) { mockCfg := Config{} mockLogger := zap.NewNop().Sugar() mockPluginDir := "/some/path/plugins" - mockDatabaseFile := filepath.Join(testutil.NewTempDir(t), "test.db") - mockNamespace := "mynamespace" mockOutput := testutil.NewFakeOutput(t) agent, err := NewBuilder(mockLogger). WithConfig(&mockCfg). WithPluginDir(mockPluginDir). WithDefaultOutput(mockOutput). - WithDatabaseFile(mockDatabaseFile, mockNamespace). Build() require.NoError(t, err) require.Equal(t, mockLogger, agent.SugaredLogger) } -func TestBuildAgentFailureOnDatabase(t *testing.T) { - tempDir := testutil.NewTempDir(t) - invalidDatabaseFile := filepath.Join(tempDir, "test.db") - err := ioutil.WriteFile(invalidDatabaseFile, []byte("invalid"), 0755) - require.NoError(t, err) - - mockCfg := Config{} - mockLogger := zap.NewNop().Sugar() - mockPluginDir := "/some/path/plugins" - mockDatabaseFile := invalidDatabaseFile - mockNamespace := "mynamespace" - mockOutput := testutil.NewFakeOutput(t) - - agent, err := NewBuilder(mockLogger). - WithConfig(&mockCfg). - WithPluginDir(mockPluginDir). - WithDatabaseFile(mockDatabaseFile, mockNamespace). - WithDefaultOutput(mockOutput). - Build() - require.Error(t, err) - require.Nil(t, agent) -} - -func TestBuildAgentFailureOnNamespace(t *testing.T) { - mockCfg := Config{} - mockLogger := zap.NewNop().Sugar() - mockPluginDir := "/some/path/plugins" - mockDatabaseFile := filepath.Join(testutil.NewTempDir(t), "test.db") - mockNamespace := "" - mockOutput := testutil.NewFakeOutput(t) - - _, err := NewBuilder(mockLogger). - WithConfig(&mockCfg). - WithPluginDir(mockPluginDir). - WithDatabaseFile(mockDatabaseFile, mockNamespace). - WithDefaultOutput(mockOutput). - Build() - require.Error(t, err) -} - func TestBuildAgentFailureOnPluginRegistry(t *testing.T) { mockCfg := Config{} mockLogger := zap.NewNop().Sugar() diff --git a/database/database.go b/database/database.go deleted file mode 100644 index 26ed2dcc..00000000 --- a/database/database.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -//go:generate mockery --name=^(Database)$ --output=../testutil --outpkg=testutil --case=snake - -import ( - "fmt" - "os" - "path/filepath" - "time" - - "go.etcd.io/bbolt" -) - -// Database is a database used to save offsets -type Database interface { - Close() error - Sync() error - Update(func(*bbolt.Tx) error) error - View(func(*bbolt.Tx) error) error -} - -// StubDatabase is an implementation of Database that -// succeeds on all calls without persisting anything to disk. -// This is used when --database is unspecified. -type StubDatabase struct{} - -// Close will be ignored by the stub database -func (d *StubDatabase) Close() error { return nil } - -// Sync will be ignored by the stub database -func (d *StubDatabase) Sync() error { return nil } - -// Update will be ignored by the stub database -func (d *StubDatabase) Update(func(tx *bbolt.Tx) error) error { return nil } - -// View will be ignored by the stub database -func (d *StubDatabase) View(func(tx *bbolt.Tx) error) error { return nil } - -// NewStubDatabase creates a new StubDatabase -func NewStubDatabase() *StubDatabase { - return &StubDatabase{} -} - -// OpenDatabase will open and create a database -func OpenDatabase(file string) (Database, error) { - if file == "" { - return NewStubDatabase(), nil - } - - if _, err := os.Stat(filepath.Dir(file)); err != nil { - if os.IsNotExist(err) { - err := os.MkdirAll(filepath.Dir(file), 0755) - if err != nil { - return nil, fmt.Errorf("creating database directory: %s", err) - } - } else { - return nil, err - } - } - - options := &bbolt.Options{Timeout: 1 * time.Second} - return bbolt.Open(file, 0666, options) -} diff --git a/database/database_test.go b/database/database_test.go deleted file mode 100644 index fa089c8f..00000000 --- a/database/database_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -import ( - "io/ioutil" - "os" - "path/filepath" - "runtime" - "testing" - - "github.com/stretchr/testify/require" -) - -// NewTempDir will return a new temp directory for testing -func NewTempDir(t testing.TB) string { - tempDir, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf(err.Error()) - t.FailNow() - } - - t.Cleanup(func() { - os.RemoveAll(tempDir) - }) - - return tempDir -} - -func TestOpenDatabase(t *testing.T) { - t.Run("Simple", func(t *testing.T) { - tempDir := NewTempDir(t) - db, err := OpenDatabase(filepath.Join(tempDir, "test.db")) - require.NoError(t, err) - require.NotNil(t, db) - }) - - t.Run("NoFile", func(t *testing.T) { - db, err := OpenDatabase("") - require.NoError(t, err) - require.NotNil(t, db) - require.IsType(t, &StubDatabase{}, db) - }) - - t.Run("NonexistantPathIsCreated", func(t *testing.T) { - tempDir := NewTempDir(t) - db, err := OpenDatabase(filepath.Join(tempDir, "nonexistdir", "test.db")) - require.NoError(t, err) - require.NotNil(t, db) - }) - - t.Run("BadPermissions", func(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows does not have the same kind of file permissions") - } - tempDir := NewTempDir(t) - err := os.MkdirAll(filepath.Join(tempDir, "badperms"), 0666) - require.NoError(t, err) - db, err := OpenDatabase(filepath.Join(tempDir, "badperms", "nonexistdir", "test.db")) - require.Error(t, err) - require.Nil(t, db) - }) - - t.Run("ExecuteOnlyPermissions", func(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows does not have the same kind of file permissions") - } - tempDir := NewTempDir(t) - err := os.MkdirAll(filepath.Join(tempDir, "badperms"), 0111) - require.NoError(t, err) - db, err := OpenDatabase(filepath.Join(tempDir, "badperms", "nonexistdir", "test.db")) - require.Error(t, err) - require.Nil(t, db) - }) -} - -func TestStubDatabase(t *testing.T) { - stubDatabase := NewStubDatabase() - err := stubDatabase.Close() - require.NoError(t, err) - - err = stubDatabase.Sync() - require.NoError(t, err) - - err = stubDatabase.Update(nil) - require.NoError(t, err) - - err = stubDatabase.View(nil) - require.NoError(t, err) -} diff --git a/go.mod b/go.mod index 27d1c296..aef82391 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/antonmedv/expr v1.8.9 + github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f github.com/json-iterator/go v1.1.10 github.com/mitchellh/mapstructure v1.4.1 github.com/observiq/ctimefmt v1.0.0 diff --git a/go.sum b/go.sum index 152af75b..4b483190 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -16,7 +17,9 @@ cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0 h1:kpgPA77kSSbjSs+fWHkPTxQ6J5Z2Qkruo5jfXEkHxNQ= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -122,6 +125,7 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/bsm/sarama-cluster v2.1.13+incompatible/go.mod h1:r7ao+4tTNXvWm+VRpRJchr2kQhqxgmAp2iEX5W96gMM= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -214,9 +218,12 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.7.3/go.mod h1:V1d2J5pfxYH6EjBAgSK7YNXcXlTWxUHdE1sVDXkjnig= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f h1:Sk0u0gIncQaQD23zAoAZs2DNi2u2l5UTLi4CmCBL5v8= +github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -235,7 +242,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -326,6 +332,7 @@ github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= @@ -359,11 +366,12 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg= +github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f/go.mod h1:ijRvpgDJDI262hYq/IQVYgf8hd8IHUs93Ol0kvMBAx4= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -372,6 +380,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20170918230701-e5d664eb928e/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -391,7 +400,6 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= @@ -399,14 +407,15 @@ github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -437,6 +446,7 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= @@ -454,6 +464,7 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -498,6 +509,7 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -509,12 +521,12 @@ github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKEN github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hetznercloud/hcloud-go v1.23.1/go.mod h1:xng8lbDUg+xM1dgc0yGHX5EeqbwIq7UYlMWMTx3SQVg= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.4/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= @@ -592,6 +604,7 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -604,9 +617,11 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.0.10-0.20170816031813-ad5389df28cd/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.2/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -633,6 +648,7 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -664,7 +680,6 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -683,7 +698,6 @@ github.com/olivere/elastic v6.2.35+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGe github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -691,7 +705,6 @@ github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= @@ -720,6 +733,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= @@ -833,21 +847,26 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/jwalterweatherman v0.0.0-20170901151539-12bd96e66386/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20170901120850-7aff26db30c1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= @@ -857,7 +876,6 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -867,7 +885,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -931,7 +948,6 @@ go.opentelemetry.io/collector v0.22.0/go.mod h1:sBkAGYUQSh1f+owCK0aPV2uLcUB6rPHE go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -964,7 +980,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -983,7 +998,6 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 h1:FR+oGxGfbQu1d+jglI3rCkjAjUnhRSZcUxr+DqlDLNo= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= @@ -999,7 +1013,6 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -1010,7 +1023,6 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -1058,7 +1070,6 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1068,10 +1079,10 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -1079,6 +1090,7 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210210192628-66670185b0cd h1:2arJsLyTCJGek+eeptQ3z49Rqndm0f+zvvpwNIXWNIA= golang.org/x/oauth2 v0.0.0-20210210192628-66670185b0cd/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20170517211232-f52d1811a629/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1173,15 +1185,14 @@ golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fq golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= @@ -1252,7 +1263,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858 h1:xLt+iB5ksWcZVxqc+g9K41ZHy+6MKWfXCDsjSThnsPA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1276,6 +1286,7 @@ gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6d gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/api v0.0.0-20170921000349-586095a6e407/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1301,12 +1312,12 @@ google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1349,6 +1360,7 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210302174412-5ede27ff9881/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1382,7 +1394,6 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= @@ -1390,13 +1401,11 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= @@ -1422,11 +1431,9 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/operator/build_context.go b/operator/build_context.go index 67801033..78cfd6db 100644 --- a/operator/build_context.go +++ b/operator/build_context.go @@ -20,13 +20,11 @@ import ( "go.uber.org/zap" - "github.com/open-telemetry/opentelemetry-log-collection/database" "github.com/open-telemetry/opentelemetry-log-collection/logger" ) // BuildContext supplies contextual resources when building an operator. type BuildContext struct { - Database database.Database Parameters map[string]interface{} Logger *logger.Logger Namespace string @@ -69,7 +67,6 @@ func (bc BuildContext) WithIncrementedDepth() BuildContext { // Copy creates a copy of the build context func (bc BuildContext) Copy() BuildContext { return BuildContext{ - Database: bc.Database, Parameters: bc.Parameters, Logger: bc.Logger, Namespace: bc.Namespace, @@ -80,9 +77,8 @@ func (bc BuildContext) Copy() BuildContext { // NewBuildContext creates a new build context with the given database, logger, and the // default namespace -func NewBuildContext(db database.Database, lg *zap.SugaredLogger) BuildContext { +func NewBuildContext(lg *zap.SugaredLogger) BuildContext { return BuildContext{ - Database: db, Parameters: nil, Logger: logger.New(lg), Namespace: "$", diff --git a/operator/builtin/input/file/config.go b/operator/builtin/input/file/config.go index 2b243e12..a3ec7dd7 100644 --- a/operator/builtin/input/file/config.go +++ b/operator/builtin/input/file/config.go @@ -155,7 +155,6 @@ func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator, Exclude: c.Exclude, SplitFunc: splitFunc, PollInterval: c.PollInterval.Raw(), - persist: helper.NewScopedDBPersister(context.Database, c.ID()), FilePathField: filePathField, FileNameField: fileNameField, startAtBeginning: startAtBeginning, diff --git a/operator/builtin/input/file/file.go b/operator/builtin/input/file/file.go index 1b326d6b..a46cbc5b 100644 --- a/operator/builtin/input/file/file.go +++ b/operator/builtin/input/file/file.go @@ -29,6 +29,7 @@ import ( "golang.org/x/text/encoding" "github.com/open-telemetry/opentelemetry-log-collection/entry" + "github.com/open-telemetry/opentelemetry-log-collection/operator" "github.com/open-telemetry/opentelemetry-log-collection/operator/helper" ) @@ -46,7 +47,7 @@ type InputOperator struct { MaxConcurrentFiles int SeenPaths map[string]struct{} - persist helper.Persister + persister operator.Persister knownFiles []*Reader queuedMatches []string @@ -63,11 +64,12 @@ type InputOperator struct { } // Start will start the file monitoring process -func (f *InputOperator) Start() error { +func (f *InputOperator) Start(persister operator.Persister) error { ctx, cancel := context.WithCancel(context.Background()) f.cancel = cancel f.firstCheck = true + f.persister = persister // Load offsets from disk if err := f.loadLastPollFiles(); err != nil { return fmt.Errorf("read known files from database: %s", err) @@ -335,20 +337,18 @@ func (f *InputOperator) syncLastPollFiles() { } } - f.persist.Set(knownFilesKey, buf.Bytes()) - if err := f.persist.Sync(); err != nil { + if err := f.persister.Set(knownFilesKey, buf.Bytes()); err != nil { f.Errorw("Failed to sync to database", zap.Error(err)) } } // syncLastPollFiles loads the most recent set of files to the database func (f *InputOperator) loadLastPollFiles() error { - err := f.persist.Load() + encoded, err := f.persister.Get(knownFilesKey) if err != nil { return err } - encoded := f.persist.Get(knownFilesKey) if encoded == nil { f.knownFiles = make([]*Reader, 0, 10) return nil diff --git a/operator/builtin/input/file/file_test.go b/operator/builtin/input/file/file_test.go index 3205967a..bee92cdb 100644 --- a/operator/builtin/input/file/file_test.go +++ b/operator/builtin/input/file/file_test.go @@ -272,7 +272,7 @@ See this issue for details: https://github.com/census-instrumentation/opencensus operator, _, tempDir := newTestFileOperator(t, nil, nil) _ = openTemp(t, tempDir) - err := operator.Start() + err := operator.Start(testutil.NewMockPersister()) require.NoError(t, err) operator.Stop() } @@ -290,7 +290,7 @@ func TestAddFileFields(t *testing.T) { temp := openTemp(t, tempDir) writeString(t, temp, "testlog\n") - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() e := waitForOne(t, logReceived) @@ -308,7 +308,7 @@ func TestReadExistingLogs(t *testing.T) { temp := openTemp(t, tempDir) writeString(t, temp, "testlog1\ntestlog2\n") - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() waitForMessage(t, logReceived, "testlog1") @@ -320,6 +320,7 @@ func TestReadExistingLogs(t *testing.T) { func TestReadNewLogs(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + operator.persister = testutil.NewMockPersister() // Poll once so we know this isn't a new file operator.poll(context.Background()) @@ -342,6 +343,7 @@ func TestReadNewLogs(t *testing.T) { func TestReadExistingAndNewLogs(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + operator.persister = testutil.NewMockPersister() // Start with a file with an entry in it, and expect that entry // to come through when we poll for the first time @@ -364,6 +366,7 @@ func TestStartAtEnd(t *testing.T) { operator, logReceived, tempDir := newTestFileOperator(t, func(cfg *InputConfig) { cfg.StartAt = "end" }, nil) + operator.persister = testutil.NewMockPersister() temp := openTemp(t, tempDir) writeString(t, temp, "testlog1\n") @@ -384,6 +387,7 @@ func TestStartAtEnd(t *testing.T) { func TestStartAtEndNewFile(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + operator.persister = testutil.NewMockPersister() operator.startAtBeginning = false operator.poll(context.Background()) @@ -406,7 +410,7 @@ func TestNoNewline(t *testing.T) { temp := openTemp(t, tempDir) writeString(t, temp, "testlog1\ntestlog2") - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() waitForMessage(t, logReceived, "testlog1") @@ -421,7 +425,7 @@ func TestSkipEmpty(t *testing.T) { temp := openTemp(t, tempDir) writeString(t, temp, "testlog1\n\ntestlog2\n") - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() waitForMessage(t, logReceived, "testlog1") @@ -433,6 +437,7 @@ func TestSkipEmpty(t *testing.T) { func TestSplitWrite(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + operator.persister = testutil.NewMockPersister() temp := openTemp(t, tempDir) writeString(t, temp, "testlog1") @@ -449,7 +454,7 @@ func TestDecodeBufferIsResized(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() temp := openTemp(t, tempDir) @@ -469,7 +474,7 @@ func TestMultiFileSimple(t *testing.T) { writeString(t, temp1, "testlog1\n") writeString(t, temp2, "testlog2\n") - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() waitForMessages(t, logReceived, []string{"testlog1", "testlog2"}) @@ -504,7 +509,7 @@ func TestMultiFileParallel_PreloadedFiles(t *testing.T) { }(temp, i) } - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() waitForMessages(t, logReceived, expected) @@ -528,7 +533,7 @@ func TestMultiFileParallel_LiveFiles(t *testing.T) { } } - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() temps := make([]*os.File, 0, numFiles) @@ -571,7 +576,7 @@ func TestMultiFileRotate(t *testing.T) { } } - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() temps := make([]*os.File, 0, numFiles) @@ -628,7 +633,7 @@ func TestMultiFileRotateSlow(t *testing.T) { } } - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() var wg sync.WaitGroup @@ -674,7 +679,7 @@ func TestMultiCopyTruncateSlow(t *testing.T) { } } - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() var wg sync.WaitGroup @@ -771,7 +776,7 @@ func (rt rotationTest) run(tc rotationTest, copyTruncate, sequential bool) func( expected = append(expected, fmt.Sprintf("%s %3d", baseStr, i)) } - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() for _, message := range expected { @@ -854,6 +859,7 @@ func TestMoveFile(t *testing.T) { } t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + operator.persister = testutil.NewMockPersister() temp1 := openTemp(t, tempDir) writeString(t, temp1, "testlog1\n") @@ -878,6 +884,7 @@ func TestMoveFile(t *testing.T) { func TestTruncateThenWrite(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + operator.persister = testutil.NewMockPersister() temp1 := openTemp(t, tempDir) writeString(t, temp1, "testlog1\ntestlog2\n") @@ -904,6 +911,7 @@ func TestTruncateThenWrite(t *testing.T) { func TestCopyTruncateWriteBoth(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + operator.persister = testutil.NewMockPersister() temp1 := openTemp(t, tempDir) writeString(t, temp1, "testlog1\ntestlog2\n") @@ -938,19 +946,20 @@ func TestCopyTruncateWriteBoth(t *testing.T) { func TestOffsetsAfterRestart(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + persister := testutil.NewMockPersister() temp1 := openTemp(t, tempDir) writeString(t, temp1, "testlog1\n") // Start the operator and expect a message - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(persister)) defer operator.Stop() waitForMessage(t, logReceived, "testlog1") // Restart the operator. Stop and build a new // one to guarantee freshness require.NoError(t, operator.Stop()) - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(persister)) // Write a new log and expect only that log writeString(t, temp1, "testlog2\n") @@ -960,6 +969,7 @@ func TestOffsetsAfterRestart(t *testing.T) { func TestOffsetsAfterRestart_BigFiles(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + persister := testutil.NewMockPersister() log1 := stringWithLength(2000) log2 := stringWithLength(2000) @@ -968,13 +978,13 @@ func TestOffsetsAfterRestart_BigFiles(t *testing.T) { writeString(t, temp1, log1+"\n") // Start the operator - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(persister)) defer operator.Stop() waitForMessage(t, logReceived, log1) // Restart the operator require.NoError(t, operator.Stop()) - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(persister)) writeString(t, temp1, log2+"\n") waitForMessage(t, logReceived, log2) @@ -983,6 +993,7 @@ func TestOffsetsAfterRestart_BigFiles(t *testing.T) { func TestOffsetsAfterRestart_BigFilesWrittenWhileOff(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + persister := testutil.NewMockPersister() log1 := stringWithLength(2000) log2 := stringWithLength(2000) @@ -991,7 +1002,7 @@ func TestOffsetsAfterRestart_BigFilesWrittenWhileOff(t *testing.T) { writeString(t, temp, log1+"\n") // Start the operator and expect the first message - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(persister)) defer operator.Stop() waitForMessage(t, logReceived, log1) @@ -1000,13 +1011,14 @@ func TestOffsetsAfterRestart_BigFilesWrittenWhileOff(t *testing.T) { writeString(t, temp, log2+"\n") // Start the operator and expect the message - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(persister)) waitForMessage(t, logReceived, log2) } func TestFileMovedWhileOff_BigFiles(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) + persister := testutil.NewMockPersister() log1 := stringWithLength(1000) log2 := stringWithLength(1000) @@ -1016,7 +1028,7 @@ func TestFileMovedWhileOff_BigFiles(t *testing.T) { require.NoError(t, temp.Close()) // Start the operator - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(persister)) defer operator.Stop() waitForMessage(t, logReceived, log1) @@ -1031,7 +1043,7 @@ func TestFileMovedWhileOff_BigFiles(t *testing.T) { writeString(t, temp, log2+"\n") // Expect the message written to the new log to come through - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(persister)) waitForMessage(t, logReceived, log2) } @@ -1046,7 +1058,7 @@ func TestManyLogsDelivered(t *testing.T) { } // Start the operator - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() // Write lots of logs @@ -1083,6 +1095,7 @@ func TestFileBatching(t *testing.T) { out.Received = make(chan *entry.Entry, expectedLinesPerBatch) }, ) + operator.persister = testutil.NewMockPersister() temps := make([]*os.File, 0, files) for i := 0; i < files; i++ { @@ -1283,7 +1296,7 @@ func TestEncodings(t *testing.T) { _, err := temp.Write(tc.contents) require.NoError(t, err) - require.NoError(t, operator.Start()) + require.NoError(t, operator.Start(testutil.NewMockPersister())) defer operator.Stop() for _, expected := range tc.expected { @@ -1337,7 +1350,7 @@ func BenchmarkFileInput(b *testing.B) { err = op.SetOutputs([]operator.Operator{fakeOutput}) require.NoError(b, err) - err = op.Start() + err = op.Start(testutil.NewMockPersister()) defer op.Stop() require.NoError(b, err) diff --git a/operator/builtin/input/generate/generate.go b/operator/builtin/input/generate/generate.go index b1407934..8dc7a2e9 100644 --- a/operator/builtin/input/generate/generate.go +++ b/operator/builtin/input/generate/generate.go @@ -73,7 +73,7 @@ type GenerateInput struct { } // Start will start generating log entries. -func (g *GenerateInput) Start() error { +func (g *GenerateInput) Start(_ operator.Persister) error { ctx, cancel := context.WithCancel(context.Background()) g.cancel = cancel diff --git a/operator/builtin/input/generate/generate_test.go b/operator/builtin/input/generate/generate_test.go index cd0727cb..17b7b252 100644 --- a/operator/builtin/input/generate/generate_test.go +++ b/operator/builtin/input/generate/generate_test.go @@ -40,7 +40,7 @@ func TestInputGenerate(t *testing.T) { err = op.SetOutputs([]operator.Operator{fake}) require.NoError(t, err) - require.NoError(t, op.Start()) + require.NoError(t, op.Start(testutil.NewMockPersister())) defer op.Stop() for i := 0; i < 5; i++ { diff --git a/operator/builtin/input/journald/journald.go b/operator/builtin/input/journald/journald.go index 03119668..e9a5475c 100644 --- a/operator/builtin/input/journald/journald.go +++ b/operator/builtin/input/journald/journald.go @@ -92,7 +92,6 @@ func (c JournaldInputConfig) Build(buildContext operator.BuildContext) ([]operat journaldInput := &JournaldInput{ InputOperator: inputOperator, - persist: helper.NewScopedDBPersister(buildContext.Database, c.ID()), newCmd: func(ctx context.Context, cursor []byte) cmd { if cursor != nil { args = append(args, "--after-cursor", string(cursor)) @@ -110,10 +109,10 @@ type JournaldInput struct { newCmd func(ctx context.Context, cursor []byte) cmd - persist helper.Persister - json jsoniter.API - cancel context.CancelFunc - wg sync.WaitGroup + persister operator.Persister + json jsoniter.API + cancel context.CancelFunc + wg sync.WaitGroup } type cmd interface { @@ -124,17 +123,17 @@ type cmd interface { var lastReadCursorKey = "lastReadCursor" // Start will start generating log entries. -func (operator *JournaldInput) Start() error { +func (operator *JournaldInput) Start(persister operator.Persister) error { ctx, cancel := context.WithCancel(context.Background()) operator.cancel = cancel - err := operator.persist.Load() + // Start from a cursor if there is a saved offset + cursor, err := persister.Get(lastReadCursorKey) if err != nil { - return err + return fmt.Errorf("failed to get journalctl state: %s", err) } - // Start from a cursor if there is a saved offset - cursor := operator.persist.Get(lastReadCursorKey) + operator.persister = persister // Start journalctl cmd := operator.newCmd(ctx, cursor) @@ -147,25 +146,10 @@ func (operator *JournaldInput) Start() error { return fmt.Errorf("start journalctl: %s", err) } - // Start a goroutine to periodically flush the offsets - operator.wg.Add(1) - go func() { - defer operator.wg.Done() - for { - select { - case <-ctx.Done(): - return - case <-time.After(time.Second): - operator.syncOffsets() - } - } - }() - // Start the reader goroutine operator.wg.Add(1) go func() { defer operator.wg.Done() - defer operator.syncOffsets() stdoutBuf := bufio.NewReader(stdout) @@ -183,7 +167,9 @@ func (operator *JournaldInput) Start() error { operator.Warnw("Failed to parse journal entry", zap.Error(err)) continue } - operator.persist.Set(lastReadCursorKey, []byte(cursor)) + if err := operator.persister.Set(lastReadCursorKey, []byte(cursor)); err != nil { + operator.Warnw("Failed to set offset", zap.Error(err)) + } operator.Write(ctx, entry) } }() @@ -234,13 +220,6 @@ func (operator *JournaldInput) parseJournalEntry(line []byte) (*entry.Entry, str return entry, cursorString, nil } -func (operator *JournaldInput) syncOffsets() { - err := operator.persist.Sync() - if err != nil { - operator.Errorw("Failed to sync offsets", zap.Error(err)) - } -} - // Stop will stop generating logs. func (operator *JournaldInput) Stop() error { operator.cancel() diff --git a/operator/builtin/input/journald/journald_test.go b/operator/builtin/input/journald/journald_test.go index 87f22e3c..3eb03272 100644 --- a/operator/builtin/input/journald/journald_test.go +++ b/operator/builtin/input/journald/journald_test.go @@ -67,7 +67,7 @@ func TestInputJournald(t *testing.T) { return &fakeJournaldCmd{} } - err = op.Start() + err = op.Start(testutil.NewMockPersister()) require.NoError(t, err) defer op.Stop() diff --git a/operator/builtin/input/k8sevent/k8s_event.go b/operator/builtin/input/k8sevent/k8s_event.go index 34df9d3a..1f29e3cc 100644 --- a/operator/builtin/input/k8sevent/k8s_event.go +++ b/operator/builtin/input/k8sevent/k8s_event.go @@ -91,7 +91,7 @@ type K8sEvents struct { } // Start implements the operator.Operator interface -func (k *K8sEvents) Start() error { +func (k *K8sEvents) Start(_ operator.Persister) error { ctx, cancel := context.WithCancel(context.Background()) k.cancel = cancel diff --git a/operator/builtin/input/stanza/stanza.go b/operator/builtin/input/stanza/stanza.go index c78a3b06..63bb4694 100644 --- a/operator/builtin/input/stanza/stanza.go +++ b/operator/builtin/input/stanza/stanza.go @@ -69,7 +69,7 @@ type Input struct { } // Start will start reading incoming stanza logs. -func (i *Input) Start() error { +func (i *Input) Start(_ operator.Persister) error { ctx, cancel := context.WithCancel(context.Background()) i.cancel = cancel i.startReading(ctx) diff --git a/operator/builtin/input/stanza/stanza_test.go b/operator/builtin/input/stanza/stanza_test.go index 70ac9326..51b890be 100644 --- a/operator/builtin/input/stanza/stanza_test.go +++ b/operator/builtin/input/stanza/stanza_test.go @@ -36,7 +36,7 @@ func TestStanzaOperator(t *testing.T) { fake := testutil.NewFakeOutput(t) op.SetOutputs([]operator.Operator{fake}) - require.NoError(t, op.Start()) + require.NoError(t, op.Start(testutil.NewMockPersister())) defer op.Stop() bc.Logger.Errorw("test failure", "key", "value") diff --git a/operator/builtin/input/stdin/stdin.go b/operator/builtin/input/stdin/stdin.go index 0dedbb66..dbbb504a 100644 --- a/operator/builtin/input/stdin/stdin.go +++ b/operator/builtin/input/stdin/stdin.go @@ -67,7 +67,7 @@ type StdinInput struct { } // Start will start generating log entries. -func (g *StdinInput) Start() error { +func (g *StdinInput) Start(_ operator.Persister) error { ctx, cancel := context.WithCancel(context.Background()) g.cancel = cancel diff --git a/operator/builtin/input/stdin/stdin_test.go b/operator/builtin/input/stdin/stdin_test.go index 7b05c91c..c1a529b3 100644 --- a/operator/builtin/input/stdin/stdin_test.go +++ b/operator/builtin/input/stdin/stdin_test.go @@ -40,7 +40,7 @@ func TestStdin(t *testing.T) { stdin := op[0].(*StdinInput) stdin.stdin = r - require.NoError(t, stdin.Start()) + require.NoError(t, stdin.Start(testutil.NewMockPersister())) defer stdin.Stop() w.WriteString("test") diff --git a/operator/builtin/input/syslog/syslog_test.go b/operator/builtin/input/syslog/syslog_test.go index 8b9fb721..6e2d5a60 100644 --- a/operator/builtin/input/syslog/syslog_test.go +++ b/operator/builtin/input/syslog/syslog_test.go @@ -58,7 +58,7 @@ func SyslogInputTest(t *testing.T, cfg *SyslogInputConfig, tc syslog.Case) { p, err := pipeline.NewDirectedPipeline(ops) require.NoError(t, err) - err = p.Start() + err = p.Start(testutil.NewMockPersister()) require.NoError(t, err) var conn net.Conn diff --git a/operator/builtin/input/tcp/tcp.go b/operator/builtin/input/tcp/tcp.go index 7bcb4fd9..8b1fa389 100644 --- a/operator/builtin/input/tcp/tcp.go +++ b/operator/builtin/input/tcp/tcp.go @@ -115,7 +115,7 @@ type TCPInput struct { } // Start will start listening for log entries over tcp. -func (t *TCPInput) Start() error { +func (t *TCPInput) Start(_ operator.Persister) error { if err := t.configureListener(); err != nil { return fmt.Errorf("failed to listen on interface: %w", err) } diff --git a/operator/builtin/input/tcp/tcp_test.go b/operator/builtin/input/tcp/tcp_test.go index 0da1b6de..4616e6c0 100644 --- a/operator/builtin/input/tcp/tcp_test.go +++ b/operator/builtin/input/tcp/tcp_test.go @@ -101,7 +101,7 @@ func tcpInputTest(input []byte, expected []string) func(t *testing.T) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = tcpInput.Start() + err = tcpInput.Start(testutil.NewMockPersister()) require.NoError(t, err) defer tcpInput.Stop() @@ -170,7 +170,7 @@ func tlsTCPInputTest(input []byte, expected []string) func(t *testing.T) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = tcpInput.Start() + err = tcpInput.Start(testutil.NewMockPersister()) require.NoError(t, err) defer tcpInput.Stop() @@ -299,7 +299,7 @@ func BenchmarkTcpInput(b *testing.B) { tcpInput := op.(*TCPInput) tcpInput.InputOperator.OutputOperators = []operator.Operator{fakeOutput} - err = tcpInput.Start() + err = tcpInput.Start(testutil.NewMockPersister()) require.NoError(b, err) done := make(chan struct{}) diff --git a/operator/builtin/input/udp/udp.go b/operator/builtin/input/udp/udp.go index 8359e4df..430c60e5 100644 --- a/operator/builtin/input/udp/udp.go +++ b/operator/builtin/input/udp/udp.go @@ -80,7 +80,7 @@ type UDPInput struct { } // Start will start listening for messages on a socket. -func (u *UDPInput) Start() error { +func (u *UDPInput) Start(persister operator.Persister) error { ctx, cancel := context.WithCancel(context.Background()) u.cancel = cancel diff --git a/operator/builtin/input/udp/udp_test.go b/operator/builtin/input/udp/udp_test.go index 92eafbd3..4a2e802a 100644 --- a/operator/builtin/input/udp/udp_test.go +++ b/operator/builtin/input/udp/udp_test.go @@ -47,7 +47,7 @@ func udpInputTest(input []byte, expected []string) func(t *testing.T) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = udpInput.Start() + err = udpInput.Start(testutil.NewMockPersister()) require.NoError(t, err) defer udpInput.Stop() @@ -95,7 +95,7 @@ func BenchmarkUdpInput(b *testing.B) { udpInput := op.(*UDPInput) udpInput.InputOperator.OutputOperators = []operator.Operator{fakeOutput} - err = udpInput.Start() + err = udpInput.Start(testutil.NewMockPersister()) require.NoError(b, err) done := make(chan struct{}) diff --git a/operator/builtin/input/windows/operator.go b/operator/builtin/input/windows/operator.go index cc6bc71c..117d9773 100644 --- a/operator/builtin/input/windows/operator.go +++ b/operator/builtin/input/windows/operator.go @@ -58,8 +58,6 @@ func (c *EventLogConfig) Build(context operator.BuildContext) ([]operator.Operat return nil, fmt.Errorf("the `start_at` field must be set to `beginning` or `end`") } - offsets := helper.NewScopedDBPersister(context.Database, c.ID()) - eventLogInput := &EventLogInput{ InputOperator: inputOperator, buffer: NewBuffer(), @@ -67,7 +65,6 @@ func (c *EventLogConfig) Build(context operator.BuildContext) ([]operator.Operat maxReads: c.MaxReads, startAt: c.StartAt, pollInterval: c.PollInterval, - offsets: offsets, } return []operator.Operator{eventLogInput}, nil } @@ -94,16 +91,18 @@ type EventLogInput struct { maxReads int startAt string pollInterval helper.Duration - offsets helper.Persister + persister operator.Persister cancel context.CancelFunc wg sync.WaitGroup } // Start will start reading events from a subscription. -func (e *EventLogInput) Start() error { +func (e *EventLogInput) Start(persister operator.Persister) error { ctx, cancel := context.WithCancel(context.Background()) e.cancel = cancel + e.persister = persister + e.bookmark = NewBookmark() offsetXML, err := e.getBookmarkOffset() if err != nil { @@ -234,12 +233,8 @@ func (e *EventLogInput) sendEvent(ctx context.Context, eventXML EventXML) { // getBookmarkXML will get the bookmark xml from the offsets database. func (e *EventLogInput) getBookmarkOffset() (string, error) { - if err := e.offsets.Load(); err != nil { - return "", fmt.Errorf("failed to load offsets database: %s", err) - } - - bytes := e.offsets.Get(e.channel) - return string(bytes), nil + bytes, err := e.persister.Get(e.channel) + return string(bytes), err } // updateBookmark will update the bookmark xml and save it in the offsets database. @@ -255,9 +250,8 @@ func (e *EventLogInput) updateBookmarkOffset(event Event) { return } - e.offsets.Set(e.channel, []byte(bookmarkXML)) - if err := e.offsets.Sync(); err != nil { - e.Errorf("failed to sync offsets database: %s", err) + if err := e.persister.Set(e.channel, []byte(bookmarkXML)); err != nil { + e.Errorf("failed to set offsets: %s", err) return } } diff --git a/operator/builtin/output/file/file.go b/operator/builtin/output/file/file.go index 83746edb..5eca9525 100644 --- a/operator/builtin/output/file/file.go +++ b/operator/builtin/output/file/file.go @@ -86,7 +86,7 @@ type FileOutput struct { } // Start will open the output file. -func (fo *FileOutput) Start() error { +func (fo *FileOutput) Start(_ operator.Persister) error { var err error fo.file, err = os.OpenFile(fo.path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660) if err != nil { diff --git a/operator/builtin/transformer/recombine/recombine.go b/operator/builtin/transformer/recombine/recombine.go index f78630ea..a6d97825 100644 --- a/operator/builtin/transformer/recombine/recombine.go +++ b/operator/builtin/transformer/recombine/recombine.go @@ -124,7 +124,7 @@ type RecombineOperator struct { batch []*entry.Entry } -func (r *RecombineOperator) Start() error { +func (r *RecombineOperator) Start(_ operator.Persister) error { return nil } diff --git a/operator/helper/operator.go b/operator/helper/operator.go index 565eb586..f86b0a02 100644 --- a/operator/helper/operator.go +++ b/operator/helper/operator.go @@ -103,7 +103,7 @@ func (p *BasicOperator) Logger() *zap.SugaredLogger { } // Start will start the operator. -func (p *BasicOperator) Start() error { +func (p *BasicOperator) Start(_ operator.Persister) error { return nil } diff --git a/operator/helper/operator_test.go b/operator/helper/operator_test.go index bf30ad1d..2eefa683 100644 --- a/operator/helper/operator_test.go +++ b/operator/helper/operator_test.go @@ -113,7 +113,7 @@ func TestBasicOperatorStart(t *testing.T) { OperatorID: "test-id", OperatorType: "test-type", } - err := operator.Start() + err := operator.Start(testutil.NewMockPersister()) require.NoError(t, err) } diff --git a/operator/helper/persister.go b/operator/helper/persister.go deleted file mode 100644 index 3c5dc1bd..00000000 --- a/operator/helper/persister.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package helper - -import ( - "sync" - - "go.etcd.io/bbolt" - - "github.com/open-telemetry/opentelemetry-log-collection/database" -) - -// Persister is a helper used to persist data -type Persister interface { - Get(string) []byte - Set(string, []byte) - Sync() error - Load() error -} - -// ScopedBBoltPersister is a persister that uses a database for the backend -type ScopedBBoltPersister struct { - scope []byte - db database.Database - cache map[string][]byte - cacheMux sync.Mutex -} - -// NewScopedDBPersister returns a new ScopedBBoltPersister -func NewScopedDBPersister(db database.Database, scope string) *ScopedBBoltPersister { - return &ScopedBBoltPersister{ - scope: []byte(scope), - db: db, - cache: make(map[string][]byte), - } -} - -// Get retrieves a key from the cache -func (p *ScopedBBoltPersister) Get(key string) []byte { - p.cacheMux.Lock() - defer p.cacheMux.Unlock() - return p.cache[key] -} - -// Set saves a key in the cache -func (p *ScopedBBoltPersister) Set(key string, val []byte) { - p.cacheMux.Lock() - p.cache[key] = val - p.cacheMux.Unlock() -} - -// OffsetsBucket is the scope provided to offset persistence -var OffsetsBucket = []byte(`offsets`) - -// Sync saves the cache to the backend, ensuring values are -// safely written to disk before returning -func (p *ScopedBBoltPersister) Sync() error { - return p.db.Update(func(tx *bbolt.Tx) error { - offsetBucket, err := tx.CreateBucketIfNotExists(OffsetsBucket) - if err != nil { - return err - } - - bucket, err := offsetBucket.CreateBucketIfNotExists(p.scope) - if err != nil { - return err - } - - p.cacheMux.Lock() - for k, v := range p.cache { - err := bucket.Put([]byte(k), v) - if err != nil { - return err - } - } - p.cacheMux.Unlock() - - return nil - }) -} - -// Load populates the cache with the values from the database, -// overwriting anything currently in the cache. -func (p *ScopedBBoltPersister) Load() error { - p.cacheMux.Lock() - defer p.cacheMux.Unlock() - p.cache = make(map[string][]byte) - - return p.db.Update(func(tx *bbolt.Tx) error { - offsetBucket, err := tx.CreateBucketIfNotExists(OffsetsBucket) - if err != nil { - return err - } - - bucket, err := offsetBucket.CreateBucketIfNotExists(p.scope) - if err != nil { - return err - } - - return bucket.ForEach(func(k, v []byte) error { - p.cache[string(k)] = v - return nil - }) - }) -} diff --git a/operator/helper/persister_test.go b/operator/helper/persister_test.go deleted file mode 100644 index 8a3a3fd2..00000000 --- a/operator/helper/persister_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package helper - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/open-telemetry/opentelemetry-log-collection/database" - "github.com/open-telemetry/opentelemetry-log-collection/testutil" -) - -func TestPersisterCache(t *testing.T) { - stubDatabase := database.NewStubDatabase() - persister := NewScopedDBPersister(stubDatabase, "test") - persister.Set("key", []byte("value")) - value := persister.Get("key") - require.Equal(t, []byte("value"), value) -} - -func TestPersisterLoad(t *testing.T) { - tempDir := testutil.NewTempDir(t) - db, err := database.OpenDatabase(filepath.Join(tempDir, "test.db")) - require.NoError(t, err) - persister := NewScopedDBPersister(db, "test") - persister.Set("key", []byte("value")) - - err = persister.Sync() - require.NoError(t, err) - - newPersister := NewScopedDBPersister(db, "test") - err = newPersister.Load() - require.NoError(t, err) - - value := newPersister.Get("key") - require.Equal(t, []byte("value"), value) -} diff --git a/operator/operator.go b/operator/operator.go index b6874744..d4e28469 100644 --- a/operator/operator.go +++ b/operator/operator.go @@ -32,7 +32,7 @@ type Operator interface { Type() string // Start will start the operator. - Start() error + Start(Persister) error // Stop will stop the operator. Stop() error diff --git a/operator/persister.go b/operator/persister.go new file mode 100644 index 00000000..81639e1f --- /dev/null +++ b/operator/persister.go @@ -0,0 +1,22 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +// Persister is an interface used to persist data +type Persister interface { + Get(string) ([]byte, error) + Set(string, []byte) error + Delete(string) error +} diff --git a/pipeline/directed.go b/pipeline/directed.go index 481138fc..dd7ba632 100644 --- a/pipeline/directed.go +++ b/pipeline/directed.go @@ -34,12 +34,12 @@ type DirectedPipeline struct { } // Start will start the operators in a pipeline in reverse topological order -func (p *DirectedPipeline) Start() error { +func (p *DirectedPipeline) Start(persister operator.Persister) error { sortedNodes, _ := topo.Sort(p.Graph) for i := len(sortedNodes) - 1; i >= 0; i-- { operator := sortedNodes[i].(OperatorNode).Operator() operator.Logger().Debug("Starting operator") - if err := operator.Start(); err != nil { + if err := operator.Start(persister); err != nil { return err } operator.Logger().Debug("Started operator") diff --git a/pipeline/directed_test.go b/pipeline/directed_test.go index 60f4ac58..3888c6b1 100644 --- a/pipeline/directed_test.go +++ b/pipeline/directed_test.go @@ -87,10 +87,10 @@ func TestPipeline(t *testing.T) { pipeline, err := NewDirectedPipeline([]operator.Operator{}) require.NoError(t, err) - err = pipeline.Start() + err = pipeline.Start(testutil.NewMockPersister()) require.NoError(t, err) - err = pipeline.Start() + err = pipeline.Start(testutil.NewMockPersister()) require.NoError(t, err) require.NoError(t, pipeline.Stop()) @@ -100,7 +100,7 @@ func TestPipeline(t *testing.T) { pipeline, err := NewDirectedPipeline([]operator.Operator{}) require.NoError(t, err) - err = pipeline.Start() + err = pipeline.Start(testutil.NewMockPersister()) require.NoError(t, err) require.NoError(t, pipeline.Stop()) @@ -198,6 +198,7 @@ func TestPipelineStartOrder(t *testing.T) { mockOperator1 := testutil.NewMockOperator("operator1") mockOperator2 := testutil.NewMockOperator("operator2") mockOperator3 := testutil.NewMockOperator("operator3") + mockPersister := testutil.NewMockPersister() mockOperator1.On("Outputs").Return([]operator.Operator{mockOperator2}) mockOperator2.On("Outputs").Return([]operator.Operator{mockOperator3}) @@ -211,14 +212,14 @@ func TestPipelineStartOrder(t *testing.T) { mockOperator2.On("Logger", mock.Anything).Return(zap.NewNop().Sugar()) mockOperator3.On("Logger", mock.Anything).Return(zap.NewNop().Sugar()) - mockOperator1.On("Start").Return(fmt.Errorf("operator 1 failed to start")) - mockOperator2.On("Start").Run(func(mock.Arguments) { mock2Started = true }).Return(nil) - mockOperator3.On("Start").Run(func(mock.Arguments) { mock3Started = true }).Return(nil) + mockOperator1.On("Start", mockPersister).Return(fmt.Errorf("operator 1 failed to start")) + mockOperator2.On("Start", mockPersister).Run(func(mock.Arguments) { mock2Started = true }).Return(nil) + mockOperator3.On("Start", mockPersister).Run(func(mock.Arguments) { mock3Started = true }).Return(nil) pipeline, err := NewDirectedPipeline([]operator.Operator{mockOperator1, mockOperator2, mockOperator3}) require.NoError(t, err) - err = pipeline.Start() + err = pipeline.Start(mockPersister) require.Error(t, err) require.Contains(t, err.Error(), "operator 1 failed to start") require.True(t, mock2Started) @@ -231,6 +232,7 @@ func TestPipelineStopOrder(t *testing.T) { mockOperator1 := testutil.NewMockOperator("operator1") mockOperator2 := testutil.NewMockOperator("operator2") mockOperator3 := testutil.NewMockOperator("operator3") + mockPersister := testutil.NewMockPersister() mockOperator1.On("Outputs").Return([]operator.Operator{mockOperator2}) mockOperator2.On("Outputs").Return([]operator.Operator{mockOperator3}) @@ -244,9 +246,9 @@ func TestPipelineStopOrder(t *testing.T) { mockOperator2.On("Logger", mock.Anything).Return(zap.NewNop().Sugar()) mockOperator3.On("Logger", mock.Anything).Return(zap.NewNop().Sugar()) - mockOperator1.On("Start").Return(nil) - mockOperator2.On("Start").Return(nil) - mockOperator3.On("Start").Return(nil) + mockOperator1.On("Start", mockPersister).Return(nil) + mockOperator2.On("Start", mockPersister).Return(nil) + mockOperator3.On("Start", mockPersister).Return(nil) mockOperator1.On("Stop").Run(func(mock.Arguments) { stopOrder = append(stopOrder, 1) }).Return(nil) mockOperator2.On("Stop").Run(func(mock.Arguments) { stopOrder = append(stopOrder, 2) }).Return(nil) @@ -255,7 +257,7 @@ func TestPipelineStopOrder(t *testing.T) { pipeline, err := NewDirectedPipeline([]operator.Operator{mockOperator1, mockOperator2, mockOperator3}) require.NoError(t, err) - err = pipeline.Start() + err = pipeline.Start(mockPersister) require.NoError(t, err) err = pipeline.Stop() diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index 04d4992c..303721a6 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -16,11 +16,13 @@ package pipeline //go:generate mockery --name=^(Pipeline)$ --output=../testutil --outpkg=testutil --case=snake -import "github.com/open-telemetry/opentelemetry-log-collection/operator" +import ( + "github.com/open-telemetry/opentelemetry-log-collection/operator" +) // Pipeline is a collection of connected operators that exchange entries type Pipeline interface { - Start() error + Start(persister operator.Persister) error Stop() error Operators() []operator.Operator Render() ([]byte, error) diff --git a/plugin/config_test.go b/plugin/config_test.go index 3ac8fa81..70c0df49 100644 --- a/plugin/config_test.go +++ b/plugin/config_test.go @@ -129,7 +129,7 @@ pipeline: err = yaml.Unmarshal(pipelineConfig, &pipeline) require.NoError(t, err) - _, err = pipeline.BuildOperators(operator.NewBuildContext(nil, zaptest.NewLogger(t).Sugar())) + _, err = pipeline.BuildOperators(operator.NewBuildContext(zaptest.NewLogger(t).Sugar())) require.Error(t, err) require.Contains(t, err.Error(), "reached max plugin depth") } diff --git a/testutil/database.go b/testutil/database.go deleted file mode 100644 index cd3f2102..00000000 --- a/testutil/database.go +++ /dev/null @@ -1,70 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package testutil - -import ( - bbolt "go.etcd.io/bbolt" - - mock "github.com/stretchr/testify/mock" -) - -// Database is an autogenerated mock type for the Database type -type Database struct { - mock.Mock -} - -// Close provides a mock function with given fields: -func (_m *Database) Close() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Sync provides a mock function with given fields: -func (_m *Database) Sync() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Update provides a mock function with given fields: _a0 -func (_m *Database) Update(_a0 func(*bbolt.Tx) error) error { - ret := _m.Called(_a0) - - var r0 error - if rf, ok := ret.Get(0).(func(func(*bbolt.Tx) error) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// View provides a mock function with given fields: _a0 -func (_m *Database) View(_a0 func(*bbolt.Tx) error) error { - ret := _m.Called(_a0) - - var r0 error - if rf, ok := ret.Get(0).(func(func(*bbolt.Tx) error) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/testutil/mocks.go b/testutil/mocks.go index 7bf57c23..143dd831 100644 --- a/testutil/mocks.go +++ b/testutil/mocks.go @@ -69,7 +69,7 @@ func (f *FakeOutput) Outputs() []operator.Operator { return nil } func (f *FakeOutput) SetOutputs(outputs []operator.Operator) error { return nil } // Start immediately returns nil for a fake output -func (f *FakeOutput) Start() error { return nil } +func (f *FakeOutput) Start(_ operator.Persister) error { return nil } // Stop immediately returns nil for a fake output func (f *FakeOutput) Stop() error { return nil } diff --git a/testutil/operator.go b/testutil/operator.go index d1bf6f81..20ad941b 100644 --- a/testutil/operator.go +++ b/testutil/operator.go @@ -120,13 +120,13 @@ func (_m *Operator) SetOutputs(_a0 []operator.Operator) error { return r0 } -// Start provides a mock function with given fields: -func (_m *Operator) Start() error { - ret := _m.Called() +// Start provides a mock function with given fields: _a0 +func (_m *Operator) Start(_a0 operator.Persister) error { + ret := _m.Called(_a0) var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(operator.Persister) error); ok { + r0 = rf(_a0) } else { r0 = ret.Error(0) } diff --git a/testutil/operator_builder.go b/testutil/operator_builder.go index 6b024618..653bd854 100644 --- a/testutil/operator_builder.go +++ b/testutil/operator_builder.go @@ -13,15 +13,15 @@ type OperatorBuilder struct { } // Build provides a mock function with given fields: _a0 -func (_m *OperatorBuilder) Build(_a0 operator.BuildContext) (operator.Operator, error) { +func (_m *OperatorBuilder) Build(_a0 operator.BuildContext) ([]operator.Operator, error) { ret := _m.Called(_a0) - var r0 operator.Operator - if rf, ok := ret.Get(0).(func(operator.BuildContext) operator.Operator); ok { + var r0 []operator.Operator + if rf, ok := ret.Get(0).(func(operator.BuildContext) []operator.Operator); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(operator.Operator) + r0 = ret.Get(0).([]operator.Operator) } } diff --git a/testutil/pipeline.go b/testutil/pipeline.go index 06eea9c0..6111ee39 100644 --- a/testutil/pipeline.go +++ b/testutil/pipeline.go @@ -51,13 +51,13 @@ func (_m *Pipeline) Render() ([]byte, error) { return r0, r1 } -// Start provides a mock function with given fields: -func (_m *Pipeline) Start() error { - ret := _m.Called() +// Start provides a mock function with given fields: persister +func (_m *Pipeline) Start(persister operator.Persister) error { + ret := _m.Called(persister) var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(operator.Persister) error); ok { + r0 = rf(persister) } else { r0 = ret.Error(0) } diff --git a/testutil/util.go b/testutil/util.go index 32af5ab5..21a99fbc 100644 --- a/testutil/util.go +++ b/testutil/util.go @@ -19,6 +19,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "go.etcd.io/bbolt" @@ -71,12 +72,42 @@ func NewTestDatabase(t testing.TB) *bbolt.DB { // NewBuildContext will return a new build context for testing func NewBuildContext(t testing.TB) operator.BuildContext { return operator.BuildContext{ - Database: NewTestDatabase(t), Logger: logger.New(zaptest.NewLogger(t).Sugar()), Namespace: "$", } } +type mockPersister struct { + data map[string][]byte + dataMux sync.Mutex +} + +func (p *mockPersister) Get(k string) ([]byte, error) { + p.dataMux.Lock() + defer p.dataMux.Unlock() + return p.data[k], nil +} + +func (p *mockPersister) Set(k string, v []byte) error { + p.dataMux.Lock() + defer p.dataMux.Unlock() + p.data[k] = v + return nil +} + +func (p *mockPersister) Delete(k string) error { + p.dataMux.Lock() + defer p.dataMux.Unlock() + delete(p.data, k) + return nil +} + +// NewMockPersister will return a new persister for testing +func NewMockPersister() operator.Persister { + data := make(map[string][]byte) + return &mockPersister{data: data} +} + // Trim removes white space from the lines of a string func Trim(s string) string { lines := strings.Split(s, "\n") From 771855b758244b766e821d71e7cb34ee3512af82 Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Mon, 5 Apr 2021 16:43:13 -0400 Subject: [PATCH 2/4] Remove test databse --- go.mod | 2 -- go.sum | 35 ----------------------------------- testutil/util.go | 27 --------------------------- 3 files changed, 64 deletions(-) diff --git a/go.mod b/go.mod index aef82391..9e03a206 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,12 @@ go 1.14 require ( github.com/antonmedv/expr v1.8.9 - github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f github.com/json-iterator/go v1.1.10 github.com/mitchellh/mapstructure v1.4.1 github.com/observiq/ctimefmt v1.0.0 github.com/observiq/go-syslog/v3 v3.0.2 github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc github.com/stretchr/testify v1.7.0 - go.etcd.io/bbolt v1.3.4 go.opentelemetry.io/collector v0.22.0 go.uber.org/zap v1.16.0 golang.org/x/sys v0.0.0-20210217105451-b926d437f341 diff --git a/go.sum b/go.sum index 4b483190..464faafb 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -17,9 +16,7 @@ cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0 h1:kpgPA77kSSbjSs+fWHkPTxQ6J5Z2Qkruo5jfXEkHxNQ= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -125,7 +122,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/bsm/sarama-cluster v2.1.13+incompatible/go.mod h1:r7ao+4tTNXvWm+VRpRJchr2kQhqxgmAp2iEX5W96gMM= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -218,12 +214,9 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.7.3/go.mod h1:V1d2J5pfxYH6EjBAgSK7YNXcXlTWxUHdE1sVDXkjnig= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f h1:Sk0u0gIncQaQD23zAoAZs2DNi2u2l5UTLi4CmCBL5v8= -github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -332,7 +325,6 @@ github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= @@ -370,8 +362,6 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg= -github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f/go.mod h1:ijRvpgDJDI262hYq/IQVYgf8hd8IHUs93Ol0kvMBAx4= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -380,7 +370,6 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/lint v0.0.0-20170918230701-e5d664eb928e/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -407,12 +396,10 @@ github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -446,7 +433,6 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= @@ -464,7 +450,6 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -509,7 +494,6 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -526,7 +510,6 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.4/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= @@ -604,7 +587,6 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -617,11 +599,9 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.0.10-0.20170816031813-ad5389df28cd/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.2/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -648,7 +628,6 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -733,7 +712,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= @@ -847,26 +825,21 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v0.0.0-20170901151539-12bd96e66386/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1-0.20170901120850-7aff26db30c1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= @@ -922,8 +895,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= -go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -1079,7 +1050,6 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1090,7 +1060,6 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210210192628-66670185b0cd h1:2arJsLyTCJGek+eeptQ3z49Rqndm0f+zvvpwNIXWNIA= golang.org/x/oauth2 v0.0.0-20210210192628-66670185b0cd/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20170517211232-f52d1811a629/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1189,7 +1158,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1286,7 +1254,6 @@ gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6d gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.0.0-20170921000349-586095a6e407/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1317,7 +1284,6 @@ google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1360,7 +1326,6 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210302174412-5ede27ff9881/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/testutil/util.go b/testutil/util.go index 21a99fbc..9a57bf7e 100644 --- a/testutil/util.go +++ b/testutil/util.go @@ -17,12 +17,10 @@ package testutil import ( "io/ioutil" "os" - "path/filepath" "strings" "sync" "testing" - "go.etcd.io/bbolt" "go.uber.org/zap/zaptest" "github.com/open-telemetry/opentelemetry-log-collection/logger" @@ -44,31 +42,6 @@ func NewTempDir(t testing.TB) string { return tempDir } -// NewTestDatabase will return a new database for testing -func NewTestDatabase(t testing.TB) *bbolt.DB { - tempDir, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf(err.Error()) - t.FailNow() - } - - t.Cleanup(func() { - os.RemoveAll(tempDir) - }) - - db, err := bbolt.Open(filepath.Join(tempDir, "test.db"), 0666, nil) - if err != nil { - t.Errorf(err.Error()) - t.FailNow() - } - - t.Cleanup(func() { - db.Close() - }) - - return db -} - // NewBuildContext will return a new build context for testing func NewBuildContext(t testing.TB) operator.BuildContext { return operator.BuildContext{ From 750a9ea8db61179fb6ec60ca82cb8c9c10fb5459 Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Tue, 6 Apr 2021 11:36:44 -0400 Subject: [PATCH 3/4] Add context to persister --- internal/tools/go.mod | 1 + internal/tools/go.sum | 3 +++ operator/builtin/input/file/file.go | 12 ++++++------ operator/builtin/input/journald/journald.go | 4 ++-- operator/builtin/input/windows/operator.go | 12 ++++++------ operator/persister.go | 8 +++++--- testutil/util.go | 7 ++++--- 7 files changed, 27 insertions(+), 20 deletions(-) diff --git a/internal/tools/go.mod b/internal/tools/go.mod index 1e3f97ae..c86ed728 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -6,4 +6,5 @@ require ( github.com/golangci/golangci-lint v1.38.0 github.com/google/addlicense v0.0.0-20200906110928-a0294312aa76 github.com/tcnksm/ghr v0.13.0 + github.com/vektra/mockery v1.1.2 // indirect ) diff --git a/internal/tools/go.sum b/internal/tools/go.sum index 653ed8ef..6b69fe6e 100644 --- a/internal/tools/go.sum +++ b/internal/tools/go.sum @@ -455,6 +455,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/vektra/mockery v1.1.2 h1:uc0Yn67rJpjt8U/mAZimdCKn9AeA97BOkjpmtBSlfP4= +github.com/vektra/mockery v1.1.2/go.mod h1:VcfZjKaFOPO+MpN4ZvwPjs4c48lkq1o3Ym8yHZJu0jU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -598,6 +600,7 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/operator/builtin/input/file/file.go b/operator/builtin/input/file/file.go index a46cbc5b..9400bf9f 100644 --- a/operator/builtin/input/file/file.go +++ b/operator/builtin/input/file/file.go @@ -71,7 +71,7 @@ func (f *InputOperator) Start(persister operator.Persister) error { f.persister = persister // Load offsets from disk - if err := f.loadLastPollFiles(); err != nil { + if err := f.loadLastPollFiles(ctx); err != nil { return fmt.Errorf("read known files from database: %s", err) } @@ -176,7 +176,7 @@ func (f *InputOperator) poll(ctx context.Context) { } f.saveCurrent(readers) - f.syncLastPollFiles() + f.syncLastPollFiles(ctx) } // getMatches gets a list of paths given an array of glob patterns to include and exclude @@ -320,7 +320,7 @@ func (f *InputOperator) findFingerprintMatch(fp *Fingerprint) (*Reader, bool) { const knownFilesKey = "knownFiles" // syncLastPollFiles syncs the most recent set of files to the database -func (f *InputOperator) syncLastPollFiles() { +func (f *InputOperator) syncLastPollFiles(ctx context.Context) { var buf bytes.Buffer enc := json.NewEncoder(&buf) @@ -337,14 +337,14 @@ func (f *InputOperator) syncLastPollFiles() { } } - if err := f.persister.Set(knownFilesKey, buf.Bytes()); err != nil { + if err := f.persister.Set(ctx, knownFilesKey, buf.Bytes()); err != nil { f.Errorw("Failed to sync to database", zap.Error(err)) } } // syncLastPollFiles loads the most recent set of files to the database -func (f *InputOperator) loadLastPollFiles() error { - encoded, err := f.persister.Get(knownFilesKey) +func (f *InputOperator) loadLastPollFiles(ctx context.Context) error { + encoded, err := f.persister.Get(ctx, knownFilesKey) if err != nil { return err } diff --git a/operator/builtin/input/journald/journald.go b/operator/builtin/input/journald/journald.go index e9a5475c..bf3cfe8b 100644 --- a/operator/builtin/input/journald/journald.go +++ b/operator/builtin/input/journald/journald.go @@ -128,7 +128,7 @@ func (operator *JournaldInput) Start(persister operator.Persister) error { operator.cancel = cancel // Start from a cursor if there is a saved offset - cursor, err := persister.Get(lastReadCursorKey) + cursor, err := persister.Get(ctx, lastReadCursorKey) if err != nil { return fmt.Errorf("failed to get journalctl state: %s", err) } @@ -167,7 +167,7 @@ func (operator *JournaldInput) Start(persister operator.Persister) error { operator.Warnw("Failed to parse journal entry", zap.Error(err)) continue } - if err := operator.persister.Set(lastReadCursorKey, []byte(cursor)); err != nil { + if err := operator.persister.Set(ctx, lastReadCursorKey, []byte(cursor)); err != nil { operator.Warnw("Failed to set offset", zap.Error(err)) } operator.Write(ctx, entry) diff --git a/operator/builtin/input/windows/operator.go b/operator/builtin/input/windows/operator.go index 117d9773..a66392b4 100644 --- a/operator/builtin/input/windows/operator.go +++ b/operator/builtin/input/windows/operator.go @@ -104,7 +104,7 @@ func (e *EventLogInput) Start(persister operator.Persister) error { e.persister = persister e.bookmark = NewBookmark() - offsetXML, err := e.getBookmarkOffset() + offsetXML, err := e.getBookmarkOffset(ctx) if err != nil { return fmt.Errorf("failed to retrieve bookmark offset: %s", err) } @@ -183,7 +183,7 @@ func (e *EventLogInput) read(ctx context.Context) int { for i, event := range events { e.processEvent(ctx, event) if len(events) == i+1 { - e.updateBookmarkOffset(event) + e.updateBookmarkOffset(ctx, event) } event.Close() } @@ -232,13 +232,13 @@ func (e *EventLogInput) sendEvent(ctx context.Context, eventXML EventXML) { } // getBookmarkXML will get the bookmark xml from the offsets database. -func (e *EventLogInput) getBookmarkOffset() (string, error) { - bytes, err := e.persister.Get(e.channel) +func (e *EventLogInput) getBookmarkOffset(ctx context.Context) (string, error) { + bytes, err := e.persister.Get(ctx, e.channel) return string(bytes), err } // updateBookmark will update the bookmark xml and save it in the offsets database. -func (e *EventLogInput) updateBookmarkOffset(event Event) { +func (e *EventLogInput) updateBookmarkOffset(ctx context.Context, event Event) { if err := e.bookmark.Update(event); err != nil { e.Errorf("Failed to update bookmark from event: %s", err) return @@ -250,7 +250,7 @@ func (e *EventLogInput) updateBookmarkOffset(event Event) { return } - if err := e.persister.Set(e.channel, []byte(bookmarkXML)); err != nil { + if err := e.persister.Set(ctx, e.channel, []byte(bookmarkXML)); err != nil { e.Errorf("failed to set offsets: %s", err) return } diff --git a/operator/persister.go b/operator/persister.go index 81639e1f..4c3fb837 100644 --- a/operator/persister.go +++ b/operator/persister.go @@ -14,9 +14,11 @@ package operator +import "context" + // Persister is an interface used to persist data type Persister interface { - Get(string) ([]byte, error) - Set(string, []byte) error - Delete(string) error + Get(context.Context, string) ([]byte, error) + Set(context.Context, string, []byte) error + Delete(context.Context, string) error } diff --git a/testutil/util.go b/testutil/util.go index 9a57bf7e..67aa5cdc 100644 --- a/testutil/util.go +++ b/testutil/util.go @@ -15,6 +15,7 @@ package testutil import ( + context "context" "io/ioutil" "os" "strings" @@ -55,20 +56,20 @@ type mockPersister struct { dataMux sync.Mutex } -func (p *mockPersister) Get(k string) ([]byte, error) { +func (p *mockPersister) Get(ctx context.Context, k string) ([]byte, error) { p.dataMux.Lock() defer p.dataMux.Unlock() return p.data[k], nil } -func (p *mockPersister) Set(k string, v []byte) error { +func (p *mockPersister) Set(ctx context.Context, k string, v []byte) error { p.dataMux.Lock() defer p.dataMux.Unlock() p.data[k] = v return nil } -func (p *mockPersister) Delete(k string) error { +func (p *mockPersister) Delete(ctx context.Context, k string) error { p.dataMux.Lock() defer p.dataMux.Unlock() delete(p.data, k) From 6f71466f4701e8207c761a8b542480ddc37b45f9 Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Tue, 6 Apr 2021 12:29:11 -0400 Subject: [PATCH 4/4] Operators receive a scoped persister on start --- agent/agent_test.go | 4 +- operator/builtin/input/file/file_test.go | 58 +++++++++---------- .../builtin/input/generate/generate_test.go | 2 +- .../builtin/input/journald/journald_test.go | 2 +- operator/builtin/input/stanza/stanza_test.go | 2 +- operator/builtin/input/stdin/stdin_test.go | 2 +- operator/builtin/input/syslog/syslog_test.go | 2 +- operator/builtin/input/tcp/tcp_test.go | 6 +- operator/builtin/input/udp/udp_test.go | 4 +- operator/helper/operator_test.go | 2 +- operator/persister.go | 27 ++++++++- pipeline/directed.go | 10 ++-- pipeline/directed_test.go | 22 +++---- testutil/util.go | 9 ++- 14 files changed, 92 insertions(+), 60 deletions(-) diff --git a/agent/agent_test.go b/agent/agent_test.go index d2559c0c..68ddebc8 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -27,7 +27,7 @@ import ( func TestStartAgentSuccess(t *testing.T) { logger := zap.NewNop().Sugar() pipeline := &testutil.Pipeline{} - persister := testutil.NewMockPersister() + persister := testutil.NewMockPersister("test") pipeline.On("Start", persister).Return(nil) agent := LogAgent{ @@ -42,7 +42,7 @@ func TestStartAgentSuccess(t *testing.T) { func TestStartAgentFailure(t *testing.T) { logger := zap.NewNop().Sugar() pipeline := &testutil.Pipeline{} - persister := testutil.NewMockPersister() + persister := testutil.NewMockPersister("test") failure := fmt.Errorf("failed to start pipeline") pipeline.On("Start", persister).Return(failure) diff --git a/operator/builtin/input/file/file_test.go b/operator/builtin/input/file/file_test.go index bee92cdb..c538aaa4 100644 --- a/operator/builtin/input/file/file_test.go +++ b/operator/builtin/input/file/file_test.go @@ -272,7 +272,7 @@ See this issue for details: https://github.com/census-instrumentation/opencensus operator, _, tempDir := newTestFileOperator(t, nil, nil) _ = openTemp(t, tempDir) - err := operator.Start(testutil.NewMockPersister()) + err := operator.Start(testutil.NewMockPersister("test")) require.NoError(t, err) operator.Stop() } @@ -290,7 +290,7 @@ func TestAddFileFields(t *testing.T) { temp := openTemp(t, tempDir) writeString(t, temp, "testlog\n") - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() e := waitForOne(t, logReceived) @@ -308,7 +308,7 @@ func TestReadExistingLogs(t *testing.T) { temp := openTemp(t, tempDir) writeString(t, temp, "testlog1\ntestlog2\n") - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() waitForMessage(t, logReceived, "testlog1") @@ -320,7 +320,7 @@ func TestReadExistingLogs(t *testing.T) { func TestReadNewLogs(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - operator.persister = testutil.NewMockPersister() + operator.persister = testutil.NewMockPersister("test") // Poll once so we know this isn't a new file operator.poll(context.Background()) @@ -343,7 +343,7 @@ func TestReadNewLogs(t *testing.T) { func TestReadExistingAndNewLogs(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - operator.persister = testutil.NewMockPersister() + operator.persister = testutil.NewMockPersister("test") // Start with a file with an entry in it, and expect that entry // to come through when we poll for the first time @@ -366,7 +366,7 @@ func TestStartAtEnd(t *testing.T) { operator, logReceived, tempDir := newTestFileOperator(t, func(cfg *InputConfig) { cfg.StartAt = "end" }, nil) - operator.persister = testutil.NewMockPersister() + operator.persister = testutil.NewMockPersister("test") temp := openTemp(t, tempDir) writeString(t, temp, "testlog1\n") @@ -387,7 +387,7 @@ func TestStartAtEnd(t *testing.T) { func TestStartAtEndNewFile(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - operator.persister = testutil.NewMockPersister() + operator.persister = testutil.NewMockPersister("test") operator.startAtBeginning = false operator.poll(context.Background()) @@ -410,7 +410,7 @@ func TestNoNewline(t *testing.T) { temp := openTemp(t, tempDir) writeString(t, temp, "testlog1\ntestlog2") - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() waitForMessage(t, logReceived, "testlog1") @@ -425,7 +425,7 @@ func TestSkipEmpty(t *testing.T) { temp := openTemp(t, tempDir) writeString(t, temp, "testlog1\n\ntestlog2\n") - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() waitForMessage(t, logReceived, "testlog1") @@ -437,7 +437,7 @@ func TestSkipEmpty(t *testing.T) { func TestSplitWrite(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - operator.persister = testutil.NewMockPersister() + operator.persister = testutil.NewMockPersister("test") temp := openTemp(t, tempDir) writeString(t, temp, "testlog1") @@ -454,7 +454,7 @@ func TestDecodeBufferIsResized(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() temp := openTemp(t, tempDir) @@ -474,7 +474,7 @@ func TestMultiFileSimple(t *testing.T) { writeString(t, temp1, "testlog1\n") writeString(t, temp2, "testlog2\n") - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() waitForMessages(t, logReceived, []string{"testlog1", "testlog2"}) @@ -509,7 +509,7 @@ func TestMultiFileParallel_PreloadedFiles(t *testing.T) { }(temp, i) } - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() waitForMessages(t, logReceived, expected) @@ -533,7 +533,7 @@ func TestMultiFileParallel_LiveFiles(t *testing.T) { } } - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() temps := make([]*os.File, 0, numFiles) @@ -576,7 +576,7 @@ func TestMultiFileRotate(t *testing.T) { } } - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() temps := make([]*os.File, 0, numFiles) @@ -633,7 +633,7 @@ func TestMultiFileRotateSlow(t *testing.T) { } } - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() var wg sync.WaitGroup @@ -679,7 +679,7 @@ func TestMultiCopyTruncateSlow(t *testing.T) { } } - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() var wg sync.WaitGroup @@ -776,7 +776,7 @@ func (rt rotationTest) run(tc rotationTest, copyTruncate, sequential bool) func( expected = append(expected, fmt.Sprintf("%s %3d", baseStr, i)) } - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() for _, message := range expected { @@ -859,7 +859,7 @@ func TestMoveFile(t *testing.T) { } t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - operator.persister = testutil.NewMockPersister() + operator.persister = testutil.NewMockPersister("test") temp1 := openTemp(t, tempDir) writeString(t, temp1, "testlog1\n") @@ -884,7 +884,7 @@ func TestMoveFile(t *testing.T) { func TestTruncateThenWrite(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - operator.persister = testutil.NewMockPersister() + operator.persister = testutil.NewMockPersister("test") temp1 := openTemp(t, tempDir) writeString(t, temp1, "testlog1\ntestlog2\n") @@ -911,7 +911,7 @@ func TestTruncateThenWrite(t *testing.T) { func TestCopyTruncateWriteBoth(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - operator.persister = testutil.NewMockPersister() + operator.persister = testutil.NewMockPersister("test") temp1 := openTemp(t, tempDir) writeString(t, temp1, "testlog1\ntestlog2\n") @@ -946,7 +946,7 @@ func TestCopyTruncateWriteBoth(t *testing.T) { func TestOffsetsAfterRestart(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - persister := testutil.NewMockPersister() + persister := testutil.NewMockPersister("test") temp1 := openTemp(t, tempDir) writeString(t, temp1, "testlog1\n") @@ -969,7 +969,7 @@ func TestOffsetsAfterRestart(t *testing.T) { func TestOffsetsAfterRestart_BigFiles(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - persister := testutil.NewMockPersister() + persister := testutil.NewMockPersister("test") log1 := stringWithLength(2000) log2 := stringWithLength(2000) @@ -993,7 +993,7 @@ func TestOffsetsAfterRestart_BigFiles(t *testing.T) { func TestOffsetsAfterRestart_BigFilesWrittenWhileOff(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - persister := testutil.NewMockPersister() + persister := testutil.NewMockPersister("test") log1 := stringWithLength(2000) log2 := stringWithLength(2000) @@ -1018,7 +1018,7 @@ func TestOffsetsAfterRestart_BigFilesWrittenWhileOff(t *testing.T) { func TestFileMovedWhileOff_BigFiles(t *testing.T) { t.Parallel() operator, logReceived, tempDir := newTestFileOperator(t, nil, nil) - persister := testutil.NewMockPersister() + persister := testutil.NewMockPersister("test") log1 := stringWithLength(1000) log2 := stringWithLength(1000) @@ -1058,7 +1058,7 @@ func TestManyLogsDelivered(t *testing.T) { } // Start the operator - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() // Write lots of logs @@ -1095,7 +1095,7 @@ func TestFileBatching(t *testing.T) { out.Received = make(chan *entry.Entry, expectedLinesPerBatch) }, ) - operator.persister = testutil.NewMockPersister() + operator.persister = testutil.NewMockPersister("test") temps := make([]*os.File, 0, files) for i := 0; i < files; i++ { @@ -1296,7 +1296,7 @@ func TestEncodings(t *testing.T) { _, err := temp.Write(tc.contents) require.NoError(t, err) - require.NoError(t, operator.Start(testutil.NewMockPersister())) + require.NoError(t, operator.Start(testutil.NewMockPersister("test"))) defer operator.Stop() for _, expected := range tc.expected { @@ -1350,7 +1350,7 @@ func BenchmarkFileInput(b *testing.B) { err = op.SetOutputs([]operator.Operator{fakeOutput}) require.NoError(b, err) - err = op.Start(testutil.NewMockPersister()) + err = op.Start(testutil.NewMockPersister("test")) defer op.Stop() require.NoError(b, err) diff --git a/operator/builtin/input/generate/generate_test.go b/operator/builtin/input/generate/generate_test.go index 17b7b252..66b5aac7 100644 --- a/operator/builtin/input/generate/generate_test.go +++ b/operator/builtin/input/generate/generate_test.go @@ -40,7 +40,7 @@ func TestInputGenerate(t *testing.T) { err = op.SetOutputs([]operator.Operator{fake}) require.NoError(t, err) - require.NoError(t, op.Start(testutil.NewMockPersister())) + require.NoError(t, op.Start(testutil.NewMockPersister("test"))) defer op.Stop() for i := 0; i < 5; i++ { diff --git a/operator/builtin/input/journald/journald_test.go b/operator/builtin/input/journald/journald_test.go index 3eb03272..04c853e0 100644 --- a/operator/builtin/input/journald/journald_test.go +++ b/operator/builtin/input/journald/journald_test.go @@ -67,7 +67,7 @@ func TestInputJournald(t *testing.T) { return &fakeJournaldCmd{} } - err = op.Start(testutil.NewMockPersister()) + err = op.Start(testutil.NewMockPersister("test")) require.NoError(t, err) defer op.Stop() diff --git a/operator/builtin/input/stanza/stanza_test.go b/operator/builtin/input/stanza/stanza_test.go index 51b890be..03350e63 100644 --- a/operator/builtin/input/stanza/stanza_test.go +++ b/operator/builtin/input/stanza/stanza_test.go @@ -36,7 +36,7 @@ func TestStanzaOperator(t *testing.T) { fake := testutil.NewFakeOutput(t) op.SetOutputs([]operator.Operator{fake}) - require.NoError(t, op.Start(testutil.NewMockPersister())) + require.NoError(t, op.Start(testutil.NewMockPersister("test"))) defer op.Stop() bc.Logger.Errorw("test failure", "key", "value") diff --git a/operator/builtin/input/stdin/stdin_test.go b/operator/builtin/input/stdin/stdin_test.go index c1a529b3..3849e385 100644 --- a/operator/builtin/input/stdin/stdin_test.go +++ b/operator/builtin/input/stdin/stdin_test.go @@ -40,7 +40,7 @@ func TestStdin(t *testing.T) { stdin := op[0].(*StdinInput) stdin.stdin = r - require.NoError(t, stdin.Start(testutil.NewMockPersister())) + require.NoError(t, stdin.Start(testutil.NewMockPersister("test"))) defer stdin.Stop() w.WriteString("test") diff --git a/operator/builtin/input/syslog/syslog_test.go b/operator/builtin/input/syslog/syslog_test.go index 6e2d5a60..d3b87db7 100644 --- a/operator/builtin/input/syslog/syslog_test.go +++ b/operator/builtin/input/syslog/syslog_test.go @@ -58,7 +58,7 @@ func SyslogInputTest(t *testing.T, cfg *SyslogInputConfig, tc syslog.Case) { p, err := pipeline.NewDirectedPipeline(ops) require.NoError(t, err) - err = p.Start(testutil.NewMockPersister()) + err = p.Start(testutil.NewMockPersister("test")) require.NoError(t, err) var conn net.Conn diff --git a/operator/builtin/input/tcp/tcp_test.go b/operator/builtin/input/tcp/tcp_test.go index 4616e6c0..44284edc 100644 --- a/operator/builtin/input/tcp/tcp_test.go +++ b/operator/builtin/input/tcp/tcp_test.go @@ -101,7 +101,7 @@ func tcpInputTest(input []byte, expected []string) func(t *testing.T) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = tcpInput.Start(testutil.NewMockPersister()) + err = tcpInput.Start(testutil.NewMockPersister("test")) require.NoError(t, err) defer tcpInput.Stop() @@ -170,7 +170,7 @@ func tlsTCPInputTest(input []byte, expected []string) func(t *testing.T) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = tcpInput.Start(testutil.NewMockPersister()) + err = tcpInput.Start(testutil.NewMockPersister("test")) require.NoError(t, err) defer tcpInput.Stop() @@ -299,7 +299,7 @@ func BenchmarkTcpInput(b *testing.B) { tcpInput := op.(*TCPInput) tcpInput.InputOperator.OutputOperators = []operator.Operator{fakeOutput} - err = tcpInput.Start(testutil.NewMockPersister()) + err = tcpInput.Start(testutil.NewMockPersister("test")) require.NoError(b, err) done := make(chan struct{}) diff --git a/operator/builtin/input/udp/udp_test.go b/operator/builtin/input/udp/udp_test.go index 4a2e802a..32d54a08 100644 --- a/operator/builtin/input/udp/udp_test.go +++ b/operator/builtin/input/udp/udp_test.go @@ -47,7 +47,7 @@ func udpInputTest(input []byte, expected []string) func(t *testing.T) { entryChan <- args.Get(1).(*entry.Entry) }).Return(nil) - err = udpInput.Start(testutil.NewMockPersister()) + err = udpInput.Start(testutil.NewMockPersister("test")) require.NoError(t, err) defer udpInput.Stop() @@ -95,7 +95,7 @@ func BenchmarkUdpInput(b *testing.B) { udpInput := op.(*UDPInput) udpInput.InputOperator.OutputOperators = []operator.Operator{fakeOutput} - err = udpInput.Start(testutil.NewMockPersister()) + err = udpInput.Start(testutil.NewMockPersister("test")) require.NoError(b, err) done := make(chan struct{}) diff --git a/operator/helper/operator_test.go b/operator/helper/operator_test.go index 2eefa683..00aed54c 100644 --- a/operator/helper/operator_test.go +++ b/operator/helper/operator_test.go @@ -113,7 +113,7 @@ func TestBasicOperatorStart(t *testing.T) { OperatorID: "test-id", OperatorType: "test-type", } - err := operator.Start(testutil.NewMockPersister()) + err := operator.Start(testutil.NewMockPersister("test")) require.NoError(t, err) } diff --git a/operator/persister.go b/operator/persister.go index 4c3fb837..928c083b 100644 --- a/operator/persister.go +++ b/operator/persister.go @@ -14,7 +14,10 @@ package operator -import "context" +import ( + "context" + "fmt" +) // Persister is an interface used to persist data type Persister interface { @@ -22,3 +25,25 @@ type Persister interface { Set(context.Context, string, []byte) error Delete(context.Context, string) error } + +type scopedPersister struct { + Persister + scope string +} + +func NewScopedPersister(s string, p Persister) Persister { + return &scopedPersister{ + Persister: p, + scope: s, + } +} + +func (p scopedPersister) Get(ctx context.Context, key string) ([]byte, error) { + return p.Persister.Get(ctx, fmt.Sprintf("%s.%s", p.scope, key)) +} +func (p scopedPersister) Set(ctx context.Context, key string, value []byte) error { + return p.Persister.Set(ctx, fmt.Sprintf("%s.%s", p.scope, key), value) +} +func (p scopedPersister) Delete(ctx context.Context, key string) error { + return p.Persister.Delete(ctx, fmt.Sprintf("%s.%s", p.scope, key)) +} diff --git a/pipeline/directed.go b/pipeline/directed.go index dd7ba632..bba1939f 100644 --- a/pipeline/directed.go +++ b/pipeline/directed.go @@ -37,12 +37,14 @@ type DirectedPipeline struct { func (p *DirectedPipeline) Start(persister operator.Persister) error { sortedNodes, _ := topo.Sort(p.Graph) for i := len(sortedNodes) - 1; i >= 0; i-- { - operator := sortedNodes[i].(OperatorNode).Operator() - operator.Logger().Debug("Starting operator") - if err := operator.Start(persister); err != nil { + op := sortedNodes[i].(OperatorNode).Operator() + + scopedPersister := operator.NewScopedPersister(op.ID(), persister) + op.Logger().Debug("Starting operator") + if err := op.Start(scopedPersister); err != nil { return err } - operator.Logger().Debug("Started operator") + op.Logger().Debug("Started operator") } return nil diff --git a/pipeline/directed_test.go b/pipeline/directed_test.go index 3888c6b1..73bd7379 100644 --- a/pipeline/directed_test.go +++ b/pipeline/directed_test.go @@ -87,10 +87,10 @@ func TestPipeline(t *testing.T) { pipeline, err := NewDirectedPipeline([]operator.Operator{}) require.NoError(t, err) - err = pipeline.Start(testutil.NewMockPersister()) + err = pipeline.Start(testutil.NewMockPersister("test1")) require.NoError(t, err) - err = pipeline.Start(testutil.NewMockPersister()) + err = pipeline.Start(testutil.NewMockPersister("test2")) require.NoError(t, err) require.NoError(t, pipeline.Stop()) @@ -100,7 +100,7 @@ func TestPipeline(t *testing.T) { pipeline, err := NewDirectedPipeline([]operator.Operator{}) require.NoError(t, err) - err = pipeline.Start(testutil.NewMockPersister()) + err = pipeline.Start(testutil.NewMockPersister("test3")) require.NoError(t, err) require.NoError(t, pipeline.Stop()) @@ -198,7 +198,7 @@ func TestPipelineStartOrder(t *testing.T) { mockOperator1 := testutil.NewMockOperator("operator1") mockOperator2 := testutil.NewMockOperator("operator2") mockOperator3 := testutil.NewMockOperator("operator3") - mockPersister := testutil.NewMockPersister() + mockPersister := testutil.NewUnscopedMockPersister() mockOperator1.On("Outputs").Return([]operator.Operator{mockOperator2}) mockOperator2.On("Outputs").Return([]operator.Operator{mockOperator3}) @@ -212,9 +212,9 @@ func TestPipelineStartOrder(t *testing.T) { mockOperator2.On("Logger", mock.Anything).Return(zap.NewNop().Sugar()) mockOperator3.On("Logger", mock.Anything).Return(zap.NewNop().Sugar()) - mockOperator1.On("Start", mockPersister).Return(fmt.Errorf("operator 1 failed to start")) - mockOperator2.On("Start", mockPersister).Run(func(mock.Arguments) { mock2Started = true }).Return(nil) - mockOperator3.On("Start", mockPersister).Run(func(mock.Arguments) { mock3Started = true }).Return(nil) + mockOperator1.On("Start", testutil.NewMockPersister(mockOperator1.ID())).Return(fmt.Errorf("operator 1 failed to start")) + mockOperator2.On("Start", testutil.NewMockPersister(mockOperator2.ID())).Run(func(mock.Arguments) { mock2Started = true }).Return(nil) + mockOperator3.On("Start", testutil.NewMockPersister(mockOperator3.ID())).Run(func(mock.Arguments) { mock3Started = true }).Return(nil) pipeline, err := NewDirectedPipeline([]operator.Operator{mockOperator1, mockOperator2, mockOperator3}) require.NoError(t, err) @@ -232,7 +232,7 @@ func TestPipelineStopOrder(t *testing.T) { mockOperator1 := testutil.NewMockOperator("operator1") mockOperator2 := testutil.NewMockOperator("operator2") mockOperator3 := testutil.NewMockOperator("operator3") - mockPersister := testutil.NewMockPersister() + mockPersister := testutil.NewUnscopedMockPersister() mockOperator1.On("Outputs").Return([]operator.Operator{mockOperator2}) mockOperator2.On("Outputs").Return([]operator.Operator{mockOperator3}) @@ -246,9 +246,9 @@ func TestPipelineStopOrder(t *testing.T) { mockOperator2.On("Logger", mock.Anything).Return(zap.NewNop().Sugar()) mockOperator3.On("Logger", mock.Anything).Return(zap.NewNop().Sugar()) - mockOperator1.On("Start", mockPersister).Return(nil) - mockOperator2.On("Start", mockPersister).Return(nil) - mockOperator3.On("Start", mockPersister).Return(nil) + mockOperator1.On("Start", testutil.NewMockPersister(mockOperator1.ID())).Return(nil) + mockOperator2.On("Start", testutil.NewMockPersister(mockOperator2.ID())).Return(nil) + mockOperator3.On("Start", testutil.NewMockPersister(mockOperator3.ID())).Return(nil) mockOperator1.On("Stop").Run(func(mock.Arguments) { stopOrder = append(stopOrder, 1) }).Return(nil) mockOperator2.On("Stop").Run(func(mock.Arguments) { stopOrder = append(stopOrder, 2) }).Return(nil) diff --git a/testutil/util.go b/testutil/util.go index 67aa5cdc..9e186eff 100644 --- a/testutil/util.go +++ b/testutil/util.go @@ -76,12 +76,17 @@ func (p *mockPersister) Delete(ctx context.Context, k string) error { return nil } -// NewMockPersister will return a new persister for testing -func NewMockPersister() operator.Persister { +// NewUnscopedMockPersister will return a new persister for testing +func NewUnscopedMockPersister() operator.Persister { data := make(map[string][]byte) return &mockPersister{data: data} } +// NewMockPersister will return a new persister for testing +func NewMockPersister(scope string) operator.Persister { + return operator.NewScopedPersister(scope, NewUnscopedMockPersister()) +} + // Trim removes white space from the lines of a string func Trim(s string) string { lines := strings.Split(s, "\n")