diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ac504d32..35e7f78d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased +### Added +- Disk buffer for output operators +### Changed +- Split buffers into buffers and flushers for better modularity +- New memory buffer design for a uniform interface between disk and memory buffers + ## [0.9.14] - 2020-08-31 ### Fixed - Rendering issue with the `kubernetes_events` plugin diff --git a/commands/graph.go b/commands/graph.go index e0b3101bd..3eb433d08 100644 --- a/commands/graph.go +++ b/commands/graph.go @@ -49,6 +49,7 @@ func runGraph(_ *cobra.Command, _ []string, flags *RootFlags) { } buildContext := pg.BuildContext{ + Database: operator.NewStubDatabase(), PluginRegistry: pluginRegistry, Logger: logger, } diff --git a/commands/offsets.go b/commands/offsets.go index a919693a3..2c736170d 100644 --- a/commands/offsets.go +++ b/commands/offsets.go @@ -42,7 +42,7 @@ func NewOffsetsClearCmd(rootFlags *RootFlags) *cobra.Command { db, err := agent.OpenDatabase(rootFlags.DatabaseFile) exitOnErr("Failed to open database", err) defer db.Close() - defer db.Sync() + defer func() { _ = db.Sync() }() if all { if len(args) != 0 { @@ -94,7 +94,7 @@ func NewOffsetsListCmd(rootFlags *RootFlags) *cobra.Command { exitOnErr("Failed to open database", err) defer db.Close() - db.View(func(tx *bbolt.Tx) error { + err = db.View(func(tx *bbolt.Tx) error { offsetBucket := tx.Bucket(helper.OffsetsBucket) if offsetBucket == nil { return nil @@ -105,7 +105,9 @@ func NewOffsetsListCmd(rootFlags *RootFlags) *cobra.Command { return nil }) }) - + if err != nil { + exitOnErr("Failed to read database", err) + } }, } diff --git a/commands/offsets_test.go b/commands/offsets_test.go index ae3add79a..b8cd24fa1 100644 --- a/commands/offsets_test.go +++ b/commands/offsets_test.go @@ -70,5 +70,4 @@ func TestOffsets(t *testing.T) { err = offsetsList.Execute() require.NoError(t, err) require.Equal(t, "$.testoperatorid1\n", buf.String()) - } diff --git a/docs/operators/elastic_output.md b/docs/operators/elastic_output.md index 916a5c568..9e4d743f0 100644 --- a/docs/operators/elastic_output.md +++ b/docs/operators/elastic_output.md @@ -14,6 +14,8 @@ The `elastic_output` operator will send entries to an Elasticsearch instance | `api_key` | | Base64-encoded token for authorization. If set, overrides username and password | | `index_field` | default | A [field](/docs/types/field.md) that indicates which index to send the log entry to | | `id_field` | | A [field](/docs/types/field.md) that contains an id for the entry. If unset, a unique id is generated | +| `buffer` | | A [buffer](/docs/types/buffer.md) block indicating how to buffer entries before flushing | +| `flusher` | | A [flusher](/docs/types/flusher.md) block configuring flushing behavior | ### Example Configurations @@ -27,3 +29,18 @@ Configuration: - "http://localhost:9200" api_key: ``` + +#### Configuration with non-default buffer and flusher params + +Configuration: +```yaml +- type: elastic_output + addresses: + - "http://localhost:9200" + api_key: + buffer: + type: disk + path: /tmp/stanza_buffer + flusher: + max_concurrent: 8 +``` diff --git a/docs/operators/google_cloud_output.md b/docs/operators/google_cloud_output.md index c437c1df0..1a91c7032 100644 --- a/docs/operators/google_cloud_output.md +++ b/docs/operators/google_cloud_output.md @@ -16,6 +16,8 @@ The `google_cloud_output` operator will send entries to Google Cloud Logging. | `span_id_field` | | A [field](/docs/types/field.md) for the span_id on the log entry | | `use_compression` | `true` | Whether to compress the log entry payloads with gzip before sending to Google Cloud | | `timeout` | 10s | A [duration](/docs/types/duration.md) indicating how long to wait for the API to respond before timing out | +| `buffer` | | A [buffer](/docs/types/buffer.md) block indicating how to buffer entries before flushing | +| `flusher` | | A [flusher](/docs/types/flusher.md) block configuring flushing behavior | If both `credentials` and `credentials_file` are left empty, the agent will attempt to find [Application Default Credentials](https://cloud.google.com/docs/authentication/production) from the environment. @@ -30,3 +32,17 @@ Configuration: project_id: sample_project credentials_file: /tmp/credentials.json ``` + +#### Configuration with non-default buffer and flusher params + +Configuration: +```yaml +- type: google_cloud_output + project_id: sample_project + credentials_file: /tmp/credentials.json + buffer: + type: disk + path: /tmp/stanza_buffer + flusher: + max_concurrent: 8 +``` diff --git a/docs/types/buffer.md b/docs/types/buffer.md new file mode 100644 index 000000000..c468a35a3 --- /dev/null +++ b/docs/types/buffer.md @@ -0,0 +1,61 @@ +# Buffers + +Buffers are used to temporarily store log entries until they can be flushed to their final destination. + +There are two types of buffers: `memory` buffers and `disk` buffers. + +## Memory Buffers + +Memory buffers keep log entries in memory until they are flushed, which makes them very fast. However, because +entries are only stored in memory, they will be lost if the agent is shut down uncleanly. If the agent is shut down +cleanly, they will be saved to the agent's database. + +### Memory Buffer Configuration + +Memory buffers are configured by setting the `type` field of the `buffer` block on an output to `memory`. The only other +configurable field is `max_entries`, which is maximum number of entries that will be held in memory before blocking and +waiting for some entries to be flushed. The default value of `max_entries` is `1048576` (2^20). + +Example: +```yaml +- type: google_cloud_output + project_id: my_project_id + buffer: + type: memory + max_entries: 10000 +``` + + +## Disk Buffers + +Disk buffers store all log entries on disk until they have been successfully flushed to their destination. This means +that, even in the case of an unclean shutdown (kill signal or power loss), no entries will be lost. However, this comes at the cost of +some performance. + +By default, a disk buffer can handle roughly 10,000 logs per second. This number is highly subject to the specs of the +machine running the agent, so if exact numbers are important, we'd advise running your own tests. + +If you'd like better performance and power loss is not a concern, disabling sync writes improves performance to +(roughly) 100,000 entries per second. This comes at the tradeoff that, if there is a power failure, there may +be logs that are lost or a corruption of the database. + +### Disk Buffer Configuration + +Disk buffers are configured by setting the `type` field of the `buffer` block on an output to `disk`. Other fields are described below: + +| Field | Default | Description | +| --- | --- | --- | +| `max_size` | `4294967296` (4GiB) | The maximum size of the disk buffer file in bytes | +| `path` | required | The path to the directory which will contain the disk buffer data | +| `sync` | `true` | Whether to open the database files with the O_SYNC flag. Disabling this improves performance, but relaxes guarantees about log delivery. | + +Example: +```yaml +- type: google_cloud_output + project_id: my_project_id + buffer: + type: disk + max_size: 10000000 # 10MB + path: /tmp/stanza_buffer + sync: true +``` diff --git a/docs/types/flusher.md b/docs/types/flusher.md new file mode 100644 index 000000000..d874cc982 --- /dev/null +++ b/docs/types/flusher.md @@ -0,0 +1,23 @@ +# Flushers + +Flushers handle reading entries from buffers in chunks, flushing them to their final destination, and retrying on failure. + +In most cases, the default options will work well, but they may be need tuning for optimal performance or for reducing load +on the destination API. + +For example, if you hit an API limit on the number of requests per second, consider decreasing `max_concurrent` and +increasing `max_chunk_entries`. This will make fewer, larger requests which should increase efficiency at the cost of +some latency. + +Or, if you have low load and don't care about the higher latency, consider increasing `max_wait` so that entries are sent +less often in larger requests. + +## Flusher configuration + +Flushers are configured with the `flusher` block on output plugins. + +| Field | Default | Description | +| --- | --- | --- | +| `max_concurrent` | `16` | The maximum number of goroutines flushing entries concurrently | +| `max_wait` | 1s | The maximum amount of time to wait for a chunk to fill before flushing it. Higher values can reduce load, but also increase delivery latency. | +| `max_chunk_entries` | 1000 | The maximum number of entries to flush in a single chunk. | diff --git a/entry/field.go b/entry/field.go index 664f96856..b0ed20242 100644 --- a/entry/field.go +++ b/entry/field.go @@ -5,6 +5,12 @@ import ( "fmt" ) +const ( + labelsPrefix = "$labels" + resourcePrefix = "$resource" + recordPrefix = "$record" +) + // Field represents a potential field on an entry. // It is used to get, set, and delete values at this field. // It is deserialized from JSON dot notation. @@ -49,17 +55,17 @@ func fieldFromString(s string) (Field, error) { } switch split[0] { - case "$labels": + case labelsPrefix: if len(split) != 2 { return Field{}, fmt.Errorf("labels cannot be nested") } return Field{LabelField{split[1]}}, nil - case "$resource": + case resourcePrefix: if len(split) != 2 { return Field{}, fmt.Errorf("resource fields cannot be nested") } return Field{ResourceField{split[1]}}, nil - case "$record", "$": + case recordPrefix, "$": return Field{RecordField{split[1:]}}, nil default: return Field{RecordField{split}}, nil @@ -127,12 +133,13 @@ func splitField(s string) ([]string, error) { } state = OUT_BRACKET case OUT_BRACKET: - if c == '.' { + switch c { + case '.': state = IN_UNBRACKETED_TOKEN tokenStart = i + 1 - } else if c == '[' { + case '[': state = IN_BRACKET - } else { + default: return nil, fmt.Errorf("bracketed access must be followed by a dot or another bracketed access") } case IN_UNBRACKETED_TOKEN: diff --git a/entry/label_field_test.go b/entry/label_field_test.go index 3bd5d2dc0..23d212c3f 100644 --- a/entry/label_field_test.go +++ b/entry/label_field_test.go @@ -50,7 +50,6 @@ func TestLabelFieldGet(t *testing.T) { require.Equal(t, tc.expected, val) }) } - } func TestLabelFieldDelete(t *testing.T) { @@ -103,7 +102,6 @@ func TestLabelFieldDelete(t *testing.T) { require.Equal(t, tc.expected, val) }) } - } func TestLabelFieldSet(t *testing.T) { @@ -172,7 +170,6 @@ func TestLabelFieldSet(t *testing.T) { require.Equal(t, tc.expected, entry.Labels) }) } - } func TestLabelFieldString(t *testing.T) { @@ -198,5 +195,4 @@ func TestLabelFieldString(t *testing.T) { require.Equal(t, tc.expected, tc.field.String()) }) } - } diff --git a/entry/record_field.go b/entry/record_field.go index 9834e1b8b..e952ac6a1 100644 --- a/entry/record_field.go +++ b/entry/record_field.go @@ -197,7 +197,7 @@ func (f RecordField) MarshalYAML() (interface{}, error) { func fromJSONDot(value string) RecordField { keys := strings.Split(value, ".") - if keys[0] == "$" || keys[0] == "$record" { + if keys[0] == "$" || keys[0] == recordPrefix { keys = keys[1:] } @@ -207,7 +207,7 @@ func fromJSONDot(value string) RecordField { // toJSONDot returns the JSON dot notation for a field. func toJSONDot(field RecordField) string { if field.isRoot() { - return "$record" + return recordPrefix } containsDots := false @@ -219,7 +219,7 @@ func toJSONDot(field RecordField) string { var b strings.Builder if containsDots { - b.WriteString("$record") + b.WriteString(recordPrefix) for _, key := range field.Keys { b.WriteString(`['`) b.WriteString(key) @@ -232,7 +232,6 @@ func toJSONDot(field RecordField) string { } b.WriteString(key) } - } return b.String() diff --git a/entry/record_field_test.go b/entry/record_field_test.go index 45b31ef7d..dec7714d2 100644 --- a/entry/record_field_test.go +++ b/entry/record_field_test.go @@ -250,7 +250,7 @@ func TestRecordFieldSet(t *testing.T) { t.Run(tc.name, func(t *testing.T) { entry := New() entry.Record = tc.record - entry.Set(tc.field, tc.setTo) + require.NoError(t, entry.Set(tc.field, tc.setTo)) assert.Equal(t, tc.expectedVal, entry.Record) }) } diff --git a/entry/resource_field_test.go b/entry/resource_field_test.go index 0605cf946..d4bcdc9d9 100644 --- a/entry/resource_field_test.go +++ b/entry/resource_field_test.go @@ -50,7 +50,6 @@ func TestResourceFieldGet(t *testing.T) { require.Equal(t, tc.expected, val) }) } - } func TestResourceFieldDelete(t *testing.T) { @@ -103,7 +102,6 @@ func TestResourceFieldDelete(t *testing.T) { require.Equal(t, tc.expected, val) }) } - } func TestResourceFieldSet(t *testing.T) { @@ -172,7 +170,6 @@ func TestResourceFieldSet(t *testing.T) { require.Equal(t, tc.expected, entry.Resource) }) } - } func TestResourceFieldString(t *testing.T) { @@ -198,5 +195,4 @@ func TestResourceFieldString(t *testing.T) { require.Equal(t, tc.expected, tc.field.String()) }) } - } diff --git a/go.mod b/go.mod index 2fc976573..0cd3f7533 100644 --- a/go.mod +++ b/go.mod @@ -3,36 +3,40 @@ module github.com/observiq/stanza go 1.14 require ( + cloud.google.com/go v0.46.3 // indirect cloud.google.com/go/logging v1.0.0 github.com/antonmedv/expr v1.8.2 github.com/cenkalti/backoff/v4 v4.0.2 github.com/elastic/go-elasticsearch/v7 v7.7.0 github.com/golang/protobuf v1.4.2 - github.com/golangci/golangci-lint v1.30.0 // indirect - github.com/googleapis/gax-go v1.0.3 + github.com/google/go-cmp v0.5.0 // indirect github.com/hashicorp/go-uuid v1.0.2 - github.com/influxdata/go-syslog/v3 v3.0.0 // indirect github.com/json-iterator/go v1.1.9 github.com/kardianos/service v1.0.0 - github.com/kr/pretty v0.2.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/observiq/ctimefmt v1.0.0 github.com/observiq/go-syslog/v3 v3.0.2 + github.com/onsi/ginkgo v1.13.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/spf13/cobra v1.0.0 github.com/stretchr/testify v1.6.1 - github.com/vektra/mockery v1.1.2 // indirect go.etcd.io/bbolt v1.3.4 go.uber.org/zap v1.15.0 + golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect - golang.org/x/net v0.0.0-20200625001655-4c5254603344 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 + golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae golang.org/x/text v0.3.3 + golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305 // indirect gonum.org/v1/gonum v0.6.2 google.golang.org/api v0.20.0 google.golang.org/genproto v0.0.0-20200304201815-d429ff31ee6c google.golang.org/grpc v1.27.1 + gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.3.0 + honnef.co/go/tools v0.0.1-2020.1.4 // indirect k8s.io/api v0.18.6 k8s.io/apimachinery v0.18.6 k8s.io/client-go v0.18.6 diff --git a/go.sum b/go.sum index 85a1c5662..4d1f4621d 100644 --- a/go.sum +++ b/go.sum @@ -10,11 +10,9 @@ cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/logging v1.0.0 h1:kaunpnoEh9L4hu6JUsBa8Y20LBfKnCuDhKUgdZp7oK8= cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= @@ -27,55 +25,36 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5 h1:XTrzB+F8+SpRmbhAH8HLxhiiG6nYNwaBZjrFps1oWEk= -github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Mottl/ctimefmt v0.0.0-20190803144728-fd2ac23a585a/go.mod h1:eyj2WSIdoPMPs2eNTLpSmM6Nzqo4V80/d6jHpnJ1SAI= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= -github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antonmedv/expr v1.8.2 h1:BfkVHGudYqq7jp3Ji33kTn+qZ9D19t/Mndg0ag/Ycq4= github.com/antonmedv/expr v1.8.2/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/bombsimon/wsl/v3 v3.1.0 h1:E5SRssoBgtVFPcYWUOFJEcgaySgdtTNYzsSKDOY7ss8= -github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/daixiang0/gci v0.0.0-20200727065011-66f1df783cb2 h1:3Lhhps85OdA8ezsEKu+IA1hE+DBTjt/fjd7xNCrHbVA= -github.com/daixiang0/gci v0.0.0-20200727065011-66f1df783cb2/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denis-tingajkin/go-header v0.3.1 h1:ymEpSiFjeItCy1FOP+x0M2KdCELdEAHUsNa8F+hHc6w= -github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5na0fBt4/CRe7Lrji0= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -87,9 +66,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -99,48 +75,16 @@ github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-critic/go-critic v0.5.0 h1:Ic2p5UCl5fX/2WX2w8nroPpPhxRNsNTMlJzsu/uqwnM= -github.com/go-critic/go-critic v0.5.0/go.mod h1:4jeRh3ZAVnRYhuWdOEvwzVqLUpxMSoAT0xZ74JsTPlo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= -github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= -github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= -github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= -github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ= -github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= -github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k= -github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= -github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= -github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= -github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg= -github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= -github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= -github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= -github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4= -github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= -github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= -github.com/go-toolsmith/typep v1.0.2 h1:8xdsa1+FSIH/RhEkgnD1j2CJOy5mNllW1Q9tRiYwvlk= -github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= -github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b h1:khEcpUM4yFcxg4/FHQWkvVRmgijNXRfzkIDHh23ggEo= -github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc= -github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= @@ -150,7 +94,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -159,8 +102,6 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -168,36 +109,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= -github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= -github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= -github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= -github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w= -github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= -github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw= -github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= -github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8= -github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= -github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0= -github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks= -github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.30.0 h1:UhdK5WbO0GBd7W+k2lOD7BEJH4Wsa7zKfw8m3/aEJGQ= -github.com/golangci/golangci-lint v1.30.0/go.mod h1:5t0i3wHlqQc9deBBvZsP+a/4xz7cfjV+zhp5U0Mzp14= -github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= -github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= -github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= -github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= -github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= -github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= -github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk= -github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= -github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us= -github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= -github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg= -github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= -github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= -github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -214,73 +125,36 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go v1.0.3 h1:9dMLqhaibYONnDRcnHdUs9P8Mw64jLlZTYlDe3leBtQ= -github.com/googleapis/gax-go v1.0.3/go.mod h1:QyXYajJFdARxGzjwUfbDFIse7Spkw81SJ4LrBJXtlQ8= -github.com/googleapis/gax-go/v2 v2.0.2/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= -github.com/gostaticanalysis/analysisutil v0.0.3 h1:iwp+5/UAyzQSFgQ4uR2sni99sJ8Eo9DEacKWM5pekIg= -github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/go-syslog v1.0.1 h1:a/ARpnCDr/sX/hVH7dyQVi+COXlEzM4bNIoolOfw99Y= -github.com/influxdata/go-syslog/v3 v3.0.0 h1:jichmjSZlYK0VMmlz+k4WeOQd7z745YLsvGMqwtYt4I= -github.com/influxdata/go-syslog/v3 v3.0.0/go.mod h1:tulsOp+CecTAYC27u9miMgq21GqXRW6VdKbOG+QSP4Q= -github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:GmsqmapfzSJkm28dhRoHz2tLRbJmqhU86IPgBtN3mmk= -github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= -github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4= -github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kardianos/service v1.0.0 h1:HgQS3mFfOlyntWX8Oke98JcJLqt1DBcHR4kxShpYef0= @@ -289,58 +163,24 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kyoh86/exportloopref v0.1.7 h1:u+iHuTbkbTS2D/JP7fCuZDo/t3rBVGo3Hf58Rc+lQVY= -github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/maratori/testpackage v1.0.1 h1:QtJ5ZjqapShm0w5DosRjg0PRlSdAdlx+W6cCKoALdbQ= -github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= -github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE= -github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= -github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= @@ -351,23 +191,15 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaPw= -github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c= -github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= -github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nishanths/exhaustive v0.0.0-20200708172631-8866003e3856 h1:W3KBC2LFyfgd+wNudlfgCCsTo4q97MeNWrfz8/wSdSc= -github.com/nishanths/exhaustive v0.0.0-20200708172631-8866003e3856/go.mod h1:wBEpHwM2OdmeNpdCvRPUlkEbBuaFmcK4Wv8Q7FuGW3c= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/observiq/ctimefmt v1.0.0 h1:r7vTJ+Slkrt9fZ67mkf+mA6zAdR5nGIJRMTzkUyvilk= github.com/observiq/ctimefmt v1.0.0/go.mod h1:mxi62//WbSpG/roCO1c6MqZ7zQTvjVtYheqHN3eOjvc= -github.com/observiq/go-syslog/v3 v3.0.1 h1:lillWZkfRqDAg1UvzDc6UeEsoLuT3fFnq+6FOf9UHW4= -github.com/observiq/go-syslog/v3 v3.0.1/go.mod h1:9abcumkQwDUY0VgWdH6CaaJ3Ks39A7NvIelMlavPru0= github.com/observiq/go-syslog/v3 v3.0.2 h1:vaeINFErM/E3cKE2Ot1FAhhGq5mv7uGBOzjnGL3qhbY= github.com/observiq/go-syslog/v3 v3.0.2/go.mod h1:9abcumkQwDUY0VgWdH6CaaJ3Ks39A7NvIelMlavPru0= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -375,18 +207,16 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0 h1:M76yO2HkZASFjXL0HSoZJ1AYEmQxNJmY41Jx1zNUq1Y= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA= -github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -394,7 +224,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -405,44 +234,16 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= -github.com/quasilyte/go-ruleguard v0.1.2-0.20200318202121-b00d7a75d3d8 h1:DvnesvLtRPQOvaUbfXfh0tpMHg29by0H7F2U+QIkSu8= -github.com/quasilyte/go-ruleguard v0.1.2-0.20200318202121-b00d7a75d3d8/go.mod h1:CGFX09Ci3pq9QZdj86B+VGIdNj4VyCo2iPOGS9esB/k= -github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY= -github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryancurrah/gomodguard v1.1.0 h1:DWbye9KyMgytn8uYpuHkwf0RHqAYO6Ay/D0TbCpPtVU= -github.com/ryancurrah/gomodguard v1.1.0/go.mod h1:4O8tr7hBODaGE6VIhfJDHcwzh5GUccKSJBU0UMXJFVM= -github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8OUZI9xFw= -github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/securego/gosec/v2 v2.4.0 h1:ivAoWcY5DMs9n04Abc1VkqZBO0FL0h4ShTcVsC53lCE= -github.com/securego/gosec/v2 v2.4.0/go.mod h1:0/Q4cjmlFDfDUj1+Fib61sc+U5IQb2w+Iv9/C3wPVko= -github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU= -github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= -github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY= -github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI= -github.com/sourcegraph/go-diff v0.5.3 h1:lhIKJ2nXLZZ+AfbHpYxTn0pXpNTTui0DX7DO3xeb1Zs= -github.com/sourcegraph/go-diff v0.5.3/go.mod h1:v9JDtjCE4HHHCZGId75rg8gkKKa98RVjBcBGsVmMmak= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= @@ -458,10 +259,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/ssgreg/nlreturn/v2 v2.0.1 h1:+lm6xFjVuNw/9t/Fh5sIwfNWefiD5bddzc6vwJ1TvRI= -github.com/ssgreg/nlreturn/v2 v2.0.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -473,34 +270,10 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2 h1:Xr9gkxfOP0KQWXKNqmwe8vEeSUiUj4Rlee9CMVX2ZUQ= -github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= -github.com/tetafro/godot v0.4.8 h1:h61+hQraWhdI6WYqMwAwZYCE5yxL6a9/Orw4REbabSU= -github.com/tetafro/godot v0.4.8/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= -github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q= -github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa h1:RC4maTWLKKwb7p1cnoygsbKIgNlJqSYBeAFON3Ar8As= -github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= -github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= -github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg= -github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= -github.com/uudashr/gocognit v1.0.1 h1:MoG2fZ0b/Eo7NXoIwCVFLG5JED3qgQz5/NEE+rOsjPs= -github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.12.0/go.mod h1:229t1eWu9UXTPmoUkbpN/fctKPBY4IJoFXQnxHGXy6E= -github.com/valyala/quicktemplate v1.5.1/go.mod h1:v7yYWpBEiutDyNfVaph6oC/yKwejzVyTX/2cwwHxyok= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vektra/mockery v1.1.2 h1:uc0Yn67rJpjt8U/mAZimdCKn9AeA97BOkjpmtBSlfP4= -github.com/vektra/mockery v1.1.2/go.mod h1:VcfZjKaFOPO+MpN4ZvwPjs4c48lkq1o3Ym8yHZJu0jU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= @@ -520,7 +293,6 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -535,7 +307,6 @@ golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190221220918-438050ddec5e/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522 h1:OeRHuibLsmZkFj773W4LcfAGsSxJgfPONhr8cmO+eLA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -545,7 +316,6 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -560,17 +330,13 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -582,10 +348,6 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -598,25 +360,19 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -625,15 +381,10 @@ golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200513112337-417ce2331b5c h1:kISX68E8gSkNYAFRFiDU8rl5RIn1sJYKYb/r2vMLDrU= -golang.org/x/sys v0.0.0-20200513112337-417ce2331b5c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -649,53 +400,29 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZe golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200321224714-0d839f3cf2ed/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97 h1:DAuln/hGp+aJiHpID1Y1hYzMEPP5WLwtZHPb50mN0OE= -golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200519015757-0d0afa43d58a/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200701041122-1837592efa10/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305 h1:yaM5S0KcY0lIoZo7Fl+oi91b/DdlU2zuWpfHrpWbCS0= golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -712,7 +439,6 @@ google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEt google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -730,10 +456,8 @@ google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200304201815-d429ff31ee6c h1:Mm69MraVZ+yh1vw8pQOUW4uJkkSEQbbTr076A94lvqs= google.golang.org/genproto v0.0.0-20200304201815-d429ff31ee6c/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -752,17 +476,14 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -773,31 +494,19 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.18.4 h1:8x49nBRxuXGUlDlwlWd3RMY1SayZrzFfxea3UZSkFw4= -k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4= k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE= k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= -k8s.io/api v0.19.0 h1:XyrFIJqTYZJ2DU7FBE/bSPz7b1HvbVBuBf07oeo6eTc= -k8s.io/apimachinery v0.18.4 h1:ST2beySjhqwJoIFk6p7Hp5v5O0hYY6Gngq/gUYXTPIA= -k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/apimachinery v0.18.6 h1:RtFHnfGNfd1N0LeSrKCUznz5xtUP1elRGvHJbL3Ntag= k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/client-go v0.18.4 h1:un55V1Q/B3JO3A76eS0kUSywgGK/WR3BQ8fHQjNa6Zc= -k8s.io/client-go v0.18.4/go.mod h1:f5sXwL4yAZRkAtzOxRWUhA/N8XzGCb+nPZI8PfobZ9g= k8s.io/client-go v0.18.6 h1:I+oWqJbibLSGsZj8Xs8F0aWVXJVIoUHWaaJV3kUN/Zw= k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= -k8s.io/client-go v1.5.1 h1:XaX/lo2/u3/pmFau8HN+sB5C/b4dc4Dmm2eXjBH4p1E= -k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -807,14 +516,6 @@ k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOEC k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -mvdan.cc/gofumpt v0.0.0-20200709182408-4fd085cb6d5f h1:gi7cb8HTDZ6q8VqsUpkdoFi3vxwHMneQ6+Q5Ap5hjPE= -mvdan.cc/gofumpt v0.0.0-20200709182408-4fd085cb6d5f/go.mod h1:9VQ397fNXEnF84t90W4r4TRCQK+pg9f8ugVfyj+S26w= -mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= -mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= -mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= -mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= -mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= -mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= @@ -823,5 +524,3 @@ sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnM sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/operator/buffer/buffer.go b/operator/buffer/buffer.go index d4303c0fc..43225bd52 100644 --- a/operator/buffer/buffer.go +++ b/operator/buffer/buffer.go @@ -2,77 +2,61 @@ package buffer import ( "context" + "encoding/json" "fmt" - "time" "github.com/observiq/stanza/entry" - "github.com/observiq/stanza/errors" "github.com/observiq/stanza/operator" ) -// Buffer is an entity that buffers log entries to an operator type Buffer interface { - Flush(context.Context) error - Add(interface{}, int) error - AddWait(context.Context, interface{}, int) error - SetHandler(BundleHandler) - Process(context.Context, *entry.Entry) error + Add(context.Context, *entry.Entry) error + Read([]*entry.Entry) (FlushFunc, int, error) + ReadWait(context.Context, []*entry.Entry) (FlushFunc, int, error) + Close() error +} + +type Config struct { + BufferBuilder } -// NewConfig creates a new buffer config func NewConfig() Config { return Config{ - BufferType: "memory", - DelayThreshold: operator.Duration{Duration: time.Second}, - BundleCountThreshold: 10_000, - BundleByteThreshold: 4 * 1024 * 1024 * 1024, // 4MB - BundleByteLimit: 4 * 1024 * 1024 * 1024, // 4MB - BufferedByteLimit: 500 * 1024 * 1024 * 1024, // 500MB - HandlerLimit: 16, - Retry: NewRetryConfig(), + BufferBuilder: NewMemoryBufferConfig(), } } -// Config is the configuration of a buffer -type Config struct { - BufferType string `json:"type,omitempty" yaml:"type,omitempty"` - DelayThreshold operator.Duration `json:"delay_threshold,omitempty" yaml:"delay_threshold,omitempty"` - BundleCountThreshold int `json:"bundle_count_threshold,omitempty" yaml:"buffer_count_threshold,omitempty"` - BundleByteThreshold int `json:"bundle_byte_threshold,omitempty" yaml:"bundle_byte_threshold,omitempty"` - BundleByteLimit int `json:"bundle_byte_limit,omitempty" yaml:"bundle_byte_limit,omitempty"` - BufferedByteLimit int `json:"buffered_byte_limit,omitempty" yaml:"buffered_byte_limit,omitempty"` - HandlerLimit int `json:"handler_limit,omitempty" yaml:"handler_limit,omitempty"` - Retry RetryConfig `json:"retry,omitempty" yaml:"retry,omitempty"` +type BufferBuilder interface { + Build(context operator.BuildContext, pluginID string) (Buffer, error) } -// Build will build a buffer from the supplied configuration -func (config *Config) Build() (Buffer, error) { - switch config.BufferType { - case "memory", "": - return NewMemoryBuffer(config), nil - default: - return nil, errors.NewError( - fmt.Sprintf("Invalid buffer type %s", config.BufferType), - "The only supported buffer type is 'memory'", - ) - } +func (bc *Config) UnmarshalJSON(data []byte) error { + return bc.unmarshal(func(dst interface{}) error { + return json.Unmarshal(data, dst) + }) } -// NewRetryConfig creates a new retry config -func NewRetryConfig() RetryConfig { - return RetryConfig{ - InitialInterval: operator.Duration{Duration: 500 * time.Millisecond}, - RandomizationFactor: 0.5, - Multiplier: 1.5, - MaxInterval: operator.Duration{Duration: 15 * time.Minute}, - } +func (bc *Config) UnmarshalYAML(f func(interface{}) error) error { + return bc.unmarshal(f) } -// RetryConfig is the configuration of an entity that will retry processing after an error -type RetryConfig struct { - InitialInterval operator.Duration `json:"initial_interval,omitempty" yaml:"initial_interval,omitempty"` - RandomizationFactor float64 `json:"randomization_factor,omitempty" yaml:"randomization_factor,omitempty"` - Multiplier float64 `json:"multiplier,omitempty" yaml:"multiplier,omitempty"` - MaxInterval operator.Duration `json:"max_interval,omitempty" yaml:"max_interval,omitempty"` - MaxElapsedTime operator.Duration `json:"max_elapsed_time,omitempty" yaml:"max_elapsed_time,omitempty"` +func (bc *Config) unmarshal(unmarshal func(interface{}) error) error { + var m map[string]interface{} + err := unmarshal(&m) + if err != nil { + return err + } + + switch m["type"] { + case "memory": + bc.BufferBuilder = NewMemoryBufferConfig() + return unmarshal(bc.BufferBuilder) + case "disk": + bc.BufferBuilder = NewDiskBufferConfig() + return unmarshal(bc.BufferBuilder) + default: + return fmt.Errorf("unknown buffer type '%s'", m["type"]) + } } + +type FlushFunc func() error diff --git a/operator/buffer/buffer_test.go b/operator/buffer/buffer_test.go index eb8028888..156282c24 100644 --- a/operator/buffer/buffer_test.go +++ b/operator/buffer/buffer_test.go @@ -1,109 +1,113 @@ package buffer import ( - "context" "encoding/json" - "sync" "testing" - "time" - "github.com/observiq/stanza/entry" - "github.com/observiq/stanza/operator" "github.com/stretchr/testify/require" - "go.uber.org/zap" yaml "gopkg.in/yaml.v2" ) -type bufferHandler struct { - flushed []*entry.Entry - mux sync.Mutex - notify chan struct{} -} - -func (b *bufferHandler) ProcessMulti(ctx context.Context, entries []*entry.Entry) error { - b.mux.Lock() - b.flushed = append(b.flushed, entries...) - b.mux.Unlock() - b.notify <- struct{}{} - return nil -} - -func (b *bufferHandler) Logger() *zap.SugaredLogger { - return nil -} - -func TestBuffer(t *testing.T) { - config := NewConfig() - config.DelayThreshold = operator.Duration{ - Duration: 100 * time.Millisecond, - } - - buf := NewMemoryBuffer(&config) - numEntries := 10000 - - bh := bufferHandler{ - flushed: make([]*entry.Entry, 0), - notify: make(chan struct{}), - } - buf.SetHandler(&bh) - - for i := 0; i < numEntries; i++ { - err := buf.AddWait(context.Background(), entry.New(), 0) - require.NoError(t, err) - } - - for { - select { - case <-bh.notify: - bh.mux.Lock() - if len(bh.flushed) == numEntries { - bh.mux.Unlock() - return - } - bh.mux.Unlock() - case <-time.After(time.Second): - require.FailNow(t, "timed out waiting for all entries to be flushed") - } - } -} - -func TestBufferSerializationRoundtrip(t *testing.T) { +func TestBufferUnmarshalYAML(t *testing.T) { cases := []struct { - name string - config Config + name string + yaml []byte + json []byte + expected Config + expectError bool }{ { - "zeros", - Config{}, + "SimpleMemory", + []byte("type: memory\nmax_entries: 30\n"), + []byte(`{"type": "memory", "max_entries": 30}`), + Config{ + BufferBuilder: &MemoryBufferConfig{ + Type: "memory", + MaxEntries: 30, + }, + }, + false, + }, + { + "SimpleDisk", + []byte("type: disk\nmax_size: 1234\npath: /var/log/testpath\n"), + []byte(`{"type": "disk", "max_size": 1234, "path": "/var/log/testpath"}`), + Config{ + BufferBuilder: &DiskBufferConfig{ + Type: "disk", + MaxSize: 1234, + Path: "/var/log/testpath", + Sync: true, + }, + }, + false, }, { - "defaults", - NewConfig(), + "UnknownType", + []byte("type: invalid\n"), + []byte(`{"type": "invalid"}`), + Config{ + BufferBuilder: &DiskBufferConfig{ + Type: "disk", + MaxSize: 1234, + Path: "/var/log/testpath", + Sync: true, + }, + }, + true, + }, + { + "InvalidType", + []byte("type: !!float 123\n"), + []byte(`{"type": 12}`), + Config{ + BufferBuilder: &DiskBufferConfig{ + Type: "disk", + MaxSize: 1234, + Path: "/var/log/testpath", + Sync: true, + }, + }, + true, }, } for _, tc := range cases { - t.Run("yaml "+tc.name, func(t *testing.T) { - cfgBytes, err := yaml.Marshal(tc.config) - require.NoError(t, err) - - var cfg Config - err = yaml.UnmarshalStrict(cfgBytes, &cfg) - require.NoError(t, err) - - require.Equal(t, tc.config, cfg) - }) - - t.Run("json "+tc.name, func(t *testing.T) { - tc := tc - cfgBytes, err := json.Marshal(tc.config) - require.NoError(t, err) - - var cfg Config - err = json.Unmarshal(cfgBytes, &cfg) - require.NoError(t, err) - - require.Equal(t, tc.config, cfg) + t.Run(tc.name, func(t *testing.T) { + t.Run("YAML", func(t *testing.T) { + var b Config + err := yaml.Unmarshal(tc.yaml, &b) + if tc.expectError { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.expected, b) + }) + + t.Run("JSON", func(t *testing.T) { + var b Config + err := json.Unmarshal(tc.json, &b) + if tc.expectError { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.expected, b) + }) }) } } + +func TestBuffer(t *testing.T) { + t.Run("Default", func(t *testing.T) { + cfg := NewConfig() + expected := Config{ + BufferBuilder: &MemoryBufferConfig{ + Type: "memory", + MaxEntries: 1 << 20, + }, + } + require.Equal(t, expected, cfg) + }) +} diff --git a/operator/buffer/disk.go b/operator/buffer/disk.go new file mode 100644 index 000000000..54945fd68 --- /dev/null +++ b/operator/buffer/disk.go @@ -0,0 +1,596 @@ +package buffer + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + + "github.com/observiq/stanza/entry" + "github.com/observiq/stanza/operator" + "golang.org/x/sync/semaphore" +) + +// DiskBufferConfig is a configuration struct for a DiskBuffer +type DiskBufferConfig struct { + Type string `json:"type" yaml:"type"` + + // MaxSize is the maximum size in bytes of the data file on disk + MaxSize int `json:"max_size" yaml:"max_size"` + + // Path is a path to a directory which contains the data and metadata files + Path string `json:"path" yaml:"path"` + + // Sync indicates whether to open the files with O_SYNC. If this is set to false, + // in cases like power failures or unclean shutdowns, logs may be lost or the + // database may become corrupted. + Sync bool `json:"sync" yaml:"sync"` +} + +// NewDiskBufferConfig creates a new default disk buffer config +func NewDiskBufferConfig() *DiskBufferConfig { + return &DiskBufferConfig{ + Type: "disk", + MaxSize: 1 << 32, // 4GiB + Sync: true, + } +} + +// Build creates a new Buffer from a DiskBufferConfig +func (c DiskBufferConfig) Build(context operator.BuildContext, _ string) (Buffer, error) { + if c.Path == "" { + return nil, fmt.Errorf("missing required field 'path'") + } + b := NewDiskBuffer(c.MaxSize) + if err := b.Open(c.Path, c.Sync); err != nil { + return nil, err + } + return b, nil +} + +// DiskBuffer is a buffer for storing entries on disk until they are flushed to their +// final destination. +type DiskBuffer struct { + // Metadata holds information about the current state of the buffered entries + metadata *Metadata + + // Data is the file that stores the buffered entries + data *os.File + sync.Mutex + + // atEnd indicates whether the data file descriptor is currently seeked to the + // end. This helps save us seek calls. + atEnd bool + + // entryAdded is a channel that is notified on every time an entry is added. + // The integer sent down the channel is the new number of unread entries stored. + // Readers using ReadWait will listen on this channel, and wait to read until + // there are enough entries to fill its buffer. + entryAdded chan int64 + + maxBytes int64 + flushedBytes int64 + lastCompaction time.Time + + // readerLock ensures that there is only ever one reader listening to the + // entryAdded channel at a time. + readerLock sync.Mutex + + // diskSizeSemaphore is a semaphore that allows us to block once we've hit + // the max disk size. + diskSizeSemaphore *semaphore.Weighted + + // copyBuffer is a pre-allocated byte slice that is used during compaction + copyBuffer []byte +} + +// NewDiskBuffer creates a new DiskBuffer +func NewDiskBuffer(maxDiskSize int) *DiskBuffer { + return &DiskBuffer{ + maxBytes: int64(maxDiskSize), + entryAdded: make(chan int64, 1), + copyBuffer: make([]byte, 1<<16), + diskSizeSemaphore: semaphore.NewWeighted(int64(maxDiskSize)), + } +} + +// Open opens the disk buffer files from a database directory +func (d *DiskBuffer) Open(path string, sync bool) error { + var err error + dataPath := filepath.Join(path, "data") + flags := os.O_CREATE | os.O_RDWR + if sync { + flags |= os.O_SYNC + } + if d.data, err = os.OpenFile(dataPath, flags, 0755); err != nil { + return err + } + + metadataPath := filepath.Join(path, "metadata") + if d.metadata, err = OpenMetadata(metadataPath, sync); err != nil { + return err + } + + info, err := d.data.Stat() + if err != nil { + return err + } + + if ok := d.diskSizeSemaphore.TryAcquire(info.Size()); !ok { + return fmt.Errorf("current on-disk size is larger than max size") + } + + // First, if there is a dead range from a previous incomplete compaction, delete it + if err = d.deleteDeadRange(); err != nil { + return err + } + + // Compact on open + if err = d.Compact(); err != nil { + return err + } + + // Once everything is compacted, we can safely reset all previously read, but + // unflushed entries to unread + d.metadata.unreadStartOffset = 0 + d.addUnreadCount(int64(len(d.metadata.read))) + d.metadata.read = d.metadata.read[:0] + return d.metadata.Sync() +} + +// Close flushes the current metadata to disk, then closes the underlying files +func (d *DiskBuffer) Close() error { + d.Lock() + defer d.Unlock() + + if err := d.metadata.Close(); err != nil { + return err + } + return d.data.Close() +} + +// Add adds an entry to the buffer, blocking until it is either added or the context +// is cancelled. +func (d *DiskBuffer) Add(ctx context.Context, newEntry *entry.Entry) error { + var buf bytes.Buffer + var err error + enc := json.NewEncoder(&buf) + if err := enc.Encode(newEntry); err != nil { + return err + } + + if err = d.diskSizeSemaphore.Acquire(ctx, int64(buf.Len())); err != nil { + return err + } + + d.Lock() + defer d.Unlock() + + // Seek to end of the file if we're not there + if err = d.seekToEnd(); err != nil { + return err + } + + if _, err = d.data.Write(buf.Bytes()); err != nil { + return err + } + + d.addUnreadCount(1) + + return nil +} + +// addUnreadCount adds i to the unread count and notifies any callers of +// ReadWait that an entry has been added. The disk buffer lock must be held when +// calling this. +func (d *DiskBuffer) addUnreadCount(i int64) { + d.metadata.unreadCount += i + + // Notify a reader that new entries have been added by either + // sending on the channel, or updating the value in the channel + select { + case <-d.entryAdded: + d.entryAdded <- d.metadata.unreadCount + case d.entryAdded <- d.metadata.unreadCount: + } +} + +// ReadWait reads entries from the buffer, waiting until either there are enough entries in the +// buffer to fill dst or the context is cancelled. This amortizes the cost of reading from the +// disk. It returns a function that, when called, marks the read entries as flushed, the +// number of entries read, and an error. +func (d *DiskBuffer) ReadWait(ctx context.Context, dst []*entry.Entry) (FlushFunc, int, error) { + d.readerLock.Lock() + defer d.readerLock.Unlock() + + // Wait until the timeout is hit, or there are enough unread entries to fill the destination buffer +LOOP: + for { + select { + case n := <-d.entryAdded: + if n >= int64(len(dst)) { + break LOOP + } + case <-ctx.Done(): + break LOOP + } + } + + return d.Read(dst) +} + +// Read copies entries from the disk into the destination buffer. It returns a function that, +// when called, marks the entries as flushed, the number of entries read, and an error. +func (d *DiskBuffer) Read(dst []*entry.Entry) (f FlushFunc, i int, err error) { + d.Lock() + defer d.Unlock() + + // Return fast if there are no unread entries + if d.metadata.unreadCount == 0 { + return d.newFlushFunc(nil), 0, nil + } + + // Seek to the start of the range of unread entries + if err = d.seekToUnread(); err != nil { + return nil, 0, fmt.Errorf("seek to unread: %s", err) + } + + readCount := min(len(dst), int(d.metadata.unreadCount)) + newRead := make([]*readEntry, readCount) + + dec := json.NewDecoder(d.data) + startOffset := d.metadata.unreadStartOffset + for i := 0; i < readCount; i++ { + // Decode an entry from the file + var entry entry.Entry + err := dec.Decode(&entry) + if err != nil { + return nil, 0, fmt.Errorf("decode: %s", err) + } + dst[i] = &entry + + // Calculate the end offset of the entry. We add one because + // the decoder doesn't consume the newline at the end of every entry + endOffset := d.metadata.unreadStartOffset + dec.InputOffset() + 1 + newRead[i] = &readEntry{ + startOffset: startOffset, + length: endOffset - startOffset, + } + + // The start offset of the next entry is the end offset of the current + startOffset = endOffset + } + + // Set the offset for the next unread entry + d.metadata.unreadStartOffset = startOffset + + // Keep track of the newly read entries + d.metadata.read = append(d.metadata.read, newRead...) + + // Remove the read entries from the unread count + d.addUnreadCount(-int64(readCount)) + + return d.newFlushFunc(newRead), readCount, nil +} + +// newFlushFunc returns a function that marks read entries as flushed +func (d *DiskBuffer) newFlushFunc(newRead []*readEntry) FlushFunc { + return func() error { + d.Lock() + for _, entry := range newRead { + entry.flushed = true + d.flushedBytes += entry.length + } + d.Unlock() + return d.checkCompact() + } +} + +// checkCompact checks if a compaction should be performed, then kicks one off +func (d *DiskBuffer) checkCompact() error { + d.Lock() + switch { + case d.flushedBytes > d.maxBytes/2: + fallthrough + case time.Since(d.lastCompaction) > 5*time.Second: + d.Unlock() + if err := d.Compact(); err != nil { + return err + } + default: + d.Unlock() + } + return nil +} + +// Compact removes all flushed entries from disk +func (d *DiskBuffer) Compact() error { + d.Lock() + defer d.Unlock() + defer func() { d.lastCompaction = time.Now() }() + + // So how does this work? The goal here is to remove all flushed entries from disk, + // freeing up space for new entries. We do this by going through each entry that has + // been read and checking if it has been flushed. If it has, we know that space on + // disk is re-claimable, so we can move unflushed entries into its place. + // + // The tricky part is that we can't overwrite any data until we've both safely copied it + // to its new location and written a copy of the metadata that describes where that data + // is located on disk. This ensures that, if our process is killed mid-compaction, we will + // always have a complete, uncorrupted database. + // + // We do this by maintaining a "dead range" during compation. The dead range is + // effectively a range of bytes that can safely be deleted from disk just by shifting + // everything that comes after it backwards in the file. Then, when we open the disk + // buffer, the first thing we do is delete the dead range if it exists. + // + // To clear out flushed entries, we iterate over all the entries that have been read, + // finding ranges of either flushed or unflushed entries. If we have a range of flushed + // entries, we can expand the dead range to include the space those entries took on disk. + // If we find a range of unflushed entries, we move them to the beginning of the dead range + // and advance the start of the dead range to the end of the copied bytes. + // + // Once we iterate through all the read entries, we should be left with a dead range + // that's located right before the start of the unread entries. Since we know none of the + // unread entries need be flushed, we can simply bubble the dead range through the unread + // entries, then truncate the dead range from the end of the file once we're done. + // + // The most important part here is to sync the metadata to disk before overwriting any + // data. That way, at startup, we know where the dead zone is in the file so we can + // safely delete it without deleting any live data. + // + // Example: + // (f = flushed byte, r = read byte, u = unread byte, lowercase = dead range) + // + // FFFFRRRRFFRRRRRRRUUUUUUUUU // start of compaction + // ffffRRRRFFRRRRRRRUUUUUUUUU // mark the first flushed range as unread + // RRRRrrrrFFRRRRRRRUUUUUUUUU // move the read range to the beginning of the dead range + // RRRRrrrrffRRRRRRRUUUUUUUUU // expand the dead range to include the flushed range + // RRRRRRRRRRrrrrrrRUUUUUUUUU // move the portion of the next read range that fits into the dead range + // RRRRRRRRRRRrrrrrrUUUUUUUUU // move the remainder of the read range to start of the dead range + // RRRRRRRRRRRUUUUUUuuuuuuUUU // move the unread entries that fit into the dead range + // RRRRRRRRRRRUUUUUUUUUuuuuuu // move the remainder of the unread entries into the dead range + // RRRRRRRRRRRUUUUUUUUU // truncate the file to remove the dead range + + m := d.metadata + if m.deadRangeLength != 0 { + return fmt.Errorf("cannot compact the disk buffer before removing the dead range") + } + + for start := 0; start < len(m.read); { + if m.read[start].flushed { + // Find the end index of the range of flushed entries + end := start + 1 + for ; end < len(m.read); end++ { + if !m.read[end].flushed { + break + } + } + + // Expand the dead range + if err := d.markFlushedRangeDead(start, end); err != nil { + return err + } + } else { + // Find the end index of the range of unflushed entries + end := start + 1 + for ; end < len(m.read); end++ { + if m.read[end].flushed { + break + } + } + + // Slide the unread range left, to the start of the dead range + if err := d.shiftUnreadRange(start, end); err != nil { + return err + } + + // Update i + start = end + } + } + + // Bubble the dead space through the unflushed entries, then truncate + return d.deleteDeadRange() +} + +func (d *DiskBuffer) markFlushedRangeDead(start, end int) error { + m := d.metadata + // Expand the dead range + rangeSize := onDiskSize(m.read[start:end]) + m.deadRangeLength += rangeSize + d.flushedBytes -= rangeSize + + // Update the effective offsets if the dead range is removed + for _, entry := range m.read[end:] { + entry.startOffset -= rangeSize + } + + // Update the effective unreadStartOffset if the dead range is removed + m.unreadStartOffset -= rangeSize + + // Delete the flushed range from metadata + m.read = append(m.read[:start], m.read[end:]...) + + // Sync to disk + return d.metadata.Sync() +} + +func (d *DiskBuffer) shiftUnreadRange(start, end int) error { + m := d.metadata + // If there is no dead range, no need to move unflushed entries + rangeSize := onDiskSize(m.read[start:end]) + if m.deadRangeLength == 0 { + m.deadRangeStart += rangeSize + return nil + } + + // Slide the range left, syncing dead range after every chunk + for bytesMoved := 0; bytesMoved < int(rangeSize); { + remainingBytes := int(rangeSize) - bytesMoved + chunkSize := min(int(m.deadRangeLength), remainingBytes) + + // Move the chunk to the beginning of the dead space + _, err := d.moveRange( + m.deadRangeStart, + m.deadRangeLength, + m.read[start].startOffset+int64(bytesMoved)+m.deadRangeLength, + int64(chunkSize), + ) + if err != nil { + return err + } + + // Update the offset of the dead space + m.deadRangeStart += int64(chunkSize) + bytesMoved += chunkSize + + // Sync to disk all at once + if err = d.metadata.Sync(); err != nil { + return err + } + } + + return nil +} + +// deleteDeadRange moves the dead range to the end of the file, chunk by chunk, +// so that if it is interrupted, it can just be continued at next startup. +func (d *DiskBuffer) deleteDeadRange() error { + // Exit fast if there is no dead range + if d.metadata.deadRangeLength == 0 { + if d.metadata.deadRangeStart != 0 { + return d.metadata.setDeadRange(0, 0) + } + return nil + } + + for { + // Replace the range with the proceeding range of bytes + start := d.metadata.deadRangeStart + length := d.metadata.deadRangeLength + n, err := d.moveRange( + start, + length, + start+length, + length, + ) + if err != nil { + return err + } + + // Update the dead range, writing to disk + if err = d.metadata.setDeadRange(start+length, length); err != nil { + return err + } + + if int64(n) < d.metadata.deadRangeLength { + // We're at the end of the file + break + } + } + + info, err := d.data.Stat() + if err != nil { + return err + } + + // Truncate the extra space at the end of the file + if err = d.data.Truncate(info.Size() - d.metadata.deadRangeLength); err != nil { + return err + } + + d.diskSizeSemaphore.Release(d.metadata.deadRangeLength) + + if err = d.metadata.setDeadRange(0, 0); err != nil { + return err + } + + return nil +} + +// moveRange moves from length2 bytes starting from start2 into the space from start1 +// to start1+length1 +func (d *DiskBuffer) moveRange(start1, length1, start2, length2 int64) (int, error) { + if length2 > length1 { + return 0, fmt.Errorf("cannot move a range into a space smaller than itself") + } + + readPosition := start2 + writePosition := start1 + bytesRead := 0 + + rd := io.LimitReader(d.data, length2) + + eof := false + for !eof { + // Seek to last read position + if err := d.seekTo(readPosition); err != nil { + return 0, err + } + + // Read a chunk + n, err := rd.Read(d.copyBuffer) + if err != nil { + if err != io.EOF { + return 0, err + } + eof = true + } + readPosition += int64(n) + bytesRead += n + + // Write the chunk back into a free region + _, err = d.data.WriteAt(d.copyBuffer[:n], writePosition) + if err != nil { + return 0, err + } + writePosition += int64(n) + } + + return bytesRead, nil +} + +// seekToEnd seeks the data file descriptor to the end of the +// file, but only if we're not already at the end. +func (d *DiskBuffer) seekToEnd() error { + if !d.atEnd { + if _, err := d.data.Seek(0, 2); err != nil { + return err + } + d.atEnd = true + } + return nil +} + +// seekToUnread seeks the data file descriptor to the beginning +// of the first unread message +func (d *DiskBuffer) seekToUnread() error { + _, err := d.data.Seek(d.metadata.unreadStartOffset, 0) + d.atEnd = false + return err +} + +// seekTo seeks the data file descriptor to the given offset. +// This is used rather than (*os.File).Seek() because it also sets +// atEnd correctly. +func (d *DiskBuffer) seekTo(pos int64) error { + // Seek to last read position + _, err := d.data.Seek(pos, 0) + d.atEnd = false + return err +} + +// min returns the minimum of two ints +func min(first, second int) int { + m := first + if second < first { + m = second + } + return m +} diff --git a/operator/buffer/disk_metadata.go b/operator/buffer/disk_metadata.go new file mode 100644 index 000000000..fd9328791 --- /dev/null +++ b/operator/buffer/disk_metadata.go @@ -0,0 +1,260 @@ +package buffer + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" +) + +// Metadata is a representation of the on-disk metadata file. It contains +// information about the layout, location, and flushed status of entries +// stored in the data file +type Metadata struct { + // File is a handle to the on-disk metadata store + // + // The layout of the file is as follows: + // - 8 byte DatabaseVersion as LittleEndian int64 + // - 8 byte DeadRangeStartOffset as LittleEndian int64 + // - 8 byte DeadRangeLength as LittleEndian int64 + // - 8 byte UnreadStartOffset as LittleEndian int64 + // - 8 byte UnreadCount as LittleEndian int64 + // - 8 byte ReadCount as LittleEndian int64 + // - Repeated ReadCount times: + // - 1 byte Flushed bool LittleEndian + // - 8 byte Length as LittleEndian int64 + // - 8 byte StartOffset as LittleEndian int64 + file *os.File + + // read is the collection of entries that have been read + read []*readEntry + + // unreadStartOffset is the offset on disk where the contiguous + // range of unread entries start + unreadStartOffset int64 + + // unreadCount is the number of unread entries on disk + unreadCount int64 + + // deadRangeStart is file offset of the beginning of the dead range. + // The dead range is a range of the file that contains unused information + // and should only exist during a compaction. If this exists on startup, + // it should be removed as part of the startup compaction. + deadRangeStart int64 + + // deadRangeLength is the length of the dead range + deadRangeLength int64 +} + +// OpenMetadata opens and parses the metadata +func OpenMetadata(path string, sync bool) (*Metadata, error) { + m := &Metadata{} + + var err error + flags := os.O_CREATE | os.O_RDWR + if sync { + flags |= os.O_SYNC + } + if m.file, err = os.OpenFile(path, flags, 0755); err != nil { + return &Metadata{}, err + } + + info, err := m.file.Stat() + if err != nil { + return &Metadata{}, err + } + + if info.Size() > 0 { + err = m.UnmarshalBinary(m.file) + if err != nil { + return &Metadata{}, fmt.Errorf("read metadata file: %s", err) + } + } else { + m.read = make([]*readEntry, 0, 1000) + } + + return m, nil +} + +// Sync persists the metadata to disk +func (m *Metadata) Sync() error { + // Serialize to a buffer first so we write all at once + var buf bytes.Buffer + if err := m.MarshalBinary(&buf); err != nil { + return err + } + + // Write the whole thing to the file + n, err := m.file.WriteAt(buf.Bytes(), 0) + if err != nil { + return err + } + + // Since our on-disk format for metadata self-describes length, + // it's okay to truncate as a separate operation because an un-truncated + // file is still readable + err = m.file.Truncate(int64(n)) + if err != nil { + return err + } + + return nil +} + +// Close syncs metadata to disk and closes the underlying file descriptor +func (m *Metadata) Close() error { + err := m.Sync() + if err != nil { + return err + } + m.file.Close() + return nil +} + +// setDeadRange sets the dead range start and length, then persists it to disk +// without rewriting the whole file +func (m *Metadata) setDeadRange(start, length int64) error { + m.deadRangeStart = start + m.deadRangeLength = length + var buf bytes.Buffer + if err := binary.Write(&buf, binary.LittleEndian, m.deadRangeStart); err != nil { + return err + } + if err := binary.Write(&buf, binary.LittleEndian, m.deadRangeLength); err != nil { + return err + } + _, err := m.file.WriteAt(buf.Bytes(), 8) + return err +} + +// MarshalBinary marshals a metadata struct to a binary stream +func (m *Metadata) MarshalBinary(wr io.Writer) (err error) { + // Version (currently unused) + if err = binary.Write(wr, binary.LittleEndian, int64(1)); err != nil { + return + } + + // Dead Range info + if err = binary.Write(wr, binary.LittleEndian, m.deadRangeStart); err != nil { + return + } + if err = binary.Write(wr, binary.LittleEndian, m.deadRangeLength); err != nil { + return + } + + // Unread range info + if err = binary.Write(wr, binary.LittleEndian, m.unreadStartOffset); err != nil { + return + } + if err = binary.Write(wr, binary.LittleEndian, m.unreadCount); err != nil { + return + } + + // Read entries offsets + if err = binary.Write(wr, binary.LittleEndian, int64(len(m.read))); err != nil { + return + } + for _, readEntry := range m.read { + if err = readEntry.MarshalBinary(wr); err != nil { + return + } + } + return nil +} + +// UnmarshalBinary unmarshals metadata from a binary stream (usually a file) +func (m *Metadata) UnmarshalBinary(r io.Reader) error { + // Read version + var version int64 + if err := binary.Read(r, binary.LittleEndian, &version); err != nil { + return fmt.Errorf("failed to read version: %s", err) + } + + // Read dead range + if err := binary.Read(r, binary.LittleEndian, &m.deadRangeStart); err != nil { + return err + } + if err := binary.Read(r, binary.LittleEndian, &m.deadRangeLength); err != nil { + return err + } + + // Read unread info + if err := binary.Read(r, binary.LittleEndian, &m.unreadStartOffset); err != nil { + return fmt.Errorf("read unread start offset: %s", err) + } + if err := binary.Read(r, binary.LittleEndian, &m.unreadCount); err != nil { + return fmt.Errorf("read contiguous count: %s", err) + } + + // Read read info + var readCount int64 + if err := binary.Read(r, binary.LittleEndian, &readCount); err != nil { + return fmt.Errorf("read read count: %s", err) + } + m.read = make([]*readEntry, readCount) + for i := 0; i < int(readCount); i++ { + newEntry := &readEntry{} + if err := newEntry.UnmarshalBinary(r); err != nil { + return err + } + + m.read[i] = newEntry + } + + return nil +} + +// readEntry is a struct holding metadata about read entries +type readEntry struct { + // A flushed entry is one that has been flushed and is ready + // to be removed from disk + flushed bool + + // The number of bytes the entry takes on disk + length int64 + + // The offset in the file where the entry starts + startOffset int64 +} + +// MarshalBinary marshals a readEntry struct to a binary stream +func (re readEntry) MarshalBinary(wr io.Writer) error { + if err := binary.Write(wr, binary.LittleEndian, re.flushed); err != nil { + return err + } + if err := binary.Write(wr, binary.LittleEndian, re.length); err != nil { + return err + } + if err := binary.Write(wr, binary.LittleEndian, re.startOffset); err != nil { + return err + } + return nil +} + +// UnmarshalBinary unmarshals a binary stream into a readEntry struct +func (re *readEntry) UnmarshalBinary(r io.Reader) error { + if err := binary.Read(r, binary.LittleEndian, &re.flushed); err != nil { + return fmt.Errorf("read disk entry flushed: %s", err) + } + + if err := binary.Read(r, binary.LittleEndian, &re.length); err != nil { + return fmt.Errorf("read disk entry length: %s", err) + } + + if err := binary.Read(r, binary.LittleEndian, &re.startOffset); err != nil { + return fmt.Errorf("read disk entry start offset: %s", err) + } + return nil +} + +// onDiskSize calculates the size in bytes on disk for a contiguous +// range of diskEntries +func onDiskSize(entries []*readEntry) int64 { + if len(entries) == 0 { + return 0 + } + + last := entries[len(entries)-1] + return last.startOffset + last.length - entries[0].startOffset +} diff --git a/operator/buffer/disk_metadata_test.go b/operator/buffer/disk_metadata_test.go new file mode 100644 index 000000000..3796cbbfc --- /dev/null +++ b/operator/buffer/disk_metadata_test.go @@ -0,0 +1,64 @@ +package buffer + +import ( + "bytes" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMetadata(t *testing.T) { + t.Run("binaryRoundTrip", func(t *testing.T) { + cases := [...]Metadata{ + 0: { + read: []*readEntry{}, + unreadStartOffset: 0, + unreadCount: 0, + deadRangeStart: 0, + deadRangeLength: 0, + }, + 1: { + read: []*readEntry{}, + unreadStartOffset: 0, + unreadCount: 50, + deadRangeStart: 0, + deadRangeLength: 0, + }, + 2: { + read: []*readEntry{ + { + flushed: false, + length: 10, + startOffset: 0, + }, + }, + unreadStartOffset: 10, + unreadCount: 50, + deadRangeStart: 0, + deadRangeLength: 0, + }, + 3: { + read: []*readEntry{}, + unreadStartOffset: 0, + unreadCount: 50, + deadRangeStart: 10, + deadRangeLength: 100, + }, + } + + for i, md := range cases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + var buf bytes.Buffer + err := md.MarshalBinary(&buf) + require.NoError(t, err) + + md2 := Metadata{} + err = md2.UnmarshalBinary(&buf) + require.NoError(t, err) + + require.Equal(t, md, md2) + }) + } + }) +} diff --git a/operator/buffer/disk_test.go b/operator/buffer/disk_test.go new file mode 100644 index 000000000..4debbaec2 --- /dev/null +++ b/operator/buffer/disk_test.go @@ -0,0 +1,331 @@ +package buffer + +import ( + "context" + "fmt" + "math/rand" + "strconv" + "sync" + "time" + + "testing" + + "github.com/observiq/stanza/entry" + "github.com/observiq/stanza/testutil" + "github.com/stretchr/testify/require" +) + +func openBuffer(t testing.TB) *DiskBuffer { + buffer := NewDiskBuffer(1 << 20) + dir := testutil.NewTempDir(t) + err := buffer.Open(dir, false) + require.NoError(t, err) + t.Cleanup(func() { buffer.Close() }) + return buffer +} + +func compact(t testing.TB, b *DiskBuffer) { + err := b.Compact() + require.NoError(t, err) +} + +func TestDiskBuffer(t *testing.T) { + t.Run("Simple", func(t *testing.T) { + t.Parallel() + b := openBuffer(t) + writeN(t, b, 1, 0) + readN(t, b, 1, 0) + }) + + t.Run("Write2Read1Read1", func(t *testing.T) { + t.Parallel() + b := openBuffer(t) + writeN(t, b, 2, 0) + readN(t, b, 1, 0) + readN(t, b, 1, 1) + }) + + t.Run("Write20Read10Read10", func(t *testing.T) { + t.Parallel() + b := openBuffer(t) + writeN(t, b, 20, 0) + readN(t, b, 10, 0) + readN(t, b, 10, 10) + }) + + t.Run("SingleReadWaitMultipleWrites", func(t *testing.T) { + t.Parallel() + b := openBuffer(t) + writeN(t, b, 10, 0) + readyDone := make(chan struct{}) + go func() { + readyDone <- struct{}{} + readWaitN(t, b, 20, 0) + readyDone <- struct{}{} + }() + <-readyDone + time.Sleep(100 * time.Millisecond) + writeN(t, b, 10, 10) + <-readyDone + }) + + t.Run("ReadWaitOnlyWaitForPartialWrite", func(t *testing.T) { + t.Parallel() + b := openBuffer(t) + writeN(t, b, 10, 0) + readyDone := make(chan struct{}) + go func() { + readyDone <- struct{}{} + readWaitN(t, b, 15, 0) + readyDone <- struct{}{} + }() + <-readyDone + writeN(t, b, 10, 10) + <-readyDone + readN(t, b, 5, 15) + }) + + t.Run("Write10Read10Read0", func(t *testing.T) { + t.Parallel() + b := openBuffer(t) + writeN(t, b, 10, 0) + readN(t, b, 10, 0) + dst := make([]*entry.Entry, 10) + _, n, err := b.Read(dst) + require.NoError(t, err) + require.Equal(t, 0, n) + }) + + t.Run("Write20Read10Read10Unfull", func(t *testing.T) { + t.Parallel() + b := openBuffer(t) + writeN(t, b, 20, 0) + readN(t, b, 10, 0) + dst := make([]*entry.Entry, 20) + _, n, err := b.Read(dst) + require.NoError(t, err) + require.Equal(t, 10, n) + }) + + t.Run("Write20Read10CompactRead10", func(t *testing.T) { + t.Parallel() + b := openBuffer(t) + writeN(t, b, 20, 0) + flushN(t, b, 10, 0) + compact(t, b) + readN(t, b, 10, 10) + }) + + t.Run("Write10Read2Flush2Read2Compact", func(t *testing.T) { + t.Parallel() + b := openBuffer(t) + writeN(t, b, 10, 0) + readN(t, b, 2, 0) + flushN(t, b, 2, 2) + readN(t, b, 2, 4) + require.NoError(t, b.Compact()) + }) + + t.Run("Write20Read10CloseRead20", func(t *testing.T) { + t.Parallel() + b := NewDiskBuffer(1 << 30) + dir := testutil.NewTempDir(t) + err := b.Open(dir, false) + require.NoError(t, err) + + writeN(t, b, 20, 0) + readN(t, b, 10, 0) + err = b.Close() + require.NoError(t, err) + + b2 := NewDiskBuffer(1 << 30) + err = b2.Open(dir, false) + require.NoError(t, err) + readN(t, b2, 20, 0) + }) + + t.Run("Write20Flush10CloseRead20", func(t *testing.T) { + t.Parallel() + b := NewDiskBuffer(1 << 30) + dir := testutil.NewTempDir(t) + err := b.Open(dir, false) + require.NoError(t, err) + + writeN(t, b, 20, 0) + flushN(t, b, 10, 0) + err = b.Close() + require.NoError(t, err) + + b2 := NewDiskBuffer(1 << 30) + err = b2.Open(dir, false) + require.NoError(t, err) + readN(t, b2, 10, 10) + }) + + t.Run("ReadWaitTimesOut", func(t *testing.T) { + t.Parallel() + b := openBuffer(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + dst := make([]*entry.Entry, 10) + _, n, err := b.ReadWait(ctx, dst) + require.NoError(t, err) + require.Equal(t, 0, n) + }) + + t.Run("AddTimesOut", func(t *testing.T) { + t.Parallel() + b := NewDiskBuffer(100) // Enough space for 1, but not 2 entries + dir := testutil.NewTempDir(t) + err := b.Open(dir, false) + require.NoError(t, err) + + // Add a first entry + err = b.Add(context.Background(), entry.New()) + require.NoError(t, err) + + // Second entry should block and be cancelled + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + err = b.Add(ctx, entry.New()) + require.Error(t, err) + cancel() + + // Read, flush, and compact + dst := make([]*entry.Entry, 1) + f, n, err := b.Read(dst) + require.NoError(t, err) + require.Equal(t, 1, n) + f() + require.NoError(t, b.Compact()) + + // Now there should be space for another entry + err = b.Add(context.Background(), entry.New()) + require.NoError(t, err) + }) + + t.Run("Write1kRandomFlushReadCompact", func(t *testing.T) { + t.Parallel() + rand.Seed(time.Now().Unix()) + seed := rand.Int63() + t.Run(strconv.Itoa(int(seed)), func(t *testing.T) { + t.Parallel() + r := rand.New(rand.NewSource(seed)) + + b := NewDiskBuffer(1 << 30) + dir := testutil.NewTempDir(t) + err := b.Open(dir, false) + require.NoError(t, err) + + writes := 0 + reads := 0 + + for i := 0; i < 1000; i++ { + j := r.Int() % 1000 + switch { + case j < 900: + writeN(t, b, 1, writes) + writes++ + case j < 990: + readCount := (writes - reads) / 2 + f := readN(t, b, readCount, reads) + if j%2 == 0 { + f() + } + reads += readCount + default: + err := b.Compact() + require.NoError(t, err) + } + } + }) + }) +} + +func TestDiskBufferBuild(t *testing.T) { + t.Run("Default", func(t *testing.T) { + cfg := NewDiskBufferConfig() + cfg.Path = testutil.NewTempDir(t) + b, err := cfg.Build(testutil.NewBuildContext(t), "test") + require.NoError(t, err) + diskBuffer := b.(*DiskBuffer) + require.Equal(t, diskBuffer.atEnd, false) + require.Len(t, diskBuffer.entryAdded, 1) + require.Equal(t, diskBuffer.maxBytes, int64(1<<32)) + require.Equal(t, diskBuffer.flushedBytes, int64(0)) + require.Len(t, diskBuffer.copyBuffer, 1<<16) + }) +} + +func BenchmarkDiskBuffer(b *testing.B) { + b.Run("NoSync", func(b *testing.B) { + buffer := openBuffer(b) + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + fmt.Printf("Benchmark: %d\n", b.N) + e := entry.New() + e.Record = "test log" + ctx := context.Background() + for i := 0; i < b.N; i++ { + panicOnErr(buffer.Add(ctx, e)) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + dst := make([]*entry.Entry, 1000) + ctx := context.Background() + for i := 0; i < b.N; { + ctx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + flush, n, err := buffer.ReadWait(ctx, dst) + cancel() + panicOnErr(err) + i += n + flush() + } + }() + + wg.Wait() + }) + + b.Run("Sync", func(b *testing.B) { + buffer := NewDiskBuffer(1 << 20) + dir := testutil.NewTempDir(b) + err := buffer.Open(dir, true) + require.NoError(b, err) + b.Cleanup(func() { buffer.Close() }) + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + fmt.Printf("Benchmark: %d\n", b.N) + e := entry.New() + e.Record = "test log" + ctx := context.Background() + for i := 0; i < b.N; i++ { + panicOnErr(buffer.Add(ctx, e)) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + dst := make([]*entry.Entry, 1000) + ctx := context.Background() + for i := 0; i < b.N; { + ctx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + flush, n, err := buffer.ReadWait(ctx, dst) + cancel() + panicOnErr(err) + i += n + flush() + } + }() + + wg.Wait() + }) +} diff --git a/operator/buffer/memory.go b/operator/buffer/memory.go new file mode 100644 index 000000000..c9fae37ff --- /dev/null +++ b/operator/buffer/memory.go @@ -0,0 +1,212 @@ +package buffer + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "sync" + "sync/atomic" + + "github.com/observiq/stanza/entry" + "github.com/observiq/stanza/operator" + "go.etcd.io/bbolt" + "golang.org/x/sync/semaphore" +) + +// MemoryBufferConfig holds the configuration for a memory buffer +type MemoryBufferConfig struct { + Type string `json:"type" yaml:"type"` + MaxEntries int `json:"max_entries" yaml:"max_entries"` +} + +// NewMemoryBufferConfig creates a new default MemoryBufferConfig +func NewMemoryBufferConfig() *MemoryBufferConfig { + return &MemoryBufferConfig{ + Type: "memory", + MaxEntries: 1 << 20, + } +} + +// Build builds a MemoryBufferConfig into a Buffer, loading any entries that were previously unflushed +// back into memory +func (c MemoryBufferConfig) Build(context operator.BuildContext, pluginID string) (Buffer, error) { + mb := &MemoryBuffer{ + db: context.Database, + pluginID: pluginID, + buf: make(chan *entry.Entry, c.MaxEntries), + sem: semaphore.NewWeighted(int64(c.MaxEntries)), + inFlight: make(map[uint64]*entry.Entry, c.MaxEntries), + } + if err := mb.loadFromDB(); err != nil { + return nil, err + } + + return mb, nil +} + +// MemoryBuffer is a buffer that holds all entries in memory until Close() is called, +// at which point it saves the entries into a database. It provides no guarantees about +// lost entries if shut down uncleanly. +type MemoryBuffer struct { + db operator.Database + pluginID string + buf chan *entry.Entry + inFlight map[uint64]*entry.Entry + inFlightMux sync.Mutex + entryID uint64 + sem *semaphore.Weighted +} + +// Add inserts an entry into the memory database, blocking until there is space +func (m *MemoryBuffer) Add(ctx context.Context, e *entry.Entry) error { + if err := m.sem.Acquire(ctx, 1); err != nil { + return err + } + + m.buf <- e + return nil +} + +// Read reads entries until either there are no entries left in the buffer +// or the destination slice is full. The returned function must be called +// once the entries are flushed to remove them from the memory buffer. +func (m *MemoryBuffer) Read(dst []*entry.Entry) (FlushFunc, int, error) { + inFlight := make([]uint64, len(dst)) + i := 0 + for ; i < len(dst); i++ { + select { + case e := <-m.buf: + dst[i] = e + id := atomic.AddUint64(&m.entryID, 1) + m.inFlightMux.Lock() + m.inFlight[id] = e + m.inFlightMux.Unlock() + inFlight[i] = id + default: + return m.newFlushFunc(inFlight[:i]), i, nil + } + } + + return m.newFlushFunc(inFlight[:i]), i, nil +} + +// ReadWait reads entries until either the destination slice is full, or the context passed to it +// is cancelled. The returned function must be called once the entries are flushed to remove them +// from the memory buffer +func (m *MemoryBuffer) ReadWait(ctx context.Context, dst []*entry.Entry) (FlushFunc, int, error) { + inFlightIDs := make([]uint64, len(dst)) + i := 0 + for ; i < len(dst); i++ { + select { + case e := <-m.buf: + dst[i] = e + id := atomic.AddUint64(&m.entryID, 1) + m.inFlightMux.Lock() + m.inFlight[id] = e + m.inFlightMux.Unlock() + inFlightIDs[i] = id + case <-ctx.Done(): + return m.newFlushFunc(inFlightIDs[:i]), i, nil + } + } + + return m.newFlushFunc(inFlightIDs[:i]), i, nil +} + +// newFlushFunc returns a function that will remove the entries identified by `ids` from the buffer +func (m *MemoryBuffer) newFlushFunc(ids []uint64) FlushFunc { + return func() error { + m.inFlightMux.Lock() + for _, id := range ids { + delete(m.inFlight, id) + } + m.inFlightMux.Unlock() + m.sem.Release(int64(len(ids))) + return nil + } +} + +// Close closes the memory buffer, saving all entries currently in the memory buffer to the +// agent's database. +func (m *MemoryBuffer) Close() error { + m.inFlightMux.Lock() + defer m.inFlightMux.Unlock() + return m.db.Update(func(tx *bbolt.Tx) error { + memBufBucket, err := tx.CreateBucketIfNotExists([]byte("memory_buffer")) + if err != nil { + return err + } + + b, err := memBufBucket.CreateBucketIfNotExists([]byte(m.pluginID)) + if err != nil { + return err + } + + for k, v := range m.inFlight { + if err := putKeyValue(b, k, v); err != nil { + return err + } + } + + for { + select { + case e := <-m.buf: + m.entryID++ + if err := putKeyValue(b, m.entryID, e); err != nil { + return err + } + default: + return nil + } + } + }) +} + +func putKeyValue(b *bbolt.Bucket, k uint64, v *entry.Entry) error { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + key := [8]byte{} + + binary.LittleEndian.PutUint64(key[:], k) + if err := enc.Encode(v); err != nil { + return err + } + return b.Put(key[:], buf.Bytes()) +} + +// loadFromDB loads any entries saved to the database previously into the memory buffer, +// allowing them to be flushed +func (m *MemoryBuffer) loadFromDB() error { + return m.db.Update(func(tx *bbolt.Tx) error { + memBufBucket := tx.Bucket([]byte("memory_buffer")) + if memBufBucket == nil { + return nil + } + + b := memBufBucket.Bucket([]byte(m.pluginID)) + if b == nil { + return nil + } + + return b.ForEach(func(k, v []byte) error { + if ok := m.sem.TryAcquire(1); !ok { + return fmt.Errorf("max_entries is smaller than the number of entries stored in the database") + } + + dec := json.NewDecoder(bytes.NewReader(v)) + var e entry.Entry + if err := dec.Decode(&e); err != nil { + return err + } + + select { + case m.buf <- &e: + return nil + default: + return fmt.Errorf("max_entries is smaller than the number of entries stored in the database") + } + }) + }) +} diff --git a/operator/buffer/memory_buffer.go b/operator/buffer/memory_buffer.go deleted file mode 100644 index 29d0a7656..000000000 --- a/operator/buffer/memory_buffer.go +++ /dev/null @@ -1,113 +0,0 @@ -package buffer - -import ( - "context" - "fmt" - "sync/atomic" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/observiq/stanza/entry" - "go.uber.org/zap" - "google.golang.org/api/support/bundler" -) - -// MemoryBuffer is a buffer that holds entries in memory -type MemoryBuffer struct { - *bundler.Bundler - config *Config - cancel context.CancelFunc -} - -// NewMemoryBuffer will return a new memory buffer with the supplied configuration -func NewMemoryBuffer(config *Config) *MemoryBuffer { - return &MemoryBuffer{config: config} -} - -// BundleHandler is an interface that process multiple entries -type BundleHandler interface { - ProcessMulti(context.Context, []*entry.Entry) error - Logger() *zap.SugaredLogger -} - -// SetHandler will set the handler of the memory buffer -func (m *MemoryBuffer) SetHandler(handler BundleHandler) { - ctx, cancel := context.WithCancel(context.Background()) - currentBundleID := int64(0) - handleFunc := func(entries interface{}) { - bundleID := atomic.AddInt64(¤tBundleID, 1) - b := m.NewExponentialBackOff() - for { - err := handler.ProcessMulti(ctx, entries.([]*entry.Entry)) - if err != nil { - duration := b.NextBackOff() - if duration == backoff.Stop { - handler.Logger().Errorw("Failed to flush bundle. Not retrying because we are beyond max backoff", zap.Any("error", err), "bundle_id", bundleID) - break - } else { - handler.Logger().Warnw("Failed to flush bundle", zap.Any("error", err), "backoff_time", duration.String(), "bundle_id", bundleID) - select { - case <-ctx.Done(): - handler.Logger().Debugw("Flush retry cancelled by context", "bundle_id", bundleID) - return - case <-time.After(duration): - continue - } - } - } - - break - } - } - - bd := bundler.NewBundler(&entry.Entry{}, handleFunc) - bd.DelayThreshold = m.config.DelayThreshold.Raw() - bd.BundleCountThreshold = m.config.BundleCountThreshold - bd.BundleByteThreshold = m.config.BundleByteThreshold - bd.BundleByteLimit = m.config.BundleByteLimit - bd.BufferedByteLimit = m.config.BufferedByteLimit - bd.HandlerLimit = m.config.HandlerLimit - - m.Bundler = bd - m.cancel = cancel -} - -// Flush will flush the memory buffer -func (m *MemoryBuffer) Flush(ctx context.Context) error { - finished := make(chan struct{}) - go func() { - m.Bundler.Flush() - close(finished) - }() - - select { - case <-finished: - return nil - case <-ctx.Done(): - return fmt.Errorf("context cancelled before flush finished") - } -} - -// Process will add an entry to the current buffer -func (m *MemoryBuffer) Process(ctx context.Context, entry *entry.Entry) error { - if m.Bundler == nil { - panic("must call SetHandler before any calls to Process") - } - - return m.AddWait(ctx, entry, 100) -} - -// NewExponentialBackOff will return a new exponential backoff for the memory buffer to use -func (m *MemoryBuffer) NewExponentialBackOff() *backoff.ExponentialBackOff { - b := &backoff.ExponentialBackOff{ - InitialInterval: m.config.Retry.InitialInterval.Raw(), - RandomizationFactor: m.config.Retry.RandomizationFactor, - Multiplier: m.config.Retry.Multiplier, - MaxInterval: m.config.Retry.MaxInterval.Raw(), - MaxElapsedTime: m.config.Retry.MaxElapsedTime.Raw(), - Stop: backoff.Stop, - Clock: backoff.SystemClock, - } - b.Reset() - return b -} diff --git a/operator/buffer/memory_buffer_test.go b/operator/buffer/memory_buffer_test.go deleted file mode 100644 index 5f4194e63..000000000 --- a/operator/buffer/memory_buffer_test.go +++ /dev/null @@ -1,200 +0,0 @@ -package buffer - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/observiq/stanza/entry" - "github.com/observiq/stanza/operator" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - "go.uber.org/zap/zaptest" -) - -type mockHandler struct { - fail chan bool - received chan []*entry.Entry - success chan []*entry.Entry - logger *zap.SugaredLogger -} - -func (h *mockHandler) ProcessMulti(ctx context.Context, entries []*entry.Entry) error { - h.received <- entries - fail := <-h.fail - if fail { - return fmt.Errorf("test failure") - } - - h.success <- entries - return nil -} - -func (h *mockHandler) Logger() *zap.SugaredLogger { - return h.logger -} - -func newMockHandler(t *testing.T) *mockHandler { - return &mockHandler{ - fail: make(chan bool), - received: make(chan []*entry.Entry), - success: make(chan []*entry.Entry), - logger: zaptest.NewLogger(t).Sugar(), - } -} - -func TestMemoryBufferRetry(t *testing.T) { - t.Run("FailOnce", func(t *testing.T) { - cfg := NewConfig() - cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Millisecond} - buffer, err := cfg.Build() - require.NoError(t, err) - handler := newMockHandler(t) - buffer.SetHandler(handler) - - err = buffer.Process(context.Background(), entry.New()) - require.NoError(t, err) - - // Tell it to fail as soon as we receive logs - <-handler.received - handler.fail <- true - - // The next time receive, don't fail, and ensure that we get a success - <-handler.received - handler.fail <- false - <-handler.success - }) - - t.Run("ContextCancelled", func(t *testing.T) { - cfg := NewConfig() - cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Millisecond} - buffer, err := cfg.Build() - require.NoError(t, err) - handler := newMockHandler(t) - buffer.SetHandler(handler) - - ctx, cancel := context.WithCancel(context.Background()) - err = buffer.Process(ctx, entry.New()) - require.NoError(t, err) - - // Fail once, but cancel the context so we don't retry - <-handler.received - cancel() - handler.fail <- true - - // We shouldn't get any more receives or successes - select { - case <-handler.received: - require.FailNow(t, "Received unexpected entries") - case <-handler.success: - require.FailNow(t, "Received unexpected success") - case <-time.After(200 * time.Millisecond): - } - }) - - t.Run("ExceededLimit", func(t *testing.T) { - cfg := NewConfig() - cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Millisecond} - cfg.Retry.MaxElapsedTime = operator.Duration{Duration: time.Nanosecond} - buffer, err := cfg.Build() - require.NoError(t, err) - handler := newMockHandler(t) - buffer.SetHandler(handler) - - err = buffer.Process(context.Background(), entry.New()) - require.NoError(t, err) - - // Fail once, which should exceed our - // unreasonably low max elapsed time - <-handler.received - handler.fail <- true - - // We shouldn't get any more receives or successes - select { - case <-handler.received: - require.FailNow(t, "Received unexpected entries") - case <-handler.success: - require.FailNow(t, "Received unexpected success") - case <-time.After(200 * time.Millisecond): - } - }) -} - -func TestMemoryBufferFlush(t *testing.T) { - t.Run("Simple", func(t *testing.T) { - cfg := NewConfig() - cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Hour} - buffer, err := cfg.Build() - require.NoError(t, err) - handler := newMockHandler(t) - buffer.SetHandler(handler) - - err = buffer.Process(context.Background(), entry.New()) - require.NoError(t, err) - - // We shouldn't have any logs to handle for at - // least 10 hours - select { - case <-handler.received: - require.FailNow(t, "Received entry unexpectedly early") - case <-time.After(50 * time.Millisecond): - } - - flushed := make(chan struct{}) - go func() { - defer close(flushed) - err := buffer.Flush(context.Background()) - require.NoError(t, err) - }() - - // After flushed is called, we should receive a log - <-handler.received - handler.fail <- false - <-handler.success - }) - - t.Run("ContextCancelled", func(t *testing.T) { - cfg := NewConfig() - cfg.DelayThreshold = operator.Duration{Duration: 10 * time.Hour} - buffer, err := cfg.Build() - require.NoError(t, err) - handler := newMockHandler(t) - buffer.SetHandler(handler) - - err = buffer.Process(context.Background(), entry.New()) - require.NoError(t, err) - - // We shouldn't have any logs to handle for at - // least 10 hours - select { - case <-handler.received: - require.FailNow(t, "Received entry unexpectedly early") - case <-time.After(50 * time.Millisecond): - } - - // Start the flush - ctx, cancel := context.WithCancel(context.Background()) - flushed := make(chan struct{}) - go func() { - defer close(flushed) - err := buffer.Flush(ctx) - require.Error(t, err) - }() - - // Cancel the context and wait for flush to finish - cancel() - select { - case <-flushed: - case <-time.After(100 * time.Millisecond): - require.FailNow(t, "Failed to flush in reasonable amount of time") - } - - // After flushed is called, we should receive a log still, since we - // timed out and ignored cleanup - <-handler.received - handler.fail <- false - <-handler.success - }) - -} diff --git a/operator/buffer/memory_test.go b/operator/buffer/memory_test.go new file mode 100644 index 000000000..8abc13abf --- /dev/null +++ b/operator/buffer/memory_test.go @@ -0,0 +1,273 @@ +package buffer + +import ( + "context" + "math/rand" + "strconv" + "sync" + "time" + + "testing" + + "github.com/observiq/stanza/entry" + "github.com/observiq/stanza/testutil" + "github.com/stretchr/testify/require" +) + +func newMemoryBuffer(t testing.TB) *MemoryBuffer { + b, err := NewMemoryBufferConfig().Build(testutil.NewBuildContext(t), "test") + require.NoError(t, err) + return b.(*MemoryBuffer) +} + +func TestMemoryBuffer(t *testing.T) { + t.Run("Simple", func(t *testing.T) { + t.Parallel() + b := newMemoryBuffer(t) + writeN(t, b, 1, 0) + readN(t, b, 1, 0) + }) + + t.Run("Write2Read1Read1", func(t *testing.T) { + t.Parallel() + b := newMemoryBuffer(t) + writeN(t, b, 2, 0) + readN(t, b, 1, 0) + readN(t, b, 1, 1) + }) + + t.Run("Write20Read10Read10", func(t *testing.T) { + t.Parallel() + b := newMemoryBuffer(t) + writeN(t, b, 20, 0) + readN(t, b, 10, 0) + readN(t, b, 10, 10) + }) + + t.Run("CheckN", func(t *testing.T) { + t.Run("Read", func(t *testing.T) { + b := newMemoryBuffer(t) + writeN(t, b, 12, 0) + dst := make([]*entry.Entry, 30) + _, n, err := b.Read(dst) + require.NoError(t, err) + require.Equal(t, n, 12) + }) + + t.Run("ReadWait", func(t *testing.T) { + b := newMemoryBuffer(t) + writeN(t, b, 12, 0) + dst := make([]*entry.Entry, 30) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _, n, err := b.ReadWait(ctx, dst) + require.NoError(t, err) + require.Equal(t, 12, n) + }) + }) + + t.Run("SingleReadWaitMultipleWrites", func(t *testing.T) { + t.Parallel() + b := newMemoryBuffer(t) + writeN(t, b, 10, 0) + readyDone := make(chan struct{}) + go func() { + readyDone <- struct{}{} + readWaitN(t, b, 20, 0) + readyDone <- struct{}{} + }() + <-readyDone + time.Sleep(100 * time.Millisecond) + writeN(t, b, 10, 10) + <-readyDone + }) + + t.Run("ReadWaitOnlyWaitForPartialWrite", func(t *testing.T) { + t.Parallel() + b := newMemoryBuffer(t) + writeN(t, b, 10, 0) + readyDone := make(chan struct{}) + go func() { + readyDone <- struct{}{} + readWaitN(t, b, 15, 0) + readyDone <- struct{}{} + }() + <-readyDone + writeN(t, b, 10, 10) + <-readyDone + readN(t, b, 5, 15) + }) + + t.Run("Write10Read10Read0", func(t *testing.T) { + t.Parallel() + b := newMemoryBuffer(t) + writeN(t, b, 10, 0) + readN(t, b, 10, 0) + dst := make([]*entry.Entry, 10) + _, n, err := b.Read(dst) + require.NoError(t, err) + require.Equal(t, 0, n) + }) + + t.Run("Write20Read10Read10Unfull", func(t *testing.T) { + t.Parallel() + b := newMemoryBuffer(t) + writeN(t, b, 20, 0) + readN(t, b, 10, 0) + dst := make([]*entry.Entry, 20) + _, n, err := b.Read(dst) + require.NoError(t, err) + require.Equal(t, 10, n) + }) + + t.Run("ReadWaitTimesOut", func(t *testing.T) { + t.Parallel() + b := newMemoryBuffer(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + dst := make([]*entry.Entry, 10) + _, n, err := b.ReadWait(ctx, dst) + require.NoError(t, err) + require.Equal(t, 0, n) + }) + + t.Run("WriteCloseRead", func(t *testing.T) { + t.Parallel() + bc := testutil.NewBuildContext(t) + b, err := NewMemoryBufferConfig().Build(bc, "test") + require.NoError(t, err) + writeN(t, b, 10, 0) + flushN(t, b, 5, 0) + + // Close the buffer, writing to the database + require.NoError(t, b.Close()) + + // Reopen, and expect to be able to read 5 logs + b, err = NewMemoryBufferConfig().Build(bc, "test") + require.NoError(t, err) + + readN(t, b, 5, 5) + dst := make([]*entry.Entry, 10) + _, n, err := b.Read(dst) + require.NoError(t, err) + require.Equal(t, 0, n) + }) + + t.Run("AddTimesOut", func(t *testing.T) { + t.Parallel() + cfg := MemoryBufferConfig{ + MaxEntries: 1, + } + b, err := cfg.Build(testutil.NewBuildContext(t), "test") + require.NoError(t, err) + + // Add a first entry + err = b.Add(context.Background(), entry.New()) + require.NoError(t, err) + + // Second entry should block and be cancelled + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + err = b.Add(ctx, entry.New()) + require.Error(t, err) + cancel() + + // Read and flush + dst := make([]*entry.Entry, 1) + f, n, err := b.Read(dst) + require.NoError(t, err) + require.Equal(t, 1, n) + f() + + // Now there should be space for another entry + err = b.Add(context.Background(), entry.New()) + require.NoError(t, err) + }) + + t.Run("Write10kRandom", func(t *testing.T) { + t.Parallel() + rand.Seed(time.Now().Unix()) + seed := rand.Int63() + t.Run(strconv.Itoa(int(seed)), func(t *testing.T) { + t.Parallel() + r := rand.New(rand.NewSource(seed)) + + b := newMemoryBuffer(t) + + writes := 0 + reads := 0 + + for i := 0; i < 10000; i++ { + j := r.Int() % 1000 + switch { + case j < 900: + writeN(t, b, 1, writes) + writes++ + default: + readCount := (writes - reads) / 2 + f := readN(t, b, readCount, reads) + if j%2 == 0 { + f() + } + reads += readCount + } + } + }) + }) + + t.Run("CloseReadUnflushed", func(t *testing.T) { + t.Parallel() + buildContext := testutil.NewBuildContext(t) + b, err := NewMemoryBufferConfig().Build(buildContext, "test") + require.NoError(t, err) + + writeN(t, b, 20, 0) + readN(t, b, 5, 0) + flushN(t, b, 5, 5) + readN(t, b, 5, 10) + + err = b.Close() + require.NoError(t, err) + + b2, err := NewMemoryBufferConfig().Build(buildContext, "test") + require.NoError(t, err) + + readN(t, b2, 5, 0) + readN(t, b2, 10, 10) + }) +} + +func BenchmarkMemoryBuffer(b *testing.B) { + buffer := newMemoryBuffer(b) + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + e := entry.New() + e.Record = "test log" + ctx := context.Background() + for i := 0; i < b.N; i++ { + panicOnErr(buffer.Add(ctx, e)) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + dst := make([]*entry.Entry, 1000) + ctx := context.Background() + for i := 0; i < b.N; { + ctx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + flush, n, err := buffer.ReadWait(ctx, dst) + cancel() + panicOnErr(err) + i += n + go func() { + time.Sleep(50 * time.Millisecond) + flush() + }() + } + }() + + wg.Wait() +} diff --git a/operator/buffer/util_test.go b/operator/buffer/util_test.go new file mode 100644 index 000000000..715b3d7ea --- /dev/null +++ b/operator/buffer/util_test.go @@ -0,0 +1,60 @@ +package buffer + +import ( + "context" + "testing" + "time" + + "github.com/observiq/stanza/entry" + "github.com/stretchr/testify/require" +) + +func intEntry(i int) *entry.Entry { + e := entry.New() + e.Timestamp = time.Date(2006, 01, 02, 03, 04, 05, 06, time.UTC) + e.Record = float64(i) + return e +} + +func writeN(t testing.TB, buffer Buffer, n, start int) { + ctx := context.Background() + for i := start; i < n+start; i++ { + err := buffer.Add(ctx, intEntry(i)) + require.NoError(t, err) + } +} + +func readN(t testing.TB, buffer Buffer, n, start int) FlushFunc { + entries := make([]*entry.Entry, n) + f, readCount, err := buffer.Read(entries) + require.NoError(t, err) + require.Equal(t, n, readCount) + for i := 0; i < n; i++ { + require.Equal(t, intEntry(start+i), entries[i]) + } + return f +} + +func readWaitN(t testing.TB, buffer Buffer, n, start int) FlushFunc { + entries := make([]*entry.Entry, n) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + f, readCount, err := buffer.ReadWait(ctx, entries) + require.NoError(t, err) + require.Equal(t, n, readCount) + for i := 0; i < n; i++ { + require.Equal(t, intEntry(start+i), entries[i]) + } + return f +} + +func flushN(t testing.TB, buffer Buffer, n, start int) { + f := readN(t, buffer, n, start) + f() +} + +func panicOnErr(err error) { + if err != nil { + panic(err) + } +} diff --git a/operator/builtin/output/elastic.go b/operator/builtin/output/elastic.go index 45cad203a..fbc61eef8 100644 --- a/operator/builtin/output/elastic.go +++ b/operator/builtin/output/elastic.go @@ -13,6 +13,7 @@ import ( "github.com/observiq/stanza/errors" "github.com/observiq/stanza/operator" "github.com/observiq/stanza/operator/buffer" + "github.com/observiq/stanza/operator/flusher" "github.com/observiq/stanza/operator/helper" "go.uber.org/zap" ) @@ -32,7 +33,8 @@ func NewElasticOutputConfig(operatorID string) *ElasticOutputConfig { // ElasticOutputConfig is the configuration of an elasticsearch output operator. type ElasticOutputConfig struct { helper.OutputConfig `yaml:",inline"` - BufferConfig buffer.Config `json:"buffer" yaml:"buffer"` + BufferConfig buffer.Config `json:"buffer" yaml:"buffer"` + FlusherConfig flusher.Config `json:"flusher" yaml:"flusher"` Addresses []string `json:"addresses" yaml:"addresses,flow"` Username string `json:"username" yaml:"username"` @@ -67,20 +69,20 @@ func (c ElasticOutputConfig) Build(context operator.BuildContext) (operator.Oper ) } - buffer, err := c.BufferConfig.Build() + buffer, err := c.BufferConfig.Build(context, c.ID()) if err != nil { return nil, err } elasticOutput := &ElasticOutput{ OutputOperator: outputOperator, - Buffer: buffer, + buffer: buffer, client: client, indexField: c.IndexField, idField: c.IDField, } - buffer.SetHandler(elasticOutput) + elasticOutput.flusher = c.FlusherConfig.Build(buffer, elasticOutput.ProcessMulti, elasticOutput.SugaredLogger) return elasticOutput, nil } @@ -88,13 +90,31 @@ func (c ElasticOutputConfig) Build(context operator.BuildContext) (operator.Oper // ElasticOutput is an operator that sends entries to elasticsearch. type ElasticOutput struct { helper.OutputOperator - buffer.Buffer + buffer buffer.Buffer + flusher *flusher.Flusher client *elasticsearch.Client indexField *entry.Field idField *entry.Field } +// Start signals to the ElasticOutput to begin flushing +func (e *ElasticOutput) Start() error { + e.flusher.Start() + return nil +} + +// Stop tells the ElasticOutput to stop gracefully +func (e *ElasticOutput) Stop() error { + e.flusher.Stop() + return e.buffer.Close() +} + +// Process adds an entry to the outputs buffer +func (e *ElasticOutput) Process(ctx context.Context, entry *entry.Entry) error { + return e.buffer.Add(ctx, entry) +} + // ProcessMulti will send entries to elasticsearch. func (e *ElasticOutput) ProcessMulti(ctx context.Context, entries []*entry.Entry) error { type indexDirective struct { diff --git a/operator/builtin/output/googlecloud/google_cloud.go b/operator/builtin/output/googlecloud/google_cloud.go index 7751a7ab9..26a609da2 100644 --- a/operator/builtin/output/googlecloud/google_cloud.go +++ b/operator/builtin/output/googlecloud/google_cloud.go @@ -14,6 +14,7 @@ import ( "github.com/observiq/stanza/internal/version" "github.com/observiq/stanza/operator" "github.com/observiq/stanza/operator/buffer" + "github.com/observiq/stanza/operator/flusher" "github.com/observiq/stanza/operator/helper" "go.uber.org/zap" "golang.org/x/oauth2/google" @@ -33,6 +34,7 @@ func NewGoogleCloudOutputConfig(operatorID string) *GoogleCloudOutputConfig { return &GoogleCloudOutputConfig{ OutputConfig: helper.NewOutputConfig(operatorID, "google_cloud_output"), BufferConfig: buffer.NewConfig(), + FlusherConfig: flusher.NewConfig(), Timeout: operator.Duration{Duration: 30 * time.Second}, UseCompression: true, } @@ -41,7 +43,8 @@ func NewGoogleCloudOutputConfig(operatorID string) *GoogleCloudOutputConfig { // GoogleCloudOutputConfig is the configuration of a google cloud output operator. type GoogleCloudOutputConfig struct { helper.OutputConfig `yaml:",inline"` - BufferConfig buffer.Config `json:"buffer,omitempty" yaml:"buffer,omitempty"` + BufferConfig buffer.Config `json:"buffer,omitempty" yaml:"buffer,omitempty"` + FlusherConfig flusher.Config `json:"flusher,omitempty" yaml:"flusher,omitempty"` Credentials string `json:"credentials,omitempty" yaml:"credentials,omitempty"` CredentialsFile string `json:"credentials_file,omitempty" yaml:"credentials_file,omitempty"` @@ -60,7 +63,7 @@ func (c GoogleCloudOutputConfig) Build(buildContext operator.BuildContext) (oper return nil, err } - newBuffer, err := c.BufferConfig.Build() + newBuffer, err := c.BufferConfig.Build(buildContext, c.ID()) if err != nil { return nil, err } @@ -70,7 +73,7 @@ func (c GoogleCloudOutputConfig) Build(buildContext operator.BuildContext) (oper credentials: c.Credentials, credentialsFile: c.CredentialsFile, projectID: c.ProjectID, - Buffer: newBuffer, + buffer: newBuffer, logNameField: c.LogNameField, traceField: c.TraceField, spanIDField: c.SpanIDField, @@ -78,7 +81,8 @@ func (c GoogleCloudOutputConfig) Build(buildContext operator.BuildContext) (oper useCompression: c.UseCompression, } - newBuffer.SetHandler(googleCloudOutput) + newFlusher := c.FlusherConfig.Build(newBuffer, googleCloudOutput.ProcessMulti, outputOperator.SugaredLogger) + googleCloudOutput.flusher = newFlusher return googleCloudOutput, nil } @@ -86,7 +90,8 @@ func (c GoogleCloudOutputConfig) Build(buildContext operator.BuildContext) (oper // GoogleCloudOutput is an operator that sends logs to google cloud logging. type GoogleCloudOutput struct { helper.OutputOperator - buffer.Buffer + buffer buffer.Buffer + flusher *flusher.Flusher credentials string credentialsFile string @@ -156,7 +161,13 @@ func (p *GoogleCloudOutput) Start() error { // Test writing a log message ctx, cancel = context.WithTimeout(context.Background(), p.timeout) defer cancel() - return p.TestConnection(ctx) + err = p.TestConnection(ctx) + if err != nil { + return err + } + + p.flusher.Start() + return nil } // TestConnection will attempt to send a test entry to google cloud logging @@ -171,15 +182,17 @@ func (p *GoogleCloudOutput) TestConnection(ctx context.Context) error { // Stop will flush the google cloud logger and close the underlying connection func (p *GoogleCloudOutput) Stop() error { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - err := p.Buffer.Flush(ctx) - if err != nil { - p.Warnw("Failed to flush", zap.Error(err)) + p.flusher.Stop() + if err := p.buffer.Close(); err != nil { + return err } return p.client.Close() } +func (p *GoogleCloudOutput) Process(ctx context.Context, e *entry.Entry) error { + return p.buffer.Add(ctx, e) +} + // ProcessMulti will process multiple log entries and send them in batch to google cloud logging. func (p *GoogleCloudOutput) ProcessMulti(ctx context.Context, entries []*entry.Entry) error { pbEntries := make([]*logpb.LogEntry, 0, len(entries)) diff --git a/operator/builtin/output/googlecloud/google_cloud_test.go b/operator/builtin/output/googlecloud/google_cloud_test.go index 64abd4149..5e81e779c 100644 --- a/operator/builtin/output/googlecloud/google_cloud_test.go +++ b/operator/builtin/output/googlecloud/google_cloud_test.go @@ -33,7 +33,7 @@ type googleCloudTestCase struct { func googleCloudBasicConfig() *GoogleCloudOutputConfig { cfg := NewGoogleCloudOutputConfig("test_id") cfg.ProjectID = "test_project_id" - cfg.BufferConfig.DelayThreshold = operator.Duration{Duration: time.Millisecond} + cfg.FlusherConfig.MaxWait = operator.Duration{Duration: 10 * time.Millisecond} return cfg } @@ -214,6 +214,8 @@ func TestGoogleCloudOutput(t *testing.T) { require.NoError(t, err) cloudOutput.(*GoogleCloudOutput).client = client + cloudOutput.(*GoogleCloudOutput).flusher.Start() + defer cloudOutput.(*GoogleCloudOutput).flusher.Stop() err = cloudOutput.Process(context.Background(), tc.input) require.NoError(t, err) @@ -385,6 +387,7 @@ func (g *googleCloudOutputBenchmark) Run(b *testing.B) { cfg := NewGoogleCloudOutputConfig(g.name) cfg.ProjectID = "test_project_id" + cfg.FlusherConfig.MaxWait = operator.NewDuration(10 * time.Millisecond) if g.configMod != nil { g.configMod(cfg) } @@ -392,6 +395,8 @@ func (g *googleCloudOutputBenchmark) Run(b *testing.B) { require.NoError(b, err) op.(*GoogleCloudOutput).client = client op.(*GoogleCloudOutput).timeout = 30 * time.Second + op.(*GoogleCloudOutput).flusher.Start() + defer op.(*GoogleCloudOutput).flusher.Stop() b.ResetTimer() var wg sync.WaitGroup @@ -401,8 +406,6 @@ func (g *googleCloudOutputBenchmark) Run(b *testing.B) { for i := 0; i < b.N; i++ { op.Process(context.Background(), g.entry) } - err = op.Stop() - require.NoError(b, err) }() wg.Add(1) @@ -416,6 +419,8 @@ func (g *googleCloudOutputBenchmark) Run(b *testing.B) { }() wg.Wait() + err = op.Stop() + require.NoError(b, err) } func BenchmarkGoogleCloudOutput(b *testing.B) { diff --git a/operator/duration.go b/operator/duration.go index e53d8dbbc..ef112e170 100644 --- a/operator/duration.go +++ b/operator/duration.go @@ -11,6 +11,13 @@ type Duration struct { time.Duration } +// NewDuration creates a new duration from a time +func NewDuration(t time.Duration) Duration { + return Duration{ + Duration: t, + } +} + // Raw will return the raw duration, without modification func (d *Duration) Raw() time.Duration { return d.Duration diff --git a/operator/flusher/flusher.go b/operator/flusher/flusher.go new file mode 100644 index 000000000..52b910e55 --- /dev/null +++ b/operator/flusher/flusher.go @@ -0,0 +1,190 @@ +package flusher + +import ( + "context" + "sync" + "sync/atomic" + "time" + + backoff "github.com/cenkalti/backoff/v4" + "github.com/observiq/stanza/entry" + "github.com/observiq/stanza/operator" + "github.com/observiq/stanza/operator/buffer" + "go.uber.org/zap" + "golang.org/x/sync/semaphore" +) + +// Config holds the configuration to build a new flusher +type Config struct { + // MaxConcurrent is the maximum number of goroutines flushing entries concurrently. + // Defaults to 16. + MaxConcurrent int `json:"max_concurrent" yaml:"max_concurrent"` + + // MaxWait is the maximum amount of time to wait for a full slice of entries + // before flushing the entries. Defaults to 1s. + MaxWait operator.Duration `json:"max_wait" yaml:"max_wait"` + + // MaxChunkEntries is the maximum number of entries to flush at a time. + // Defaults to 1000. + MaxChunkEntries int `json:"max_chunk_entries" yaml:"max_chunk_entries"` + + // TODO configurable retry +} + +// NewConfig creates a new default flusher config +func NewConfig() Config { + return Config{ + MaxConcurrent: 16, + MaxWait: operator.Duration{ + Duration: time.Second, + }, + MaxChunkEntries: 1000, + } +} + +// Build uses a Config to build a new Flusher +func (c *Config) Build(buf buffer.Buffer, f FlushFunc, logger *zap.SugaredLogger) *Flusher { + return &Flusher{ + buffer: buf, + sem: semaphore.NewWeighted(int64(c.MaxConcurrent)), + flush: f, + SugaredLogger: logger, + waitTime: c.MaxWait.Duration, + entrySlicePool: sync.Pool{ + New: func() interface{} { + slice := make([]*entry.Entry, c.MaxChunkEntries) + return &slice + }, + }, + } +} + +// Flusher is used to flush entries from a buffer concurrently. It handles max concurrenty, +// retry behavior, and cancellation. +type Flusher struct { + buffer buffer.Buffer + sem *semaphore.Weighted + wg sync.WaitGroup + cancel context.CancelFunc + chunkIDCounter uint64 + flush FlushFunc + waitTime time.Duration + entrySlicePool sync.Pool + *zap.SugaredLogger +} + +// FlushFunc is a function that the flusher uses to flush a slice of entries +type FlushFunc func(context.Context, []*entry.Entry) error + +// Start begins flushing +func (f *Flusher) Start() { + ctx, cancel := context.WithCancel(context.Background()) + f.cancel = cancel + + f.wg.Add(1) + go func() { + defer f.wg.Done() + f.read(ctx) + }() +} + +// Stop cancels all the in-progress flushers and waits until they have returned +func (f *Flusher) Stop() { + f.cancel() + f.wg.Wait() +} + +func (f *Flusher) read(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + default: + } + + // Fill a slice of entries + entries := f.getEntrySlice() + readCtx, cancel := context.WithTimeout(ctx, f.waitTime) + markFlushed, n, err := f.buffer.ReadWait(readCtx, entries) + cancel() + if err != nil { + f.Errorw("Failed to read entries from buffer", zap.Error(err)) + } + + // If we've timed out, but have no entries, don't bother flushing them + if n == 0 { + continue + } + + // Wait until we have free flusher goroutines + err = f.sem.Acquire(ctx, 1) + if err != nil { + // Context cancelled + return + } + + // Start a new flusher goroutine + f.wg.Add(1) + go func() { + defer f.wg.Done() + defer f.sem.Release(1) + defer f.putEntrySlice(entries) + + if err := f.flushWithRetry(ctx, entries[:n]); err == nil { + if err := markFlushed(); err != nil { + f.Errorw("Failed while marking entries flushed", zap.Error(err)) + } + } + }() + } +} + +// flushWithRetry will continue trying to call Flusher.flushFunc with the entries passed +// in until either flushFunc returns no error or the context is cancelled. It will only +// return an error in the case that the context was cancelled. If no error was returned, +// it is safe to mark the entries in the buffer as flushed. +func (f *Flusher) flushWithRetry(ctx context.Context, entries []*entry.Entry) error { + chunkID := atomic.AddUint64(&f.chunkIDCounter, 1) + b := newExponentialBackoff() + for { + err := f.flush(ctx, entries) + if err == nil { + return nil + } + + waitTime := b.NextBackOff() + if waitTime == b.Stop { + f.Errorw("Reached max backoff time during chunk flush retry. Dropping logs in chunk", "chunk_id", chunkID) + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(waitTime): + } + } +} + +func (f *Flusher) getEntrySlice() []*entry.Entry { + return *(f.entrySlicePool.Get().(*[]*entry.Entry)) +} + +func (f *Flusher) putEntrySlice(slice []*entry.Entry) { + f.entrySlicePool.Put(&slice) +} + +// newExponentialBackoff returns a default ExponentialBackOff +func newExponentialBackoff() *backoff.ExponentialBackOff { + b := &backoff.ExponentialBackOff{ + InitialInterval: backoff.DefaultInitialInterval, + RandomizationFactor: backoff.DefaultRandomizationFactor, + Multiplier: backoff.DefaultMultiplier, + MaxInterval: 10 * time.Minute, + MaxElapsedTime: time.Duration(0), + Stop: backoff.Stop, + Clock: backoff.SystemClock, + } + b.Reset() + return b +} diff --git a/operator/flusher/flusher_test.go b/operator/flusher/flusher_test.go new file mode 100644 index 000000000..a1e56a105 --- /dev/null +++ b/operator/flusher/flusher_test.go @@ -0,0 +1,53 @@ +package flusher + +import ( + "context" + "testing" + "time" + + "github.com/observiq/stanza/entry" + "github.com/observiq/stanza/operator" + "github.com/observiq/stanza/operator/buffer" + "github.com/observiq/stanza/testutil" + "github.com/stretchr/testify/require" +) + +func TestFlusher(t *testing.T) { + buildContext := testutil.NewBuildContext(t) + buf, err := buffer.NewConfig().Build(buildContext, "testID") + require.NoError(t, err) + defer buf.Close() + + outChan := make(chan struct{}) + flushFunc := func(ctx context.Context, entries []*entry.Entry) error { + for i := 0; i < len(entries); i++ { + select { + case <-ctx.Done(): + return nil + case outChan <- struct{}{}: + } + } + return nil + } + + flusherCfg := NewConfig() + flusherCfg.MaxWait = operator.Duration{ + Duration: 10 * time.Millisecond, + } + flusher := flusherCfg.Build(buf, flushFunc, nil) + + for i := 0; i < 100; i++ { + err := buf.Add(context.Background(), entry.New()) + require.NoError(t, err) + } + + flusher.Start() + defer flusher.Stop() + for i := 0; i < 100; i++ { + select { + case <-time.After(time.Second): + require.FailNow(t, "timed out") + case <-outChan: + } + } +} diff --git a/operator/helper/input.go b/operator/helper/input.go index 3973b535d..9e2d5b92a 100644 --- a/operator/helper/input.go +++ b/operator/helper/input.go @@ -65,14 +65,16 @@ type InputOperator struct { // NewEntry will create a new entry using the `write_to`, `labels`, and `resource` configuration. func (i *InputOperator) NewEntry(value interface{}) (*entry.Entry, error) { entry := entry.New() - entry.Set(i.WriteTo, value) + if err := entry.Set(i.WriteTo, value); err != nil { + return nil, errors.Wrap(err, "add record to entry") + } if err := i.Label(entry); err != nil { - return nil, errors.Wrap(err, "failed to add labels to entry") + return nil, errors.Wrap(err, "add labels to entry") } if err := i.Identify(entry); err != nil { - return nil, errors.Wrap(err, "failed to add resource keys to entry") + return nil, errors.Wrap(err, "add resource keys to entry") } return entry, nil diff --git a/operator/helper/parser.go b/operator/helper/parser.go index 7f410288c..ca006ba7b 100644 --- a/operator/helper/parser.go +++ b/operator/helper/parser.go @@ -100,7 +100,9 @@ func (p *ParserOperator) ProcessWith(ctx context.Context, entry *entry.Entry, pa entry.Delete(p.ParseFrom) } - entry.Set(p.ParseTo, newValue) + if err := entry.Set(p.ParseTo, newValue); err != nil { + return p.HandleEntryError(ctx, entry, errors.Wrap(err, "set parse_to")) + } var timeParseErr error if p.TimeParser != nil { diff --git a/operator/helper/parser_test.go b/operator/helper/parser_test.go index 198992671..a897cf260 100644 --- a/operator/helper/parser_test.go +++ b/operator/helper/parser_test.go @@ -191,9 +191,10 @@ func TestParserInvalidTimeValidSeverityParse(t *testing.T) { } ctx := context.Background() testEntry := entry.New() - testEntry.Set(entry.NewRecordField("severity"), "info") + err := testEntry.Set(entry.NewRecordField("severity"), "info") + require.NoError(t, err) - err := parser.ProcessWith(ctx, testEntry, parse) + err = parser.ProcessWith(ctx, testEntry, parse) require.Error(t, err) require.Contains(t, err.Error(), "time parser: log entry does not have the expected parse_from field") @@ -233,9 +234,10 @@ func TestParserValidTimeInvalidSeverityParse(t *testing.T) { } ctx := context.Background() testEntry := entry.New() - testEntry.Set(entry.NewRecordField("timestamp"), "12:34PM") + err := testEntry.Set(entry.NewRecordField("timestamp"), "12:34PM") + require.NoError(t, err) - err := parser.ProcessWith(ctx, testEntry, parse) + err = parser.ProcessWith(ctx, testEntry, parse) require.Error(t, err) require.Contains(t, err.Error(), "severity parser: log entry does not have the expected parse_from field") @@ -301,8 +303,10 @@ func TestParserWithPreserve(t *testing.T) { } ctx := context.Background() testEntry := entry.New() - testEntry.Set(parser.ParseFrom, "test-value") - err := parser.ProcessWith(ctx, testEntry, parse) + err := testEntry.Set(parser.ParseFrom, "test-value") + require.NoError(t, err) + + err = parser.ProcessWith(ctx, testEntry, parse) require.NoError(t, err) actualValue, ok := testEntry.Get(parser.ParseFrom) @@ -340,8 +344,9 @@ func TestParserWithoutPreserve(t *testing.T) { } ctx := context.Background() testEntry := entry.New() - testEntry.Set(parser.ParseFrom, "test-value") - err := parser.ProcessWith(ctx, testEntry, parse) + err := testEntry.Set(parser.ParseFrom, "test-value") + require.NoError(t, err) + err = parser.ProcessWith(ctx, testEntry, parse) require.NoError(t, err) _, ok := testEntry.Get(parser.ParseFrom) diff --git a/operator/helper/writer.go b/operator/helper/writer.go index aaa000c48..af9e1cd20 100644 --- a/operator/helper/writer.go +++ b/operator/helper/writer.go @@ -60,7 +60,7 @@ func (w *WriterOperator) Write(ctx context.Context, e *entry.Entry) { _ = operator.Process(ctx, e) return } - operator.Process(ctx, e.Copy()) + _ = operator.Process(ctx, e.Copy()) } } diff --git a/operator/plugin_parameter.go b/operator/plugin_parameter.go index 0481ce39b..db6d87147 100644 --- a/operator/plugin_parameter.go +++ b/operator/plugin_parameter.go @@ -6,6 +6,14 @@ import ( "github.com/observiq/stanza/errors" ) +const ( + stringType = "string" + boolType = "bool" + intType = "int" + stringsType = "strings" + enumType = "enum" +) + // PluginParameter is a basic description of a plugin's parameter. type PluginParameter struct { Label string @@ -41,7 +49,7 @@ func (param PluginParameter) validate() error { func (param PluginParameter) validateType() error { switch param.Type { - case "string", "int", "bool", "strings", "enum": // ok + case stringType, intType, boolType, stringsType, enumType: // ok default: return errors.NewError( "invalid type for parameter", @@ -53,14 +61,14 @@ func (param PluginParameter) validateType() error { func (param PluginParameter) validateValidValues() error { switch param.Type { - case "string", "int", "bool", "strings": + case stringType, intType, boolType, stringsType: if len(param.ValidValues) > 0 { return errors.NewError( fmt.Sprintf("valid_values is undefined for parameter of type '%s'", param.Type), "remove 'valid_values' field or change type to 'enum'", ) } - case "enum": + case enumType: if len(param.ValidValues) == 0 { return errors.NewError( "parameter of type 'enum' must have 'valid_values' specified", @@ -78,15 +86,15 @@ func (param PluginParameter) validateDefault() error { // Validate that Default corresponds to Type switch param.Type { - case "string": + case stringType: return validateStringDefault(param) - case "int": + case intType: return validateIntDefault(param) - case "bool": + case boolType: return validateBoolDefault(param) - case "strings": + case stringsType: return validateStringArrayDefault(param) - case "enum": + case enumType: return validateEnumDefault(param) default: return errors.NewError( diff --git a/operator/plugin_test.go b/operator/plugin_test.go index a0b39bea2..0c1e5ccd8 100644 --- a/operator/plugin_test.go +++ b/operator/plugin_test.go @@ -119,7 +119,6 @@ func TestPluginRegistryLoad(t *testing.T) { } func TestPluginMetadata(t *testing.T) { - testCases := []struct { name string expectErr bool @@ -359,7 +358,7 @@ parameters: label: Parameter description: The thing of the thing type: strings - default: + default: - hello pipeline: `, diff --git a/pipeline/config.go b/pipeline/config.go index f8b5faa49..445a5dfef 100644 --- a/pipeline/config.go +++ b/pipeline/config.go @@ -128,15 +128,13 @@ func (p Params) TemplateOutput(namespace string, defaultOutput []string) string if len(outputs) == 0 { outputs = defaultOutput } - return fmt.Sprintf("[%s]", strings.Join(outputs[:], ", ")) + return fmt.Sprintf("[%s]", strings.Join(outputs, ", ")) } // NamespaceExclusions will return all ids to exclude from namespacing. func (p Params) NamespaceExclusions(namespace string) []string { exclusions := []string{p.NamespacedID(namespace)} - for _, output := range p.NamespacedOutputs(namespace) { - exclusions = append(exclusions, output) - } + exclusions = append(exclusions, p.NamespacedOutputs(namespace)...) return exclusions }