From 6838ddf886a4e9592efa1442f255b7a77a1f0ff7 Mon Sep 17 00:00:00 2001 From: Antoine Grondin Date: Tue, 10 Dec 2019 16:47:23 +0900 Subject: [PATCH 1/6] convert to logfmt, cleanup unmarshaling a bit the code is still super messy and not DRY at all. but it's a bit better now. hopefully nothing was broken, and the exhaustive 0 test suite will catch any regression. yolo --- go.mod | 1 + go.sum | 2 + json_handler.go | 82 ++++++++++--------- logrus_handler.go => logfmt_handler.go | 108 +++++++++++++++++-------- scanner.go | 14 ++-- 5 files changed, 130 insertions(+), 77 deletions(-) rename logrus_handler.go => logfmt_handler.go (63%) diff --git a/go.mod b/go.mod index c50d59f..594f15a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 github.com/fatih/color v1.7.1-0.20180516100307-2d684516a886 + github.com/go-logfmt/logfmt v0.4.0 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 github.com/mattn/go-colorable v0.1.0 github.com/mattn/go-isatty v0.0.4 // indirect diff --git a/go.sum b/go.sum index 68c884c..8359a61 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/fatih/color v1.7.1-0.20180516100307-2d684516a886 h1:NAFoy+QgUpERgK3y1xiVh5HcOvSeZHpXTTo5qnvnuK4= github.com/fatih/color v1.7.1-0.20180516100307-2d684516a886/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= diff --git a/json_handler.go b/json_handler.go index 4294b9c..9c02db9 100644 --- a/json_handler.go +++ b/json_handler.go @@ -29,9 +29,24 @@ type JSONHandler struct { last map[string]string } +func checkEachUntilFound(fieldList []string, found func(string) bool) bool { + for _, field := range fieldList { + if found(field) { + return true + } + } + return false +} + // supportedTimeFields enumerates supported timestamp field names var supportedTimeFields = []string{"time", "ts", "@timestamp"} +// supportedMessageFields enumarates supported Message field names +var supportedMessageFields = []string{"message", "msg"} + +// supportedLevelFields enumarates supported level field names +var supportedLevelFields = []string{"level", "lvl"} + func (h *JSONHandler) clear() { h.Level = "" h.Time = time.Time{} @@ -48,7 +63,7 @@ func (h *JSONHandler) TryHandle(d []byte) bool { var ok bool for _, field := range supportedTimeFields { - ok = bytes.Contains(d, []byte(`"`+field+`":`)) + ok = bytes.Contains(d, []byte(field)) if ok { break } @@ -58,8 +73,7 @@ func (h *JSONHandler) TryHandle(d []byte) bool { return false } - err := h.UnmarshalJSON(d) - if err != nil { + if !h.UnmarshalJSON(d) { h.clear() return false } @@ -67,52 +81,46 @@ func (h *JSONHandler) TryHandle(d []byte) bool { } // UnmarshalJSON sets the fields of the handler. -func (h *JSONHandler) UnmarshalJSON(data []byte) error { +func (h *JSONHandler) UnmarshalJSON(data []byte) bool { raw := make(map[string]interface{}) err := json.Unmarshal(data, &raw) if err != nil { - return err + return false } - var time interface{} - var ok bool - - for _, field := range supportedTimeFields { - time, ok = raw[field] + checkEachUntilFound(supportedLevelFields, func(field string) bool { + time, ok := tryParseTime(raw[field]) if ok { + h.Time = time delete(raw, field) - break } - } + return ok + }) - if ok { - h.Time, ok = tryParseTime(time) - if !ok { - return fmt.Errorf("field time is not a known timestamp: %v", time) + checkEachUntilFound(supportedMessageFields, func(field string) bool { + msg, ok := raw[field].(string) + if ok { + h.Message = msg + delete(raw, field) } - } + return ok + }) - if h.Message, ok = raw["msg"].(string); ok { - delete(raw, "msg") - } else if h.Message, ok = raw["message"].(string); ok { - delete(raw, "message") - } - - h.Level, ok = raw["level"].(string) - if !ok { - h.Level, ok = raw["lvl"].(string) - delete(raw, "lvl") + checkEachUntilFound(supportedLevelFields, func(field string) bool { + lvl, ok := raw[field] if !ok { - // bunyan uses numerical log levels - level, ok := raw["level"].(float64) - if ok { - h.Level = convertBunyanLogLevel(level) - delete(raw, "level") - } else { - h.Level = "???" - } + return false } - } + if strLvl, ok := lvl.(string); ok { + h.Level = strLvl + } else if flLvl, ok := lvl.(float64); ok { + h.Level = convertBunyanLogLevel(flLvl) + } else { + h.Level = "???" + } + delete(raw, field) + return true + }) if h.Fields == nil { h.Fields = make(map[string]string) @@ -134,7 +142,7 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) error { } } - return nil + return true } // Prettify the output in a logrus like fashion. diff --git a/logrus_handler.go b/logfmt_handler.go similarity index 63% rename from logrus_handler.go rename to logfmt_handler.go index 26dfc33..1669311 100644 --- a/logrus_handler.go +++ b/logfmt_handler.go @@ -10,10 +10,11 @@ import ( "time" "github.com/fatih/color" + "github.com/go-logfmt/logfmt" ) -// LogrusHandler can handle logs emmited by logrus.TextFormatter loggers. -type LogrusHandler struct { +// LogfmtHandler can handle logs emmited by logrus.TextFormatter loggers. +type LogfmtHandler struct { buf *bytes.Buffer out *tabwriter.Writer truncKV int @@ -28,52 +29,95 @@ type LogrusHandler struct { last map[string]string } -func (h *LogrusHandler) clear() { +func (h *LogfmtHandler) clear() { h.Level = "" h.Time = time.Time{} h.Message = "" h.last = h.Fields h.Fields = make(map[string]string) - h.buf.Reset() + if h.buf != nil { + h.buf.Reset() + } } // CanHandle tells if this line can be handled by this handler. -func (h *LogrusHandler) CanHandle(d []byte) bool { - if !(bytes.Contains(d, []byte(`level=`)) || bytes.Contains(d, []byte(`lvl=`))) { - return false +func (h *LogfmtHandler) TryHandle(d []byte) bool { + var ok bool + for _, field := range supportedTimeFields { + ok = bytes.Contains(d, []byte(field)) + if ok { + break + } } - if !(bytes.Contains(d, []byte(`time=`)) || bytes.Contains(d, []byte(`ts=`))) { + + if !ok { return false } - if !(bytes.Contains(d, []byte(`message=`)) || bytes.Contains(d, []byte(`msg=`))) { + + err := h.UnmarshalLogfmt(d) + if err != nil { + h.clear() return false } return true } // HandleLogfmt sets the fields of the handler. -func (h *LogrusHandler) visit(key, val []byte) bool { - switch { - case bytes.Equal(key, []byte("level")): - h.setLevel(val) - case bytes.Equal(key, []byte("lvl")): - h.setLevel(val) - case bytes.Equal(key, []byte("msg")): - h.setMessage(val) - case bytes.Equal(key, []byte("message")): - h.setMessage(val) - case bytes.Equal(key, []byte("time")): - h.setTime(val) - case bytes.Equal(key, []byte("ts")): - h.setTime(val) - default: - h.setField(key, val) +func (h *LogfmtHandler) UnmarshalLogfmt(data []byte) error { + + dec := logfmt.NewDecoder(bytes.NewReader(data)) + for dec.ScanRecord() { + next_kv: + for dec.ScanKeyval() { + key := dec.Key() + val := dec.Value() + if h.Time.IsZero() { + foundTime := checkEachUntilFound(supportedLevelFields, func(field string) bool { + time, ok := tryParseTime(string(val)) + if ok { + h.Time = time + } + return ok + }) + if foundTime { + continue next_kv + } + } + + if len(h.Message) == 0 { + foundMessage := checkEachUntilFound(supportedMessageFields, func(field string) bool { + if !bytes.Equal(key, []byte(field)) { + return false + } + h.Message = string(val) + return true + }) + if foundMessage { + continue next_kv + } + } + + if len(h.Level) == 0 { + foundLevel := checkEachUntilFound(supportedLevelFields, func(field string) bool { + if !bytes.Equal(key, []byte(field)) { + return false + } + h.Level = string(val) + return true + }) + if foundLevel { + continue next_kv + } + } + + h.setField(key, val) + } } - return true + return dec.Err() } // Prettify the output in a logrus like fashion. -func (h *LogrusHandler) Prettify(skipUnchanged bool) []byte { +func (h *LogfmtHandler) Prettify(skipUnchanged bool) []byte { defer h.clear() if h.out == nil { if h.Opts == nil { @@ -137,9 +181,9 @@ func (h *LogrusHandler) Prettify(skipUnchanged bool) []byte { return h.buf.Bytes() } -func (h *LogrusHandler) setLevel(val []byte) { h.Level = string(val) } -func (h *LogrusHandler) setMessage(val []byte) { h.Message = string(val) } -func (h *LogrusHandler) setTime(val []byte) (parsed bool) { +func (h *LogfmtHandler) setLevel(val []byte) { h.Level = string(val) } +func (h *LogfmtHandler) setMessage(val []byte) { h.Message = string(val) } +func (h *LogfmtHandler) setTime(val []byte) (parsed bool) { valStr := string(val) if valFloat, err := strconv.ParseFloat(valStr, 64); err == nil { h.Time, parsed = tryParseTime(valFloat) @@ -149,14 +193,14 @@ func (h *LogrusHandler) setTime(val []byte) (parsed bool) { return } -func (h *LogrusHandler) setField(key, val []byte) { +func (h *LogfmtHandler) setField(key, val []byte) { if h.Fields == nil { h.Fields = make(map[string]string) } h.Fields[string(key)] = string(val) } -func (h *LogrusHandler) joinKVs(skipUnchanged bool, sep string) []string { +func (h *LogfmtHandler) joinKVs(skipUnchanged bool, sep string) []string { kv := make([]string, 0, len(h.Fields)) for k, v := range h.Fields { diff --git a/scanner.go b/scanner.go index 597ca44..14bcde2 100644 --- a/scanner.go +++ b/scanner.go @@ -4,8 +4,6 @@ import ( "bufio" "bytes" "io" - - "github.com/aybabtme/humanlog/parser/logfmt" ) var ( @@ -21,10 +19,10 @@ func Scanner(src io.Reader, dst io.Writer, opts *HandlerOptions) error { var line uint64 - var lastLogrus bool + var lastLogfmt bool var lastJSON bool - logrusEntry := LogrusHandler{Opts: opts} + logfmtEntry := LogfmtHandler{Opts: opts} jsonEntry := JSONHandler{Opts: opts} for in.Scan() { @@ -40,12 +38,12 @@ func Scanner(src io.Reader, dst io.Writer, opts *HandlerOptions) error { dst.Write(jsonEntry.Prettify(opts.SkipUnchanged && lastJSON)) lastJSON = true - case logrusEntry.CanHandle(lineData) && logfmt.Parse(lineData, true, true, logrusEntry.visit): - dst.Write(logrusEntry.Prettify(opts.SkipUnchanged && lastLogrus)) - lastLogrus = true + case logfmtEntry.TryHandle(lineData): + dst.Write(logfmtEntry.Prettify(opts.SkipUnchanged && lastLogfmt)) + lastLogfmt = true default: - lastLogrus = false + lastLogfmt = false lastJSON = false dst.Write(lineData) } From f20667e266bd8e756ed56625f21fcf41b7c75d87 Mon Sep 17 00:00:00 2001 From: Antoine Grondin Date: Tue, 10 Dec 2019 17:10:12 +0900 Subject: [PATCH 2/6] support any parseable logfmt or json key-value, not just those with time --- json_handler.go | 13 ------------- logfmt_handler.go | 20 +++----------------- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/json_handler.go b/json_handler.go index 9c02db9..cca12c3 100644 --- a/json_handler.go +++ b/json_handler.go @@ -60,19 +60,6 @@ func (h *JSONHandler) clear() { // TryHandle tells if this line was handled by this handler. func (h *JSONHandler) TryHandle(d []byte) bool { - var ok bool - - for _, field := range supportedTimeFields { - ok = bytes.Contains(d, []byte(field)) - if ok { - break - } - } - - if !ok { - return false - } - if !h.UnmarshalJSON(d) { h.clear() return false diff --git a/logfmt_handler.go b/logfmt_handler.go index 1669311..1b2bb2e 100644 --- a/logfmt_handler.go +++ b/logfmt_handler.go @@ -42,20 +42,7 @@ func (h *LogfmtHandler) clear() { // CanHandle tells if this line can be handled by this handler. func (h *LogfmtHandler) TryHandle(d []byte) bool { - var ok bool - for _, field := range supportedTimeFields { - ok = bytes.Contains(d, []byte(field)) - if ok { - break - } - } - - if !ok { - return false - } - - err := h.UnmarshalLogfmt(d) - if err != nil { + if !h.UnmarshalLogfmt(d) { h.clear() return false } @@ -63,8 +50,7 @@ func (h *LogfmtHandler) TryHandle(d []byte) bool { } // HandleLogfmt sets the fields of the handler. -func (h *LogfmtHandler) UnmarshalLogfmt(data []byte) error { - +func (h *LogfmtHandler) UnmarshalLogfmt(data []byte) bool { dec := logfmt.NewDecoder(bytes.NewReader(data)) for dec.ScanRecord() { next_kv: @@ -113,7 +99,7 @@ func (h *LogfmtHandler) UnmarshalLogfmt(data []byte) error { h.setField(key, val) } } - return dec.Err() + return dec.Err() == nil } // Prettify the output in a logrus like fashion. From 1fe6a2ece4317f1ca5d3e2bc8b3bf81e32081a59 Mon Sep 17 00:00:00 2001 From: Antoine Grondin Date: Tue, 10 Dec 2019 17:15:22 +0900 Subject: [PATCH 3/6] switch travis to use go modules --- .travis.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9361deb..0cf9cae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,6 @@ language: go go: - master - tip -install: - - mkdir -p $GOPATH/bin - - wget -qO- https://github.com/aybabtme/untilitworks/releases/download/0.2/untilitworks_linux.tar.gz | tar xvz - - curl https://glide.sh/get > install_glide.sh - - chmod +x install_glide.sh - - ./untilitworks ./install_glide.sh + script: - - go test -cover -v $(glide nv) + - go test -mod=vendor From 520fffd76767233e95b5b5f75e20fce45fbd6bf5 Mon Sep 17 00:00:00 2001 From: Antoine Grondin Date: Tue, 10 Dec 2019 17:17:00 +0900 Subject: [PATCH 4/6] actually use github actions instead --- .github/workflows/go.yml | 19 +++++++++++++++++++ .travis.yml | 7 ------- 2 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/go.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..b2a8a5d --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,19 @@ +name: Go +on: [push] +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.12 + uses: actions/setup-go@v1 + with: + go-version: 1.12 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Test + run: go test -mod=vendor -short ./... diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0cf9cae..0000000 --- a/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go -go: - - master - - tip - -script: - - go test -mod=vendor From 4044614aae3fcdfcd181285e656da9cb846c1fb0 Mon Sep 17 00:00:00 2001 From: Antoine Grondin Date: Tue, 10 Dec 2019 17:18:29 +0900 Subject: [PATCH 5/6] vendor logfmt --- vendor/github.com/go-logfmt/logfmt/.gitignore | 4 + .../github.com/go-logfmt/logfmt/.travis.yml | 16 + .../github.com/go-logfmt/logfmt/CHANGELOG.md | 41 +++ vendor/github.com/go-logfmt/logfmt/LICENSE | 22 ++ vendor/github.com/go-logfmt/logfmt/README.md | 33 ++ vendor/github.com/go-logfmt/logfmt/decode.go | 237 +++++++++++++ vendor/github.com/go-logfmt/logfmt/doc.go | 6 + vendor/github.com/go-logfmt/logfmt/encode.go | 322 ++++++++++++++++++ vendor/github.com/go-logfmt/logfmt/fuzz.go | 126 +++++++ vendor/github.com/go-logfmt/logfmt/go.mod | 3 + vendor/github.com/go-logfmt/logfmt/go.sum | 2 + .../github.com/go-logfmt/logfmt/jsonstring.go | 277 +++++++++++++++ vendor/modules.txt | 2 + 13 files changed, 1091 insertions(+) create mode 100644 vendor/github.com/go-logfmt/logfmt/.gitignore create mode 100644 vendor/github.com/go-logfmt/logfmt/.travis.yml create mode 100644 vendor/github.com/go-logfmt/logfmt/CHANGELOG.md create mode 100644 vendor/github.com/go-logfmt/logfmt/LICENSE create mode 100644 vendor/github.com/go-logfmt/logfmt/README.md create mode 100644 vendor/github.com/go-logfmt/logfmt/decode.go create mode 100644 vendor/github.com/go-logfmt/logfmt/doc.go create mode 100644 vendor/github.com/go-logfmt/logfmt/encode.go create mode 100644 vendor/github.com/go-logfmt/logfmt/fuzz.go create mode 100644 vendor/github.com/go-logfmt/logfmt/go.mod create mode 100644 vendor/github.com/go-logfmt/logfmt/go.sum create mode 100644 vendor/github.com/go-logfmt/logfmt/jsonstring.go diff --git a/vendor/github.com/go-logfmt/logfmt/.gitignore b/vendor/github.com/go-logfmt/logfmt/.gitignore new file mode 100644 index 0000000..320e53e --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/.gitignore @@ -0,0 +1,4 @@ +_testdata/ +_testdata2/ +logfmt-fuzz.zip +logfmt.test.exe diff --git a/vendor/github.com/go-logfmt/logfmt/.travis.yml b/vendor/github.com/go-logfmt/logfmt/.travis.yml new file mode 100644 index 0000000..25976da --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/.travis.yml @@ -0,0 +1,16 @@ +language: go +sudo: false +go: + - "1.7.x" + - "1.8.x" + - "1.9.x" + - "1.10.x" + - "1.11.x" + - "tip" + +before_install: + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cmd/cover + +script: + - goveralls -service=travis-ci diff --git a/vendor/github.com/go-logfmt/logfmt/CHANGELOG.md b/vendor/github.com/go-logfmt/logfmt/CHANGELOG.md new file mode 100644 index 0000000..3455b8e --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/CHANGELOG.md @@ -0,0 +1,41 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.4.0] - 2018-11-21 + +### Added +- Go module support by [@ChrisHines] +- CHANGELOG by [@ChrisHines] + +### Changed +- Drop invalid runes from keys instead of returning ErrInvalidKey by [@ChrisHines] +- On panic while printing, attempt to print panic value by [@bboreham] + +## [0.3.0] - 2016-11-15 +### Added +- Pool buffers for quoted strings and byte slices by [@nussjustin] +### Fixed +- Fuzz fix, quote invalid UTF-8 values by [@judwhite] + +## [0.2.0] - 2016-05-08 +### Added +- Encoder.EncodeKeyvals by [@ChrisHines] + +## [0.1.0] - 2016-03-28 +### Added +- Encoder by [@ChrisHines] +- Decoder by [@ChrisHines] +- MarshalKeyvals by [@ChrisHines] + +[0.4.0]: https://github.com/go-logfmt/logfmt/compare/v0.3.0...v0.4.0 +[0.3.0]: https://github.com/go-logfmt/logfmt/compare/v0.2.0...v0.3.0 +[0.2.0]: https://github.com/go-logfmt/logfmt/compare/v0.1.0...v0.2.0 +[0.1.0]: https://github.com/go-logfmt/logfmt/commits/v0.1.0 + +[@ChrisHines]: https://github.com/ChrisHines +[@bboreham]: https://github.com/bboreham +[@judwhite]: https://github.com/judwhite +[@nussjustin]: https://github.com/nussjustin diff --git a/vendor/github.com/go-logfmt/logfmt/LICENSE b/vendor/github.com/go-logfmt/logfmt/LICENSE new file mode 100644 index 0000000..c026508 --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 go-logfmt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/vendor/github.com/go-logfmt/logfmt/README.md b/vendor/github.com/go-logfmt/logfmt/README.md new file mode 100644 index 0000000..3a8f10b --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/README.md @@ -0,0 +1,33 @@ +[![GoDoc](https://godoc.org/github.com/go-logfmt/logfmt?status.svg)](https://godoc.org/github.com/go-logfmt/logfmt) +[![Go Report Card](https://goreportcard.com/badge/go-logfmt/logfmt)](https://goreportcard.com/report/go-logfmt/logfmt) +[![TravisCI](https://travis-ci.org/go-logfmt/logfmt.svg?branch=master)](https://travis-ci.org/go-logfmt/logfmt) +[![Coverage Status](https://coveralls.io/repos/github/go-logfmt/logfmt/badge.svg?branch=master)](https://coveralls.io/github/go-logfmt/logfmt?branch=master) + +# logfmt + +Package logfmt implements utilities to marshal and unmarshal data in the [logfmt +format](https://brandur.org/logfmt). It provides an API similar to +[encoding/json](http://golang.org/pkg/encoding/json/) and +[encoding/xml](http://golang.org/pkg/encoding/xml/). + +The logfmt format was first documented by Brandur Leach in [this +article](https://brandur.org/logfmt). The format has not been formally +standardized. The most authoritative public specification to date has been the +documentation of a Go Language [package](http://godoc.org/github.com/kr/logfmt) +written by Blake Mizerany and Keith Rarick. + +## Goals + +This project attempts to conform as closely as possible to the prior art, while +also removing ambiguity where necessary to provide well behaved encoder and +decoder implementations. + +## Non-goals + +This project does not attempt to formally standardize the logfmt format. In the +event that logfmt is standardized this project would take conforming to the +standard as a goal. + +## Versioning + +Package logfmt publishes releases via [semver](http://semver.org/) compatible Git tags prefixed with a single 'v'. diff --git a/vendor/github.com/go-logfmt/logfmt/decode.go b/vendor/github.com/go-logfmt/logfmt/decode.go new file mode 100644 index 0000000..04e0eff --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/decode.go @@ -0,0 +1,237 @@ +package logfmt + +import ( + "bufio" + "bytes" + "fmt" + "io" + "unicode/utf8" +) + +// A Decoder reads and decodes logfmt records from an input stream. +type Decoder struct { + pos int + key []byte + value []byte + lineNum int + s *bufio.Scanner + err error +} + +// NewDecoder returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may read data from r beyond +// the logfmt records requested. +func NewDecoder(r io.Reader) *Decoder { + dec := &Decoder{ + s: bufio.NewScanner(r), + } + return dec +} + +// ScanRecord advances the Decoder to the next record, which can then be +// parsed with the ScanKeyval method. It returns false when decoding stops, +// either by reaching the end of the input or an error. After ScanRecord +// returns false, the Err method will return any error that occurred during +// decoding, except that if it was io.EOF, Err will return nil. +func (dec *Decoder) ScanRecord() bool { + if dec.err != nil { + return false + } + if !dec.s.Scan() { + dec.err = dec.s.Err() + return false + } + dec.lineNum++ + dec.pos = 0 + return true +} + +// ScanKeyval advances the Decoder to the next key/value pair of the current +// record, which can then be retrieved with the Key and Value methods. It +// returns false when decoding stops, either by reaching the end of the +// current record or an error. +func (dec *Decoder) ScanKeyval() bool { + dec.key, dec.value = nil, nil + if dec.err != nil { + return false + } + + line := dec.s.Bytes() + + // garbage + for p, c := range line[dec.pos:] { + if c > ' ' { + dec.pos += p + goto key + } + } + dec.pos = len(line) + return false + +key: + const invalidKeyError = "invalid key" + + start, multibyte := dec.pos, false + for p, c := range line[dec.pos:] { + switch { + case c == '=': + dec.pos += p + if dec.pos > start { + dec.key = line[start:dec.pos] + if multibyte && bytes.IndexRune(dec.key, utf8.RuneError) != -1 { + dec.syntaxError(invalidKeyError) + return false + } + } + if dec.key == nil { + dec.unexpectedByte(c) + return false + } + goto equal + case c == '"': + dec.pos += p + dec.unexpectedByte(c) + return false + case c <= ' ': + dec.pos += p + if dec.pos > start { + dec.key = line[start:dec.pos] + if multibyte && bytes.IndexRune(dec.key, utf8.RuneError) != -1 { + dec.syntaxError(invalidKeyError) + return false + } + } + return true + case c >= utf8.RuneSelf: + multibyte = true + } + } + dec.pos = len(line) + if dec.pos > start { + dec.key = line[start:dec.pos] + if multibyte && bytes.IndexRune(dec.key, utf8.RuneError) != -1 { + dec.syntaxError(invalidKeyError) + return false + } + } + return true + +equal: + dec.pos++ + if dec.pos >= len(line) { + return true + } + switch c := line[dec.pos]; { + case c <= ' ': + return true + case c == '"': + goto qvalue + } + + // value + start = dec.pos + for p, c := range line[dec.pos:] { + switch { + case c == '=' || c == '"': + dec.pos += p + dec.unexpectedByte(c) + return false + case c <= ' ': + dec.pos += p + if dec.pos > start { + dec.value = line[start:dec.pos] + } + return true + } + } + dec.pos = len(line) + if dec.pos > start { + dec.value = line[start:dec.pos] + } + return true + +qvalue: + const ( + untermQuote = "unterminated quoted value" + invalidQuote = "invalid quoted value" + ) + + hasEsc, esc := false, false + start = dec.pos + for p, c := range line[dec.pos+1:] { + switch { + case esc: + esc = false + case c == '\\': + hasEsc, esc = true, true + case c == '"': + dec.pos += p + 2 + if hasEsc { + v, ok := unquoteBytes(line[start:dec.pos]) + if !ok { + dec.syntaxError(invalidQuote) + return false + } + dec.value = v + } else { + start++ + end := dec.pos - 1 + if end > start { + dec.value = line[start:end] + } + } + return true + } + } + dec.pos = len(line) + dec.syntaxError(untermQuote) + return false +} + +// Key returns the most recent key found by a call to ScanKeyval. The returned +// slice may point to internal buffers and is only valid until the next call +// to ScanRecord. It does no allocation. +func (dec *Decoder) Key() []byte { + return dec.key +} + +// Value returns the most recent value found by a call to ScanKeyval. The +// returned slice may point to internal buffers and is only valid until the +// next call to ScanRecord. It does no allocation when the value has no +// escape sequences. +func (dec *Decoder) Value() []byte { + return dec.value +} + +// Err returns the first non-EOF error that was encountered by the Scanner. +func (dec *Decoder) Err() error { + return dec.err +} + +func (dec *Decoder) syntaxError(msg string) { + dec.err = &SyntaxError{ + Msg: msg, + Line: dec.lineNum, + Pos: dec.pos + 1, + } +} + +func (dec *Decoder) unexpectedByte(c byte) { + dec.err = &SyntaxError{ + Msg: fmt.Sprintf("unexpected %q", c), + Line: dec.lineNum, + Pos: dec.pos + 1, + } +} + +// A SyntaxError represents a syntax error in the logfmt input stream. +type SyntaxError struct { + Msg string + Line int + Pos int +} + +func (e *SyntaxError) Error() string { + return fmt.Sprintf("logfmt syntax error at pos %d on line %d: %s", e.Pos, e.Line, e.Msg) +} diff --git a/vendor/github.com/go-logfmt/logfmt/doc.go b/vendor/github.com/go-logfmt/logfmt/doc.go new file mode 100644 index 0000000..378e9ad --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/doc.go @@ -0,0 +1,6 @@ +// Package logfmt implements utilities to marshal and unmarshal data in the +// logfmt format. The logfmt format records key/value pairs in a way that +// balances readability for humans and simplicity of computer parsing. It is +// most commonly used as a more human friendly alternative to JSON for +// structured logging. +package logfmt diff --git a/vendor/github.com/go-logfmt/logfmt/encode.go b/vendor/github.com/go-logfmt/logfmt/encode.go new file mode 100644 index 0000000..4ea9d23 --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/encode.go @@ -0,0 +1,322 @@ +package logfmt + +import ( + "bytes" + "encoding" + "errors" + "fmt" + "io" + "reflect" + "strings" + "unicode/utf8" +) + +// MarshalKeyvals returns the logfmt encoding of keyvals, a variadic sequence +// of alternating keys and values. +func MarshalKeyvals(keyvals ...interface{}) ([]byte, error) { + buf := &bytes.Buffer{} + if err := NewEncoder(buf).EncodeKeyvals(keyvals...); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// An Encoder writes logfmt data to an output stream. +type Encoder struct { + w io.Writer + scratch bytes.Buffer + needSep bool +} + +// NewEncoder returns a new encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + w: w, + } +} + +var ( + space = []byte(" ") + equals = []byte("=") + newline = []byte("\n") + null = []byte("null") +) + +// EncodeKeyval writes the logfmt encoding of key and value to the stream. A +// single space is written before the second and subsequent keys in a record. +// Nothing is written if a non-nil error is returned. +func (enc *Encoder) EncodeKeyval(key, value interface{}) error { + enc.scratch.Reset() + if enc.needSep { + if _, err := enc.scratch.Write(space); err != nil { + return err + } + } + if err := writeKey(&enc.scratch, key); err != nil { + return err + } + if _, err := enc.scratch.Write(equals); err != nil { + return err + } + if err := writeValue(&enc.scratch, value); err != nil { + return err + } + _, err := enc.w.Write(enc.scratch.Bytes()) + enc.needSep = true + return err +} + +// EncodeKeyvals writes the logfmt encoding of keyvals to the stream. Keyvals +// is a variadic sequence of alternating keys and values. Keys of unsupported +// type are skipped along with their corresponding value. Values of +// unsupported type or that cause a MarshalerError are replaced by their error +// but do not cause EncodeKeyvals to return an error. If a non-nil error is +// returned some key/value pairs may not have be written. +func (enc *Encoder) EncodeKeyvals(keyvals ...interface{}) error { + if len(keyvals) == 0 { + return nil + } + if len(keyvals)%2 == 1 { + keyvals = append(keyvals, nil) + } + for i := 0; i < len(keyvals); i += 2 { + k, v := keyvals[i], keyvals[i+1] + err := enc.EncodeKeyval(k, v) + if err == ErrUnsupportedKeyType { + continue + } + if _, ok := err.(*MarshalerError); ok || err == ErrUnsupportedValueType { + v = err + err = enc.EncodeKeyval(k, v) + } + if err != nil { + return err + } + } + return nil +} + +// MarshalerError represents an error encountered while marshaling a value. +type MarshalerError struct { + Type reflect.Type + Err error +} + +func (e *MarshalerError) Error() string { + return "error marshaling value of type " + e.Type.String() + ": " + e.Err.Error() +} + +// ErrNilKey is returned by Marshal functions and Encoder methods if a key is +// a nil interface or pointer value. +var ErrNilKey = errors.New("nil key") + +// ErrInvalidKey is returned by Marshal functions and Encoder methods if, after +// dropping invalid runes, a key is empty. +var ErrInvalidKey = errors.New("invalid key") + +// ErrUnsupportedKeyType is returned by Encoder methods if a key has an +// unsupported type. +var ErrUnsupportedKeyType = errors.New("unsupported key type") + +// ErrUnsupportedValueType is returned by Encoder methods if a value has an +// unsupported type. +var ErrUnsupportedValueType = errors.New("unsupported value type") + +func writeKey(w io.Writer, key interface{}) error { + if key == nil { + return ErrNilKey + } + + switch k := key.(type) { + case string: + return writeStringKey(w, k) + case []byte: + if k == nil { + return ErrNilKey + } + return writeBytesKey(w, k) + case encoding.TextMarshaler: + kb, err := safeMarshal(k) + if err != nil { + return err + } + if kb == nil { + return ErrNilKey + } + return writeBytesKey(w, kb) + case fmt.Stringer: + ks, ok := safeString(k) + if !ok { + return ErrNilKey + } + return writeStringKey(w, ks) + default: + rkey := reflect.ValueOf(key) + switch rkey.Kind() { + case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.Slice, reflect.Struct: + return ErrUnsupportedKeyType + case reflect.Ptr: + if rkey.IsNil() { + return ErrNilKey + } + return writeKey(w, rkey.Elem().Interface()) + } + return writeStringKey(w, fmt.Sprint(k)) + } +} + +// keyRuneFilter returns r for all valid key runes, and -1 for all invalid key +// runes. When used as the mapping function for strings.Map and bytes.Map +// functions it causes them to remove invalid key runes from strings or byte +// slices respectively. +func keyRuneFilter(r rune) rune { + if r <= ' ' || r == '=' || r == '"' || r == utf8.RuneError { + return -1 + } + return r +} + +func writeStringKey(w io.Writer, key string) error { + k := strings.Map(keyRuneFilter, key) + if k == "" { + return ErrInvalidKey + } + _, err := io.WriteString(w, k) + return err +} + +func writeBytesKey(w io.Writer, key []byte) error { + k := bytes.Map(keyRuneFilter, key) + if len(k) == 0 { + return ErrInvalidKey + } + _, err := w.Write(k) + return err +} + +func writeValue(w io.Writer, value interface{}) error { + switch v := value.(type) { + case nil: + return writeBytesValue(w, null) + case string: + return writeStringValue(w, v, true) + case []byte: + return writeBytesValue(w, v) + case encoding.TextMarshaler: + vb, err := safeMarshal(v) + if err != nil { + return err + } + if vb == nil { + vb = null + } + return writeBytesValue(w, vb) + case error: + se, ok := safeError(v) + return writeStringValue(w, se, ok) + case fmt.Stringer: + ss, ok := safeString(v) + return writeStringValue(w, ss, ok) + default: + rvalue := reflect.ValueOf(value) + switch rvalue.Kind() { + case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.Slice, reflect.Struct: + return ErrUnsupportedValueType + case reflect.Ptr: + if rvalue.IsNil() { + return writeBytesValue(w, null) + } + return writeValue(w, rvalue.Elem().Interface()) + } + return writeStringValue(w, fmt.Sprint(v), true) + } +} + +func needsQuotedValueRune(r rune) bool { + return r <= ' ' || r == '=' || r == '"' || r == utf8.RuneError +} + +func writeStringValue(w io.Writer, value string, ok bool) error { + var err error + if ok && value == "null" { + _, err = io.WriteString(w, `"null"`) + } else if strings.IndexFunc(value, needsQuotedValueRune) != -1 { + _, err = writeQuotedString(w, value) + } else { + _, err = io.WriteString(w, value) + } + return err +} + +func writeBytesValue(w io.Writer, value []byte) error { + var err error + if bytes.IndexFunc(value, needsQuotedValueRune) != -1 { + _, err = writeQuotedBytes(w, value) + } else { + _, err = w.Write(value) + } + return err +} + +// EndRecord writes a newline character to the stream and resets the encoder +// to the beginning of a new record. +func (enc *Encoder) EndRecord() error { + _, err := enc.w.Write(newline) + if err == nil { + enc.needSep = false + } + return err +} + +// Reset resets the encoder to the beginning of a new record. +func (enc *Encoder) Reset() { + enc.needSep = false +} + +func safeError(err error) (s string, ok bool) { + defer func() { + if panicVal := recover(); panicVal != nil { + if v := reflect.ValueOf(err); v.Kind() == reflect.Ptr && v.IsNil() { + s, ok = "null", false + } else { + s, ok = fmt.Sprintf("PANIC:%v", panicVal), false + } + } + }() + s, ok = err.Error(), true + return +} + +func safeString(str fmt.Stringer) (s string, ok bool) { + defer func() { + if panicVal := recover(); panicVal != nil { + if v := reflect.ValueOf(str); v.Kind() == reflect.Ptr && v.IsNil() { + s, ok = "null", false + } else { + s, ok = fmt.Sprintf("PANIC:%v", panicVal), true + } + } + }() + s, ok = str.String(), true + return +} + +func safeMarshal(tm encoding.TextMarshaler) (b []byte, err error) { + defer func() { + if panicVal := recover(); panicVal != nil { + if v := reflect.ValueOf(tm); v.Kind() == reflect.Ptr && v.IsNil() { + b, err = nil, nil + } else { + b, err = nil, fmt.Errorf("panic when marshalling: %s", panicVal) + } + } + }() + b, err = tm.MarshalText() + if err != nil { + return nil, &MarshalerError{ + Type: reflect.TypeOf(tm), + Err: err, + } + } + return +} diff --git a/vendor/github.com/go-logfmt/logfmt/fuzz.go b/vendor/github.com/go-logfmt/logfmt/fuzz.go new file mode 100644 index 0000000..6553b35 --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/fuzz.go @@ -0,0 +1,126 @@ +// +build gofuzz + +package logfmt + +import ( + "bufio" + "bytes" + "fmt" + "io" + "reflect" + + kr "github.com/kr/logfmt" +) + +// Fuzz checks reserialized data matches +func Fuzz(data []byte) int { + parsed, err := parse(data) + if err != nil { + return 0 + } + var w1 bytes.Buffer + if err = write(parsed, &w1); err != nil { + panic(err) + } + parsed, err = parse(w1.Bytes()) + if err != nil { + panic(err) + } + var w2 bytes.Buffer + if err = write(parsed, &w2); err != nil { + panic(err) + } + if !bytes.Equal(w1.Bytes(), w2.Bytes()) { + panic(fmt.Sprintf("reserialized data does not match:\n%q\n%q\n", w1.Bytes(), w2.Bytes())) + } + return 1 +} + +// FuzzVsKR checks go-logfmt/logfmt against kr/logfmt +func FuzzVsKR(data []byte) int { + parsed, err := parse(data) + parsedKR, errKR := parseKR(data) + + // github.com/go-logfmt/logfmt is a stricter parser. It returns errors for + // more inputs than github.com/kr/logfmt. Ignore any inputs that have a + // stict error. + if err != nil { + return 0 + } + + // Fail if the more forgiving parser finds an error not found by the + // stricter parser. + if errKR != nil { + panic(fmt.Sprintf("unmatched error: %v", errKR)) + } + + if !reflect.DeepEqual(parsed, parsedKR) { + panic(fmt.Sprintf("parsers disagree:\n%+v\n%+v\n", parsed, parsedKR)) + } + return 1 +} + +type kv struct { + k, v []byte +} + +func parse(data []byte) ([][]kv, error) { + var got [][]kv + dec := NewDecoder(bytes.NewReader(data)) + for dec.ScanRecord() { + var kvs []kv + for dec.ScanKeyval() { + kvs = append(kvs, kv{dec.Key(), dec.Value()}) + } + got = append(got, kvs) + } + return got, dec.Err() +} + +func parseKR(data []byte) ([][]kv, error) { + var ( + s = bufio.NewScanner(bytes.NewReader(data)) + err error + h saveHandler + got [][]kv + ) + for err == nil && s.Scan() { + h.kvs = nil + err = kr.Unmarshal(s.Bytes(), &h) + got = append(got, h.kvs) + } + if err == nil { + err = s.Err() + } + return got, err +} + +type saveHandler struct { + kvs []kv +} + +func (h *saveHandler) HandleLogfmt(key, val []byte) error { + if len(key) == 0 { + key = nil + } + if len(val) == 0 { + val = nil + } + h.kvs = append(h.kvs, kv{key, val}) + return nil +} + +func write(recs [][]kv, w io.Writer) error { + enc := NewEncoder(w) + for _, rec := range recs { + for _, f := range rec { + if err := enc.EncodeKeyval(f.k, f.v); err != nil { + return err + } + } + if err := enc.EndRecord(); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/go-logfmt/logfmt/go.mod b/vendor/github.com/go-logfmt/logfmt/go.mod new file mode 100644 index 0000000..63d50f0 --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/go.mod @@ -0,0 +1,3 @@ +module github.com/go-logfmt/logfmt + +require github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 diff --git a/vendor/github.com/go-logfmt/logfmt/go.sum b/vendor/github.com/go-logfmt/logfmt/go.sum new file mode 100644 index 0000000..d0cdc16 --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/go.sum @@ -0,0 +1,2 @@ +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= diff --git a/vendor/github.com/go-logfmt/logfmt/jsonstring.go b/vendor/github.com/go-logfmt/logfmt/jsonstring.go new file mode 100644 index 0000000..030ac85 --- /dev/null +++ b/vendor/github.com/go-logfmt/logfmt/jsonstring.go @@ -0,0 +1,277 @@ +package logfmt + +import ( + "bytes" + "io" + "strconv" + "sync" + "unicode" + "unicode/utf16" + "unicode/utf8" +) + +// Taken from Go's encoding/json and modified for use here. + +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +var hex = "0123456789abcdef" + +var bufferPool = sync.Pool{ + New: func() interface{} { + return &bytes.Buffer{} + }, +} + +func getBuffer() *bytes.Buffer { + return bufferPool.Get().(*bytes.Buffer) +} + +func poolBuffer(buf *bytes.Buffer) { + buf.Reset() + bufferPool.Put(buf) +} + +// NOTE: keep in sync with writeQuotedBytes below. +func writeQuotedString(w io.Writer, s string) (int, error) { + buf := getBuffer() + buf.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if 0x20 <= b && b != '\\' && b != '"' { + i++ + continue + } + if start < i { + buf.WriteString(s[start:i]) + } + switch b { + case '\\', '"': + buf.WriteByte('\\') + buf.WriteByte(b) + case '\n': + buf.WriteByte('\\') + buf.WriteByte('n') + case '\r': + buf.WriteByte('\\') + buf.WriteByte('r') + case '\t': + buf.WriteByte('\\') + buf.WriteByte('t') + default: + // This encodes bytes < 0x20 except for \n, \r, and \t. + buf.WriteString(`\u00`) + buf.WriteByte(hex[b>>4]) + buf.WriteByte(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRuneInString(s[i:]) + if c == utf8.RuneError { + if start < i { + buf.WriteString(s[start:i]) + } + buf.WriteString(`\ufffd`) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + buf.WriteString(s[start:]) + } + buf.WriteByte('"') + n, err := w.Write(buf.Bytes()) + poolBuffer(buf) + return n, err +} + +// NOTE: keep in sync with writeQuoteString above. +func writeQuotedBytes(w io.Writer, s []byte) (int, error) { + buf := getBuffer() + buf.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if 0x20 <= b && b != '\\' && b != '"' { + i++ + continue + } + if start < i { + buf.Write(s[start:i]) + } + switch b { + case '\\', '"': + buf.WriteByte('\\') + buf.WriteByte(b) + case '\n': + buf.WriteByte('\\') + buf.WriteByte('n') + case '\r': + buf.WriteByte('\\') + buf.WriteByte('r') + case '\t': + buf.WriteByte('\\') + buf.WriteByte('t') + default: + // This encodes bytes < 0x20 except for \n, \r, and \t. + buf.WriteString(`\u00`) + buf.WriteByte(hex[b>>4]) + buf.WriteByte(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRune(s[i:]) + if c == utf8.RuneError { + if start < i { + buf.Write(s[start:i]) + } + buf.WriteString(`\ufffd`) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + buf.Write(s[start:]) + } + buf.WriteByte('"') + n, err := w.Write(buf.Bytes()) + poolBuffer(buf) + return n, err +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + r, err := strconv.ParseUint(string(s[2:6]), 16, 64) + if err != nil { + return -1 + } + return rune(r) +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { + return + } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rr := getu4(s[r:]) + if rr < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rr = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rr) + } + + // Quote, control characters are invalid. + case c == '"', c < ' ': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rr, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + return b[0:w], true +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 15d10b0..307cc93 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2,6 +2,8 @@ github.com/aybabtme/rgbterm # github.com/fatih/color v1.7.1-0.20180516100307-2d684516a886 github.com/fatih/color +# github.com/go-logfmt/logfmt v0.4.0 +github.com/go-logfmt/logfmt # github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 github.com/kr/logfmt # github.com/mattn/go-colorable v0.1.0 From 48887192df0eaa6e940a088ff7c76c4dcffe8388 Mon Sep 17 00:00:00 2001 From: Antoine Grondin Date: Tue, 10 Dec 2019 17:18:50 +0900 Subject: [PATCH 6/6] remove old logfmt parser code --- parser/logfmt/logfmt_parser.go | 109 -------- parser/logfmt/logfmt_parser_test.go | 391 ---------------------------- 2 files changed, 500 deletions(-) delete mode 100644 parser/logfmt/logfmt_parser.go delete mode 100644 parser/logfmt/logfmt_parser_test.go diff --git a/parser/logfmt/logfmt_parser.go b/parser/logfmt/logfmt_parser.go deleted file mode 100644 index 372408c..0000000 --- a/parser/logfmt/logfmt_parser.go +++ /dev/null @@ -1,109 +0,0 @@ -package logfmt - -import ( - "bytes" - "unicode" - "unicode/utf8" -) - -// Visitor receives key/value pairs as they are parse, returns `false` if -// it wishes to abort the parsing. -type Visitor func(key, val []byte) (more bool) - -// Parse does a best effort parsing of logfmt entries. If `allowEmptyKey`, -// it will parse ` =value` as `""=value`, where empty string is a valid key. -func Parse(data []byte, allowEmptyKey, keepGarbage bool, eachPair Visitor) bool { - // don't try to parse logfmt if there's no `mykey=` in the - // first few bytes - scanAllKeyValue(data, allowEmptyKey, keepGarbage, eachPair) - return true -} - -func scanAllKeyValue(data []byte, allowEmptyKey, keepGarbage bool, eachPair Visitor) { - i := 0 - firstKV := true - more := true - for i < len(data) && more { - keyStart, keyEnd, valStart, valEnd, found := scanKeyValue(data, i, allowEmptyKey) - if !found { - return - } - if firstKV { - firstKV = false - if keyStart != 0 && keepGarbage { - eachPair([]byte("garbage"), data[:keyStart]) - } - } - - if valStart == valEnd { - more = eachPair(data[keyStart:keyEnd], nil) - } else if data[valStart] == '"' { - more = eachPair(data[keyStart:keyEnd], data[valStart+1:valEnd-1]) - } else { - more = eachPair(data[keyStart:keyEnd], data[valStart:valEnd]) - } - i = valEnd + 1 - } - -} - -func scanKeyValue(data []byte, from int, allowEmptyKey bool) (keyStart, keyEnd, valStart, valEnd int, found bool) { - - keyStart, keyEnd, found = findWordFollowedBy('=', data, from, allowEmptyKey) - if !found { - return - } - valStart = keyEnd + 1 - if r, sz := utf8.DecodeRune(data[valStart:]); r == '"' { - // find next unescaped `"` - valEnd = findUnescaped('"', '\\', data, valStart+sz) - found = valEnd != -1 - valEnd++ - return - } - - nextKeyStart, _, nextFound := findWordFollowedBy('=', data, keyEnd+1, allowEmptyKey) - - if nextFound { - valEnd = nextKeyStart - 1 - } else { - valEnd = len(data) - } - - return -} - -func findWordFollowedBy(by rune, data []byte, from int, allowEmptyKey bool) (start int, end int, found bool) { - i := bytes.IndexRune(data[from:], by) - if i == -1 { - return i, i, false - } - i += from - // loop for all letters before the `by`, stop at the first space - for j := i - 1; j >= from; j-- { - if !utf8.RuneStart(data[j]) { - continue - } - r, _ := utf8.DecodeRune(data[j:]) - if unicode.IsSpace(r) { - j++ - return j, i, allowEmptyKey || j < i - } - } - return from, i, allowEmptyKey || from < i -} - -func findUnescaped(toFind, escape rune, data []byte, from int) int { - for i := from; i < len(data); { - r, sz := utf8.DecodeRune(data[i:]) - i += sz - if r == escape { - // skip next char - _, sz = utf8.DecodeRune(data[i:]) - i += sz - } else if r == toFind { - return i - sz - } - } - return -1 -} diff --git a/parser/logfmt/logfmt_parser_test.go b/parser/logfmt/logfmt_parser_test.go deleted file mode 100644 index 744ad0b..0000000 --- a/parser/logfmt/logfmt_parser_test.go +++ /dev/null @@ -1,391 +0,0 @@ -package logfmt - -import ( - "bytes" - "fmt" - "log" - "reflect" - "sort" - "testing" -) - -type kv struct{ key, val []byte } - -func (k kv) String() string { return string(k.key) + "=" + string(k.val) } - -type byKeyName []kv - -func (b *byKeyName) visit(key, val []byte) bool { - *b = append(*b, kv{key, val}) - log.Printf("visit(%q, %q) -> b=%v", string(key), string(val), b) - return true -} -func (b byKeyName) Len() int { return len(b) } -func (b byKeyName) Less(i, j int) bool { return bytes.Compare(b[i].key, b[j].key) == -1 } -func (b byKeyName) Swap(i, j int) { b[i], b[j] = b[j], b[i] } - -func TestScanKeyValue(t *testing.T) { - var tests = []struct { - input string - allowEmptyKey bool - keepGarbage bool - want []kv - }{ - { - input: "hello=bye", - want: []kv{ - {key: []byte("hello"), val: []byte("bye")}, - }, - }, - { - input: "hello=", - want: []kv{ - {key: []byte("hello"), val: nil}, - }, - }, - { - input: "hello= allo=more crap", - want: []kv{ - {key: []byte("hello"), val: nil}, - {key: []byte("allo"), val: []byte("more crap")}, - }, - }, - { - input: "hello=bye crap crap", - want: []kv{ - {key: []byte("hello"), val: []byte("bye crap crap")}, - }, - }, - { - input: "hello=bye crap crap ", - want: []kv{ - {key: []byte("hello"), val: []byte("bye crap crap ")}, - }, - }, - { - input: "hello=bye crap crap allo=more crap", - want: []kv{ - {key: []byte("hello"), val: []byte("bye crap crap")}, - {key: []byte("allo"), val: []byte("more crap")}, - }, - }, - { - input: `hello="bye crap crap" allo=more crap`, - want: []kv{ - {key: []byte("hello"), val: []byte("bye crap crap")}, - {key: []byte("allo"), val: []byte("more crap")}, - }, - }, - // { - // input: `hello="bye crap\" crap" allo=more crap`, - // want: []kv{ - // {key: []byte("hello"), val: []byte("bye crap\" crap")}, - // {key: []byte("allo"), val: []byte("more crap")}, - // }, - // }, - // { - // input: `hello="bye crap\\" allo=more crap`, - // want: []kv{ - // {key: []byte("hello"), val: []byte("bye crap\\")}, - // {key: []byte("allo"), val: []byte("more crap")}, - // }, - // }, - { - input: " hello=bye", - want: []kv{ - {key: []byte("hello"), val: []byte("bye")}, - }, - }, - { - input: " hello=bye crap crap", - want: []kv{ - {key: []byte("hello"), val: []byte("bye crap crap")}, - }, - }, - { - input: " hello=bye crap crap ", - want: []kv{ - {key: []byte("hello"), val: []byte("bye crap crap ")}, - }, - }, - { - input: " hello=bye crap crap allo=more crap", - want: []kv{ - {key: []byte("hello"), val: []byte("bye crap crap")}, - {key: []byte("allo"), val: []byte("more crap")}, - }, - }, - { - input: ` hello="bye crap crap" allo=more crap`, - want: []kv{ - {key: []byte("hello"), val: []byte("bye crap crap")}, - {key: []byte("allo"), val: []byte("more crap")}, - }, - }, - - { - input: ` hello="bye crap=crap" allo=more crap`, - want: []kv{ - {key: []byte("hello"), val: []byte("bye crap=crap")}, - {key: []byte("allo"), val: []byte("more crap")}, - }, - }, - { - input: ` hello="bye crap=crap" allo=more crap`, - keepGarbage: true, - want: []kv{ - {key: []byte("garbage"), val: []byte(" ")}, - {key: []byte("hello"), val: []byte("bye crap=crap")}, - {key: []byte("allo"), val: []byte("more crap")}, - }, - }, - - // sanitized real world input that breaks kr/logfmt - // { - // input: `time="Wed Nov 5 16:37:44 2014" pid="838383" level="1" version="ohwat" somekey="somevalue98754" msg="ohai_something --something_id='777262626' --else_id='67876789876' --something_name='derpderp' --else_flag='somevalue' --something='gyhjnhgvbhjnhuygvbhnjuygvbhnjygvbhnjkiuhygbhnjkmjhygtfrdedrtyuijkmnbvcfgyhujkmn bvftgyhujkm' --dritdirt='ghjkjhgcvghjkjhb' --hello_integer='1' --chienvache='7654' --drit_vache_spelling='romeo-julliet-111.1.2-190001010130.thing.things.thingz' --hellothing='watwat=herpherp,takcok=ff:ff:ff:ff:ff:ff,vif=animal67876789876,lolthing=127.0.0.1,halloweenmask=255.255.192.0,google_is_that_u=8.8.8.8,lolthingv6=,lolthingwatmask=,lolthingwatgw=,lolthingv4vlan=8888,lolthingwatland=' --hello_path_thingy_maybe='somevaluedirt1:/love_doge/dirt1\nsomevaluedirt2:/love_doge/dirt2\nsomevaluedirt3:/love_doge/dirt3\nsomevaluedirt4:/love_doge/dirt4\nsomevaluedirt5:/love_doge/dirt5\nsomevaluedirt6:/love_doge/dirt6\nsomevaluedirt7:/love_doge/dirt7\nsomevaluedirt8:/love_doge/dirt8\nsomevaluedirt9:/love_doge/dirt9\nsomevaluedirt10:/love_doge/dirt10\nsomevaluedirt11:/love_doge/dirt11\nsomevaluedirt12:/love_doge/dirt12\nsomevaluedirt13:/love_doge/dirt13\nsomevaluedirt14:/love_doge/dirt14\nsomevaluedirt15:/love_doge/dirt15' --hell_can_i_haz='9000' --cookies='42' --joy_disabled='0' --happyness_disabled='0' --dirtbagotry='canonical_canon=canoncanon@canon.canon\nfirst_landing_site=/\nlast_landing_site=/\nwatwat_something=true\njoy_score=0.448474747474747474\nmagic_thegathering=' --hello_thing_dirt='#itakcokomment\n\nlolwat: derpderp' --chien_vache_prerequis='dritthing=wat-thing,something=wat-thing,vachechien=wat-thing,oh_hai=wat-thing,happy_table=watdat'"`, - // want: []kv{ - // {key: []byte("time"), val: []byte("Wed Nov 5 16:37:44 2014")}, - // {key: []byte("pid"), val: []byte("838383")}, - // {key: []byte("level"), val: []byte("1")}, - // {key: []byte("version"), val: []byte("ohwat")}, - // {key: []byte("somekey"), val: []byte("somevalue98754")}, - // {key: []byte("msg"), val: []byte("ohai_something --something_id='777262626' --else_id='67876789876' --something_name='derpderp' --else_flag='somevalue' --something='gyhjnhgvbhjnhuygvbhnjuygvbhnjygvbhnjkiuhygbhnjkmjhygtfrdedrtyuijkmnbvcfgyhujkmn bvftgyhujkm' --dritdirt='ghjkjhgcvghjkjhb' --hello_integer='1' --chienvache='7654' --drit_vache_spelling='romeo-julliet-111.1.2-190001010130.thing.things.thingz' --hellothing='watwat=herpherp,takcok=ff:ff:ff:ff:ff:ff,vif=animal67876789876,lolthing=127.0.0.1,halloweenmask=255.255.192.0,google_is_that_u=8.8.8.8,lolthingv6=,lolthingwatmask=,lolthingwatgw=,lolthingv4vlan=8888,lolthingwatland=' --hello_path_thingy_maybe='somevaluedirt1:/love_doge/dirt1\nsomevaluedirt2:/love_doge/dirt2\nsomevaluedirt3:/love_doge/dirt3\nsomevaluedirt4:/love_doge/dirt4\nsomevaluedirt5:/love_doge/dirt5\nsomevaluedirt6:/love_doge/dirt6\nsomevaluedirt7:/love_doge/dirt7\nsomevaluedirt8:/love_doge/dirt8\nsomevaluedirt9:/love_doge/dirt9\nsomevaluedirt10:/love_doge/dirt10\nsomevaluedirt11:/love_doge/dirt11\nsomevaluedirt12:/love_doge/dirt12\nsomevaluedirt13:/love_doge/dirt13\nsomevaluedirt14:/love_doge/dirt14\nsomevaluedirt15:/love_doge/dirt15' --hell_can_i_haz='9000' --cookies='42' --joy_disabled='0' --happyness_disabled='0' --dirtbagotry='canonical_canon=canoncanon@canon.canon\nfirst_landing_site=/\nlast_landing_site=/\nwatwat_something=true\njoy_score=0.448474747474747474\nmagic_thegathering=' --hello_thing_dirt='#itakcokomment\n\nlolwat: derpderp' --chien_vache_prerequis='dritthing=wat-thing,something=wat-thing,vachechien=wat-thing,oh_hai=wat-thing,happy_table=watdat'")}, - // }, - // }, - } - - for n, tt := range tests { - name := fmt.Sprintf("%d", n) - t.Run(name, func(t *testing.T) { - var got byKeyName - if !Parse([]byte(tt.input), tt.allowEmptyKey, tt.keepGarbage, (&got).visit) { - t.Fatalf("should have been able to parse: %q", tt.input) - } - sort.Sort(byKeyName(tt.want)) - sort.Sort(got) - if !reflect.DeepEqual(tt.want, []kv(got)) { - t.Logf("want=%v", tt.want) - t.Logf(" got=%v", got) - t.Fatalf("different KVs for %q", tt.input) - } - }) - } -} - -func TestFindWordFollowedBy(t *testing.T) { - var tests = []struct { - input string - from int - found bool - allowEmptyKey bool - want string - }{ - { - input: "hello=bye", - from: 0, - found: true, - want: "hello", - }, - { - input: "hello=bye aloa allo=bye", - from: 6, - found: true, - want: "allo", - }, - { - input: "hello=bye aloa allo.fr=bye", - from: 6, - found: true, - want: "allo.fr", - }, - { - input: " hello=bye", - from: 0, - found: true, - want: "hello", - }, - { - input: " hello=bye", - from: 1, - found: true, - want: "hello", - }, - { - input: "allo hello=bye", - from: 0, - found: true, - want: "hello", - }, - { - input: "allo hello=bye", - from: 1, - found: true, - want: "hello", - }, - { - input: " allo hello=bye", - from: 0, - found: true, - want: "hello", - }, - { - input: " allo hello=bye", - from: 1, - found: true, - want: "hello", - }, - { - input: " hello ", - from: 0, - found: false, - }, - { - input: " hello ", - from: 1, - found: false, - }, - { - input: " hello =bye", - from: 0, - found: false, - }, - { - input: " hello =bye", - from: 0, - allowEmptyKey: true, - found: true, - want: "", - }, - { - input: "hello =bye", - from: 0, - found: false, - }, - { - input: "hello =bye", - from: 0, - allowEmptyKey: true, - found: true, - want: "", - }, - { - input: " =bye", - from: 0, - allowEmptyKey: true, - found: true, - want: "", - }, - { - input: "=bye", - from: 0, - found: false, - }, - { - input: "=bye", - from: 0, - allowEmptyKey: true, - found: true, - want: "", - }, - { - input: "", - from: 0, - found: false, - }, - { - input: "=", - from: 0, - found: false, - }, - { - input: "=", - from: 0, - allowEmptyKey: true, - found: true, - want: "", - }, - } - - for n, tt := range tests { - t.Logf("test #%d", n) - start, end, found := findWordFollowedBy('=', []byte(tt.input), tt.from, tt.allowEmptyKey) - if found != tt.found { - t.Errorf("want found %v, got %v", tt.found, found) - } - - if !found { - continue - } - - got := string([]byte(tt.input)[start:end]) - - if got != tt.want { - t.Fatalf("want start %q, got %q", tt.want, got) - } - - } -} - -func TestFindUnescaped(t *testing.T) { - var tests = []struct { - input string - find rune - escape rune - from int - found bool - wantRest string - }{ - { - input: "input", - find: '"', - escape: '\\', - from: 0, - found: false, - }, - { - input: `inp"ut`, - find: '"', - escape: '\\', - from: 0, - found: true, - wantRest: `"ut`, - }, - { - input: `inp\"ut`, - find: '"', - escape: '\\', - from: 0, - found: false, - }, - { - input: `inp\\"ut`, - find: '"', - escape: '\\', - from: 0, - found: true, - wantRest: `"ut`, - }, - { - input: `inp\\\"ut`, - find: '"', - escape: '\\', - from: 0, - found: false, - }, - } - - for n, tt := range tests { - t.Logf("test #%d", n) - idx := findUnescaped(tt.find, tt.escape, []byte(tt.input), tt.from) - if idx == -1 && tt.found { - t.Fatalf("should have found %q in %q", tt.wantRest, tt.input) - } - if !tt.found { - continue - } - gotRest := string([]byte(tt.input)[idx:]) - if tt.wantRest != gotRest { - t.Fatalf("want %q, got %q", tt.wantRest, gotRest) - } - } -}