From 5a3c9dc1265bdc95f1f72c912b61bb013e197d7d Mon Sep 17 00:00:00 2001 From: Alexander J Trelore Date: Mon, 16 Jan 2023 13:04:18 +0000 Subject: [PATCH] added option to specify size of decoder --- decode.go | 17 +++++ decode_test.go | 192 +++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 172 insertions(+), 37 deletions(-) diff --git a/decode.go b/decode.go index 2013708..a1c22dc 100644 --- a/decode.go +++ b/decode.go @@ -29,6 +29,23 @@ func NewDecoder(r io.Reader) *Decoder { return dec } +// NewDecoderSize returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may read data from r beyond +// the logfmt records requested. +// The size argument specifies the size of the initial buffer that the +// Decoder will use to read records from r. +// If a log line is longer than the size argument, the Decoder will return +// a bufio.ErrTooLong error. +func NewDecoderSize(r io.Reader, size int) *Decoder { + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, size), size) + dec := &Decoder{ + s: scanner, + } + return dec +} + // ScanRecord advances the Decoder to the next record, which can then be // parsed with the ScanKeyval method. It returns false when decoding stops, // either by reaching the end of the input or an error. After ScanRecord diff --git a/decode_test.go b/decode_test.go index 363152d..9d85ec4 100644 --- a/decode_test.go +++ b/decode_test.go @@ -1,6 +1,7 @@ package logfmt import ( + "bufio" "bytes" "fmt" "reflect" @@ -17,21 +18,56 @@ func (s kv) String() string { } func TestDecoder_scan(t *testing.T) { + defaultDecoder := func(s string) *Decoder { return NewDecoder(strings.NewReader(s)) } tests := []struct { data string + dec func(string) *Decoder want [][]kv }{ - {"", nil}, - {"\n\n", [][]kv{nil, nil}}, - {`x= `, [][]kv{{{[]byte("x"), nil}}}}, - {`y=`, [][]kv{{{[]byte("y"), nil}}}}, - {`y`, [][]kv{{{[]byte("y"), nil}}}}, - {`y=f`, [][]kv{{{[]byte("y"), []byte("f")}}}}, - {"y=\"\\tf\"", [][]kv{{{[]byte("y"), []byte("\tf")}}}}, - {"a=1\n", [][]kv{{{[]byte("a"), []byte("1")}}}}, - { - `a=1 b="bar" ƒ=2h3s r="esc\t" d x=sf `, - [][]kv{{ + { + data: "", + dec: defaultDecoder, + want: nil, + }, + { + data: "\n\n", + dec: defaultDecoder, + want: [][]kv{nil, nil}, + }, + { + data: `x= `, + dec: defaultDecoder, + want: [][]kv{{{[]byte("x"), nil}}}, + }, + { + data: `y=`, + dec: defaultDecoder, + want: [][]kv{{{[]byte("y"), nil}}}, + }, + { + data: `y`, + dec: defaultDecoder, + want: [][]kv{{{[]byte("y"), nil}}}, + }, + { + data: `y=f`, + dec: defaultDecoder, + want: [][]kv{{{[]byte("y"), []byte("f")}}}, + }, + { + data: "y=\"\\tf\"", + dec: defaultDecoder, + want: [][]kv{{{[]byte("y"), []byte("\tf")}}}, + }, + { + data: "a=1\n", + dec: defaultDecoder, + want: [][]kv{{{[]byte("a"), []byte("1")}}}, + }, + { + data: `a=1 b="bar" ƒ=2h3s r="esc\t" d x=sf `, + dec: defaultDecoder, + want: [][]kv{{ {[]byte("a"), []byte("1")}, {[]byte("b"), []byte("bar")}, {[]byte("ƒ"), []byte("2h3s")}, @@ -41,46 +77,62 @@ func TestDecoder_scan(t *testing.T) { }}, }, { - "y=f\ny=g", - [][]kv{ + data: "y=f\ny=g", + dec: defaultDecoder, + want: [][]kv{ {{[]byte("y"), []byte("f")}}, {{[]byte("y"), []byte("g")}}, }, }, { - "y=f \n\x1e y=g", - [][]kv{ + data: "y=f \n\x1e y=g", + dec: defaultDecoder, + want: [][]kv{ {{[]byte("y"), []byte("f")}}, {{[]byte("y"), []byte("g")}}, }, }, { - "y= d y=g", - [][]kv{{ + data: "y= d y=g", + dec: defaultDecoder, + want: [][]kv{{ {[]byte("y"), nil}, {[]byte("d"), nil}, {[]byte("y"), []byte("g")}, }}, }, { - "y=\"f\"\ny=g", - [][]kv{ + data: "y=\"f\"\ny=g", + dec: defaultDecoder, + want: [][]kv{ {{[]byte("y"), []byte("f")}}, {{[]byte("y"), []byte("g")}}, }, }, { - "y=\"f\\n\"y=g", - [][]kv{{ + data: "y=\"f\\n\"y=g", + dec: defaultDecoder, + want: [][]kv{{ {[]byte("y"), []byte("f\n")}, {[]byte("y"), []byte("g")}, }}, }, + { + data: strings.Repeat(`y=f `, 5), + dec: func(s string) *Decoder { return NewDecoderSize(strings.NewReader(s), 21) }, + want: [][]kv{{ + {[]byte("y"), []byte("f")}, + {[]byte("y"), []byte("f")}, + {[]byte("y"), []byte("f")}, + {[]byte("y"), []byte("f")}, + {[]byte("y"), []byte("f")}, + }}, + }, } for _, test := range tests { var got [][]kv - dec := NewDecoder(strings.NewReader(test.data)) + dec := test.dec(test.data) for dec.ScanRecord() { var kvs []kv @@ -103,28 +155,94 @@ func TestDecoder_scan(t *testing.T) { } func TestDecoder_errors(t *testing.T) { + defaultDecoder := func(s string) *Decoder { return NewDecoder(strings.NewReader(s)) } tests := []struct { data string + dec func(string) *Decoder want error }{ - {"a=1\n=bar", &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 1}}, - {"a=1\n\"k\"=bar", &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 1}}, - {"a=1\nk\"ey=bar", &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 2}}, - {"a=1\nk=b\"ar", &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 4}}, - {"a=1\nk=b =ar", &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 5}}, - {"a==", &SyntaxError{Msg: "unexpected '='", Line: 1, Pos: 3}}, - {"a=1\nk=b=ar", &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 4}}, - {"a=\"1", &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 5}}, - {"a=\"1\\", &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 6}}, - {"a=\"\\t1", &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 7}}, - {"a=\"\\u1\"", &SyntaxError{Msg: "invalid quoted value", Line: 1, Pos: 8}}, - {"a\ufffd=bar", &SyntaxError{Msg: "invalid key", Line: 1, Pos: 5}}, - {"\x80=bar", &SyntaxError{Msg: "invalid key", Line: 1, Pos: 2}}, - {"\x80", &SyntaxError{Msg: "invalid key", Line: 1, Pos: 2}}, + { + data: "a=1\n=bar", + dec: defaultDecoder, + want: &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 1}, + }, + { + data: "a=1\n\"k\"=bar", + dec: defaultDecoder, + want: &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 1}, + }, + { + data: "a=1\nk\"ey=bar", + dec: defaultDecoder, + want: &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 2}, + }, + { + data: "a=1\nk=b\"ar", + dec: defaultDecoder, + want: &SyntaxError{Msg: "unexpected '\"'", Line: 2, Pos: 4}, + }, + { + data: "a=1\nk=b =ar", + dec: defaultDecoder, + want: &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 5}, + }, + { + data: "a==", + dec: defaultDecoder, + want: &SyntaxError{Msg: "unexpected '='", Line: 1, Pos: 3}, + }, + { + data: "a=1\nk=b=ar", + dec: defaultDecoder, + want: &SyntaxError{Msg: "unexpected '='", Line: 2, Pos: 4}, + }, + { + data: "a=\"1", + dec: defaultDecoder, + want: &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 5}, + }, + { + data: "a=\"1\\", + dec: defaultDecoder, + want: &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 6}, + }, + { + data: "a=\"\\t1", + dec: defaultDecoder, + want: &SyntaxError{Msg: "unterminated quoted value", Line: 1, Pos: 7}, + }, + { + data: "a=\"\\u1\"", + dec: defaultDecoder, + want: &SyntaxError{Msg: "invalid quoted value", Line: 1, Pos: 8}, + }, + { + data: "a\ufffd=bar", + dec: defaultDecoder, + want: &SyntaxError{Msg: "invalid key", Line: 1, Pos: 5}, + }, + { + data: "\x80=bar", + dec: defaultDecoder, + want: &SyntaxError{Msg: "invalid key", Line: 1, Pos: 2}, + }, + { + data: "\x80", + dec: defaultDecoder, + want: &SyntaxError{Msg: "invalid key", Line: 1, Pos: 2}, + }, + { + data: "a=1\nb=2", + dec: func(s string) *Decoder { + dec := NewDecoderSize(strings.NewReader(s), 1) + return dec + }, + want: bufio.ErrTooLong, + }, } for _, test := range tests { - dec := NewDecoder(strings.NewReader(test.data)) + dec := test.dec(test.data) for dec.ScanRecord() { for dec.ScanKeyval() {