Improve filters

core/alias.go

@@ -27,7 +27,7 @@ func (a alias) rewritePath(src string) string {
 
 func (a alias) rewriteLTSV(src *resource.Resource) (*resource.Resource, error) {
 	buf := &bytes.Buffer{}
-	lr := ltsv.NewReader(src)
+	lr := ltsv.NewReaderSize(src, 4096)
 	for {
 		s, err := lr.Read()
 		if err != nil {

filter/count/count.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 
 	"github.com/koron/nvgd/filter"
+	"github.com/koron/nvgd/internal/filterbase"
 	"github.com/koron/nvgd/resource"
 )
 
@@ -19,12 +20,16 @@ func newCount(r *resource.Resource, p filter.Params) (*resource.Resource, error)
 }
 
 type Count struct {
-	filter.Base
-	n int64
+	filterbase.Base
+	reader *filterbase.LineReader
+	n      int64
 }
 
 func New(r io.ReadCloser) *Count {
-	c := &Count{n: -1}
+	c := &Count{
+		reader: filterbase.NewLineReader(r),
+		n:      -1,
+	}
 	c.Base.Init(r, c.readNext)
 	return c
 }
@@ -35,7 +40,7 @@ func (c *Count) readNext(buf *bytes.Buffer) error {
 	}
 	c.n = 0
 	for {
-		raw, err := c.ReadLine()
+		raw, err := c.reader.ReadLine()
 		if err != nil && len(raw) == 0 {
 			if err == io.EOF {
 				if _, err := buf.WriteString(strconv.FormatInt(c.n, 10)); err != nil {

filter/count/count_test.go

@@ -0,0 +1,14 @@
+package count
+
+import (
+	"testing"
+
+	"github.com/koron/nvgd/filter"
+	"github.com/koron/nvgd/internal/filtertest"
+)
+
+func TestCount(t *testing.T) {
+	filtertest.Check(t, newCount, filter.Params{}, "a\nb\nc\nd\n", "4")
+	filtertest.Check(t, newCount, filter.Params{}, "a\nb\nc\nd", "4")
+	filtertest.Check(t, newCount, filter.Params{}, "a\nb\nc\n", "3")
+}

filter/cut/cut.go

@@ -10,12 +10,14 @@ import (
 	"strings"
 
 	"github.com/koron/nvgd/filter"
+	"github.com/koron/nvgd/internal/filterbase"
 	"github.com/koron/nvgd/resource"
 )
 
 // Cut represents cut filter.
 type Cut struct {
-	filter.Base
+	filterbase.Base
+	reader    *filterbase.LineReader
 	delim     []byte
 	splitter  SplitFunc
 	selectors []cutSelector
@@ -30,6 +32,7 @@ type cutWriter func(io.Writer, []byte) error
 // NewCut creates an instance of cut filter.
 func NewCut(r io.ReadCloser, delim []byte, selectors []cutSelector, splitFunc SplitFunc) *Cut {
 	f := &Cut{
+		reader:   filterbase.NewLineReader(r),
 		delim:    delim,
 		splitter: splitFunc,
 	}
@@ -49,7 +52,7 @@ func NewCut(r io.ReadCloser, delim []byte, selectors []cutSelector, splitFunc Sp
 }
 
 func (f *Cut) readNext(buf *bytes.Buffer) error {
-	raw, err := f.ReadLine()
+	raw, err := f.reader.ReadLine()
 	if err != nil && (err != io.EOF || len(raw) == 0) {
 		return err
 	}

filter/grep.go → filter/grep/grep.go

@@ -1,17 +1,21 @@
-package filter
+// Package grep provides "grep" filter for NVGD.
+package grep
 
 import (
 	"bytes"
 	"fmt"
 	"io"
 	"regexp"
 
+	"github.com/koron/nvgd/filter"
+	"github.com/koron/nvgd/internal/filterbase"
 	"github.com/koron/nvgd/resource"
 )
 
 // Grep represents grep like filter.
 type Grep struct {
-	Base
+	filterbase.Base
+	reader   *filterbase.LineReader
 	currLnum int
 
 	re    *regexp.Regexp
@@ -27,11 +31,12 @@ type Grep struct {
 // NewGrep creates an instance of grep filter.
 func NewGrep(r io.ReadCloser, re *regexp.Regexp, match bool, lf LineFilter, lnum bool, cnum int) *Grep {
 	g := &Grep{
-		re:    re,
-		match: match,
-		lf:    TrimEOL.Chain(lf),
-		lnum:  lnum,
-		cnum:  cnum,
+		reader: filterbase.NewLineReader(r),
+		re:     re,
+		match:  match,
+		lf:     TrimEOL.Chain(lf),
+		lnum:   lnum,
+		cnum:   cnum,
 	}
 	if cnum > 0 {
 		g.contextBefore = make([][]byte, 0, cnum)
@@ -42,7 +47,7 @@ func NewGrep(r io.ReadCloser, re *regexp.Regexp, match bool, lf LineFilter, lnum
 
 func (g *Grep) readNext(buf *bytes.Buffer) error {
 	for {
-		raw, err := g.ReadLine()
+		raw, err := g.reader.ReadLine()
 		if err != nil && len(raw) == 0 {
 			return err
 		}
@@ -87,7 +92,7 @@ func (g *Grep) output(buf *bytes.Buffer, lnum int, data []byte) error {
 	return err
 }
 
-func newGrep(r *resource.Resource, p Params) (*resource.Resource, error) {
+func newGrep(r *resource.Resource, p filter.Params) (*resource.Resource, error) {
 	re, err := regexp.Compile(p.String("re", ""))
 	if err != nil {
 		return nil, err
@@ -108,5 +113,5 @@ func newGrep(r *resource.Resource, p Params) (*resource.Resource, error) {
 }
 
 func init() {
-	MustRegister("grep", newGrep)
+	filter.MustRegister("grep", newGrep)
 }

filter/grep/grep_test.go

@@ -0,0 +1,48 @@
+package grep
+
+import (
+	"testing"
+
+	"github.com/koron/nvgd/filter"
+	"github.com/koron/nvgd/internal/filtertest"
+)
+
+func TestGrep(t *testing.T) {
+	filtertest.Check(t, newGrep,
+		filter.Params{"re": "foo"}, "aaa\nfoo\nbbb\n", "foo\n")
+
+	filtertest.Check(t, newGrep,
+		filter.Params{"re": "foo", "number": "true"},
+		"aaa\nfoo\nbbb\n",
+		"2: foo\n")
+	filtertest.Check(t, newGrep,
+		filter.Params{"re": "foo", "number": "true"},
+		"aaa\nfoo\nbbb\nfoo\nccc\n",
+		"2: foo\n4: foo\n")
+}
+
+func TestGrepContext(t *testing.T) {
+	filtertest.Check(t, newGrep,
+		filter.Params{"re": "eee", "context": "1"},
+		"aaa\nbbb\nccc\nddd\neee\nfff\nggg\nhhh\niii\n",
+		"ddd\neee\nfff\n")
+	filtertest.Check(t, newGrep,
+		filter.Params{"re": "eee", "context": "2"},
+		"aaa\nbbb\nccc\nddd\neee\nfff\nggg\nhhh\niii\n",
+		"ccc\nddd\neee\nfff\nggg\n")
+
+	filtertest.Check(t, newGrep,
+		filter.Params{"re": "eee", "context": "1", "number": "true"},
+		"aaa\nbbb\nccc\nddd\neee\nfff\nggg\nhhh\niii\n",
+		"4: ddd\n5: eee\n6: fff\n")
+
+	filtertest.Check(t, newGrep,
+		filter.Params{"re": "eee", "context": "2", "number": "true"},
+		"aaa\nbbb\nccc\nddd\neee\nfff\nggg\nhhh\niii\n",
+		"3: ccc\n4: ddd\n5: eee\n6: fff\n7: ggg\n")
+
+	filtertest.Check(t, newGrep,
+		filter.Params{"re": "XXX", "context": "2"},
+		"aaa\nbbb\nccc\nXXX\neee\nXXX\nggg\nhhh\niii\n",
+		"bbb\nccc\nXXX\neee\nXXX\nggg\nhhh\n")
+}

filter/line_filter.go → filter/grep/line_filter.go

@@ -1,4 +1,4 @@
-package filter
+package grep
 
 import "bytes"
 

filter/hash.go → filter/hash/hash.go

@@ -1,4 +1,5 @@
-package filter
+// Package hash provides a filter to calculate hashes like MD5, SHA1 or so.
+package hash
 
 import (
 	"bytes"
@@ -12,12 +13,14 @@ import (
 	"io"
 	"strings"
 
+	"github.com/koron/nvgd/filter"
+	"github.com/koron/nvgd/internal/filterbase"
 	"github.com/koron/nvgd/resource"
 )
 
 // Hash represents hash filter.
 type Hash struct {
-	Base
+	filterbase.Base
 	s   int
 	h   hash.Hash
 	enc hashEncoder
@@ -102,7 +105,7 @@ func hashEncBin(w io.Writer, b []byte) error {
 	return err
 }
 
-func newHash(r *resource.Resource, p Params) (*resource.Resource, error) {
+func newHash(r *resource.Resource, p filter.Params) (*resource.Resource, error) {
 	h, err := toHash(p.String("algorithm", "md5"))
 	if err != nil {
 		return nil, err
@@ -115,5 +118,5 @@ func newHash(r *resource.Resource, p Params) (*resource.Resource, error) {
 }
 
 func init() {
-	MustRegister("hash", newHash)
+	filter.MustRegister("hash", newHash)
 }

filter/hash/hash_test.go

@@ -0,0 +1,42 @@
+package hash
+
+import (
+	"testing"
+
+	"github.com/koron/nvgd/filter"
+	"github.com/koron/nvgd/internal/filtertest"
+)
+
+func TestHashFilter(t *testing.T) {
+	// encoding variations
+	filtertest.Check(t, newHash,
+		filter.Params{},
+		"",
+		"d41d8cd98f00b204e9800998ecf8427e")
+	filtertest.Check(t, newHash,
+		filter.Params{ "encoding": "base64", },
+		"",
+		"1B2M2Y8AsgTpgAmY7PhCfg==")
+	filtertest.Check(t, newHash,
+		filter.Params{ "encoding": "binary", },
+		"",
+		"\xd4\x1d\x8cُ\x00\xb2\x04\xe9\x80\t\x98\xec\xf8B~")
+
+	// algorithm variations
+	filtertest.Check(t, newHash,
+		filter.Params{ "algorithm": "sha1"},
+		"",
+		"da39a3ee5e6b4b0d3255bfef95601890afd80709")
+	filtertest.Check(t, newHash,
+		filter.Params{ "algorithm": "sha1"},
+		"",
+		"da39a3ee5e6b4b0d3255bfef95601890afd80709")
+	filtertest.Check(t, newHash,
+		filter.Params{ "algorithm": "sha256"},
+		"",
+		"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+	filtertest.Check(t, newHash,
+		filter.Params{ "algorithm": "sha512"},
+		"",
+		"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e")
+}