Add regex support to query syntax (#164)

internal/pubsub/query/query.go

@@ -4,12 +4,11 @@
 // Query expressions describe properties of events and their attributes, using
 // strings like:
 //
-//    abci.invoice.number = 22 AND abci.invoice.owner = 'Ivan'
+//	abci.invoice.number = 22 AND abci.invoice.owner = 'Ivan'
 //
 // Query expressions can handle attribute values encoding numbers, strings,
 // dates, and timestamps.  The complete query grammar is described in the
 // query/syntax package.
-//
 package query
 
 import (
@@ -207,6 +206,7 @@ func parseNumber(s string) (float64, error) {
 // An entry does not exist if the combination is not valid.
 //
 // Disable the dupl lint for this map. The result isn't even correct.
+//
 //nolint:dupl
 var opTypeMap = map[syntax.Token]map[syntax.Token]func(interface{}) func(string) bool{
 	syntax.TContains: {
@@ -216,6 +216,14 @@ var opTypeMap = map[syntax.Token]map[syntax.Token]func(interface{}) func(string)
 			}
 		},
 	},
+	syntax.TMatches: {
+		syntax.TString: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				match, _ := regexp.MatchString(v.(string), s)
+				return match
+			}
+		},
+	},
 	syntax.TEq: {
 		syntax.TString: func(v interface{}) func(string) bool {
 			return func(s string) bool { return s == v.(string) }

internal/pubsub/query/query_test.go

@@ -12,14 +12,14 @@ import (
 )
 
 // Example events from the OpenAPI documentation:
-//  https://github.com/tendermint/tendermint/blob/master/rpc/openapi/openapi.yaml
+//
+//	https://github.com/tendermint/tendermint/blob/master/rpc/openapi/openapi.yaml
 //
 // Redactions:
 //
 //   - Add an explicit "tm" event for the built-in attributes.
 //   - Remove Index fields (not relevant to tests).
 //   - Add explicit balance values (to use in tests).
-//
 var apiEvents = []types.Event{
 	{
 		Type: "tm",
@@ -128,6 +128,12 @@ func TestCompiledMatches(t *testing.T) {
 		{`abci.owner.name CONTAINS 'Igor'`,
 			newTestEvents(`abci|owner.name=Pavel|owner.name=Ivan`),
 			false},
+		{`abci.owner.name MATCHES '.*or.*'`,
+			newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
+			true},
+		{`abci.owner.name MATCHES '.*or.*'`,
+			newTestEvents(`abci|owner.name=Pavel|owner.name=Ivan`),
+			false},
 		{`abci.owner.name = 'Igor'`,
 			newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
 			true},

internal/pubsub/query/syntax/parser.go

@@ -147,7 +147,7 @@ func (p *Parser) parseCond() (Condition, error) {
 		return cond, err
 	}
 	cond.Tag = p.scanner.Text()
-	if err := p.require(TLeq, TGeq, TLt, TGt, TEq, TContains, TExists); err != nil {
+	if err := p.require(TLeq, TGeq, TLt, TGt, TEq, TContains, TExists, TMatches); err != nil {
 		return cond, err
 	}
 	cond.Op = p.scanner.Token()
@@ -161,6 +161,8 @@ func (p *Parser) parseCond() (Condition, error) {
 		err = p.require(TNumber, TTime, TDate, TString)
 	case TContains:
 		err = p.require(TString)
+	case TMatches:
+		err = p.require(TString)
 	case TExists:
 		// no argument
 		return cond, nil

internal/pubsub/query/syntax/scanner.go

@@ -28,6 +28,7 @@ const (
 	TLeq             // operator: <=
 	TGt              // operator: >
 	TGeq             // operator: >=
+	TMatches         // operator: MATCHES
 
 	// Do not reorder these values without updating the scanner code.
 )
@@ -47,6 +48,7 @@ var tString = [...]string{
 	TLeq:      "<= operator",
 	TGt:       "> operator",
 	TGeq:      ">= operator",
+	TMatches:  "MATCHES operator",
 }
 
 func (t Token) String() string {
@@ -228,6 +230,8 @@ func (s *Scanner) scanTagLike(first rune) error {
 		s.tok = TExists
 	case "CONTAINS":
 		s.tok = TContains
+	case "MATCHES":
+		s.tok = TMatches
 	default:
 		s.tok = TTag
 	}

internal/pubsub/query/syntax/syntax_test.go

@@ -38,6 +38,7 @@ func TestScanner(t *testing.T) {
 		// Mixed values of various kinds.
 		{`x AND y`, []syntax.Token{syntax.TTag, syntax.TAnd, syntax.TTag}},
 		{`x.y CONTAINS 'z'`, []syntax.Token{syntax.TTag, syntax.TContains, syntax.TString}},
+		{`x.y MATCHES 'z'`, []syntax.Token{syntax.TTag, syntax.TMatches, syntax.TString}},
 		{`foo EXISTS`, []syntax.Token{syntax.TTag, syntax.TExists}},
 		{`and AND`, []syntax.Token{syntax.TTag, syntax.TAnd}},
 
@@ -128,6 +129,7 @@ func TestParseValid(t *testing.T) {
 		{"AND tm.events.type='NewBlock' ", false},
 
 		{"abci.account.name CONTAINS 'Igor'", true},
+		{"abci.account.name MATCHES '*go*'", true},
 
 		{"tx.date > DATE 2013-05-03", true},
 		{"tx.date < DATE 2013-05-03", true},

internal/state/indexer/block/kv/kv.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -415,6 +416,40 @@ func (idx *BlockerIndexer) match(
 			return nil, err
 		}
 
+	case c.Op == syntax.TMatches:
+		prefix, err := orderedcode.Append(nil, c.Tag)
+		if err != nil {
+			return nil, err
+		}
+
+		it, err := dbm.IteratePrefix(idx.store, prefix)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create prefix iterator: %w", err)
+		}
+		defer it.Close()
+
+	iterMatches:
+		for ; it.Valid(); it.Next() {
+			eventValue, err := parseValueFromEventKey(it.Key())
+			if err != nil {
+				continue
+			}
+
+			if match, _ := regexp.MatchString(c.Arg.Value(), eventValue); match {
+				tmpHeights[string(it.Value())] = it.Value()
+			}
+
+			select {
+			case <-ctx.Done():
+				break iterMatches
+
+			default:
+			}
+		}
+		if err := it.Error(); err != nil {
+			return nil, err
+		}
+
 	default:
 		return nil, errors.New("other operators should be handled already")
 	}

internal/state/indexer/block/kv/kv_test.go

@@ -120,6 +120,14 @@ func TestBlockIndexer(t *testing.T) {
 			q:       query.MustCompile(`finalize_event1.proposer CONTAINS 'FCAA001'`),
 			results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
 		},
+		"finalize_event.proposer MATCHES '.*FF.*'": {
+			q:       query.MustCompile(`finalize_event1.proposer MATCHES '.*FF.*'`),
+			results: []int64{},
+		},
+		"finalize_event.proposer MATCHES '.*F.*'": {
+			q:       query.MustCompile(`finalize_event1.proposer MATCHES '.*F.*'`),
+			results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
+		},
 	}
 
 	for name, tc := range testCases {

internal/state/indexer/tx/kv/kv.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -353,6 +354,34 @@ func (txi *TxIndex) match(
 		if err := it.Error(); err != nil {
 			panic(err)
 		}
+
+	case c.Op == syntax.TMatches:
+		it, err := dbm.IteratePrefix(txi.store, prefixFromCompositeKey(c.Tag))
+		if err != nil {
+			panic(err)
+		}
+		defer it.Close()
+
+	iterMatches:
+		for ; it.Valid(); it.Next() {
+			value, err := parseValueFromKey(it.Key())
+			if err != nil {
+				continue
+			}
+			if match, _ := regexp.MatchString(c.Arg.Value(), value); match {
+				tmpHashes[string(it.Value())] = it.Value()
+			}
+
+			// Potentially exit early.
+			select {
+			case <-ctx.Done():
+				break iterMatches
+			default:
+			}
+		}
+		if err := it.Error(); err != nil {
+			panic(err)
+		}
 	default:
 		panic("other operators should be handled already")
 	}

internal/state/indexer/tx/kv/kv_test.go

@@ -113,6 +113,12 @@ func TestTxSearch(t *testing.T) {
 		{"account.owner CONTAINS 'Vlad'", 0},
 		// search using the wrong key (of numeric type) using CONTAINS
 		{"account.number CONTAINS 'Iv'", 0},
+		// search using MATCHES
+		{"account.owner MATCHES '.*an.*'", 1},
+		// search for non existing value using MATCHES
+		{"account.owner MATCHES '.*lad'", 0},
+		// search using the wrong key (of numeric type) using MATCHES
+		{"account.number MATCHES '.*v.*'", 0},
 		// search using EXISTS
 		{"account.number EXISTS", 1},
 		// search using EXISTS for non existing key