Merge pull request #10 from DataDog/zhengda.lu/operator

normalizer_test.go

@@ -388,6 +388,7 @@ func TestNormalizerFormatting(t *testing.T) {
 			queries: []string{
 				"SELECT id,name, address FROM users where id = ?",
 				"SELECT id, name, address FROM users where id = ?",
+				"SELECT id,name, address FROM users where id =?",
 				"SELECT id as ID, name as Name, address FROM users where id = ?",
 			},
 			expected: "SELECT id, name, address FROM users where id = ?",

sqllexer.go

@@ -320,8 +320,11 @@ func (s *Lexer) scanWhitespace() Token {
 
 func (s *Lexer) scanOperator() Token {
 	s.start = s.cursor
+	lastCh := s.peek()
 	ch := s.next()
-	for isOperator(ch) {
+	for isOperator(ch) && !(lastCh == '=' && ch == '?') {
+		// hack: we don't want to treat "=?" as an single operator
+		lastCh = ch
 		ch = s.next()
 	}
 	return Token{OPERATOR, s.src[s.start:s.cursor]}

sqllexer_test.go

@@ -449,6 +449,26 @@ func TestLexer(t *testing.T) {
 				{STRING, "'j\\'s'"},
 			},
 		},
+		{
+			name:  "select with escaped string",
+			input: "SELECT * FROM users where id =?",
+			expected: []Token{
+				{IDENT, "SELECT"},
+				{WS, " "},
+				{WILDCARD, "*"},
+				{WS, " "},
+				{IDENT, "FROM"},
+				{WS, " "},
+				{IDENT, "users"},
+				{WS, " "},
+				{IDENT, "where"},
+				{WS, " "},
+				{IDENT, "id"},
+				{WS, " "},
+				{OPERATOR, "="},
+				{OPERATOR, "?"},
+			},
+		},
 	}
 
 	for _, tt := range tests {

sqllexer_utils.go

@@ -67,7 +67,7 @@ func isLeadingSign(ch rune) bool {
 }
 
 func isLetter(ch rune) bool {
-	return unicode.IsLetter(ch) || ch == '_' || ch == '#'
+	return unicode.IsLetter(ch) || ch == '_'
 }
 
 func isDoubleQuote(ch rune) bool {