update

parser
pingcap · Sep 8, 2023 · b233acd · b233acd
1 parent ca69622
commit b233acd
Show file tree

Hide file tree

Showing 5 changed files with 264 additions and 14 deletions.
diff --git a/parser/ast/format_test.go b/parser/ast/format_test.go
@@ -88,8 +88,8 @@ func TestAstFormat(t *testing.T) {
 		expr := fmt.Sprintf("select %s", tt.input)
 		charset, collation := getDefaultCharsetAndCollate()
 		stmts, _, err := parser.New().Parse(expr, charset, collation)
-		node := stmts[0].(*ast.SelectStmt).Fields.Fields[0].Expr
 		require.NoError(t, err)
+		node := stmts[0].(*ast.SelectStmt).Fields.Fields[0].Expr
 
 		writer := bytes.NewBufferString("")
 		node.Format(writer)

diff --git a/parser/consistent_test.go b/parser/consistent_test.go
@@ -46,6 +46,12 @@ func TestKeywordConsistent(t *testing.T) {
 		requires.NotEqual(t, k, v)
 		requires.Equal(t, tokenMap[v], tokenMap[k])
 	}
+
+	requires.Len(t, reservedKeywords, len(reservedTokenMap))
+	for _, v := range reservedKeywords {
+		requires.NotNil(t, reservedTokenMap[v])
+	}
+
 	keywordCount := len(reservedKeywords) + len(unreservedKeywords) + len(notKeywordTokens) + len(tidbKeywords)
 	requires.Equal(t, keywordCount-len(windowFuncTokenMap), len(tokenMap)-len(aliases))
 

diff --git a/parser/misc.go b/parser/misc.go
@@ -142,6 +142,244 @@ func isInTokenMap(target string) bool {
 	return ok
 }
 
+func isReservedKeyWord(target string) bool {
+	_, ok := reservedTokenMap[target]
+	return ok
+}
+
+var reservedTokenMap = map[string]int{
+	"ADD":                 add,
+	"ALL":                 all,
+	"ALTER":               alter,
+	"ANALYZE":             analyze,
+	"AND":                 and,
+	"ARRAY":               array,
+	"AS":                  as,
+	"ASC":                 asc,
+	"BETWEEN":             between,
+	"BIGINT":              bigIntType,
+	"BINARY":              binaryType,
+	"BLOB":                blobType,
+	"BOTH":                both,
+	"BY":                  by,
+	"CALL":                call,
+	"CASCADE":             cascade,
+	"CASE":                caseKwd,
+	"CHANGE":              change,
+	"CHARACTER":           character,
+	"CHAR":                charType,
+	"CHECK":               check,
+	"COLLATE":             collate,
+	"COLUMN":              column,
+	"CONSTRAINT":          constraint,
+	"CONTINUE":            continueKwd,
+	"CONVERT":             convert,
+	"CREATE":              create,
+	"CROSS":               cross,
+	"CUME_DIST":           cumeDist,
+	"CURRENT_DATE":        currentDate,
+	"CURRENT_TIME":        currentTime,
+	"CURRENT_TIMESTAMP":   currentTs,
+	"CURRENT_USER":        currentUser,
+	"CURRENT_ROLE":        currentRole,
+	"CURSOR":              cursor,
+	"DATABASE":            database,
+	"DATABASES":           databases,
+	"DAY_HOUR":            dayHour,
+	"DAY_MICROSECOND":     dayMicrosecond,
+	"DAY_MINUTE":          dayMinute,
+	"DAY_SECOND":          daySecond,
+	"DECIMAL":             decimalType,
+	"DEFAULT":             defaultKwd,
+	"DELAYED":             delayed,
+	"DELETE":              deleteKwd,
+	"DENSE_RANK":          denseRank,
+	"DESC":                desc,
+	"DESCRIBE":            describe,
+	"DISTINCT":            distinct,
+	"DISTINCTROW":         distinctRow,
+	"DIV":                 div,
+	"DOUBLE":              doubleType,
+	"DROP":                drop,
+	"DUAL":                dual,
+	"ELSEIF":              elseIfKwd,
+	"ELSE":                elseKwd,
+	"ENCLOSED":            enclosed,
+	"ESCAPED":             escaped,
+	"EXISTS":              exists,
+	"EXIT":                exit,
+	"EXPLAIN":             explain,
+	"EXCEPT":              except,
+	"FALSE":               falseKwd,
+	"FETCH":               fetch,
+	"FIRST_VALUE":         firstValue,
+	"FLOAT":               floatType,
+	"FOR":                 forKwd,
+	"FORCE":               force,
+	"FOREIGN":             foreign,
+	"FROM":                from,
+	"FULLTEXT":            fulltext,
+	"GENERATED":           generated,
+	"GRANT":               grant,
+	"GROUP":               group,
+	"GROUPS":              groups,
+	"HAVING":              having,
+	"HIGH_PRIORITY":       highPriority,
+	"HOUR_MICROSECOND":    hourMicrosecond,
+	"HOUR_MINUTE":         hourMinute,
+	"HOUR_SECOND":         hourSecond,
+	"IF":                  ifKwd,
+	"IGNORE":              ignore,
+	"IN":                  in,
+	"INDEX":               index,
+	"INFILE":              infile,
+	"INNER":               inner,
+	"INOUT":               inout,
+	"INTEGER":             integerType,
+	"INTERSECT":           intersect,
+	"INTERVAL":            interval,
+	"INTO":                into,
+	"OUTFILE":             outfile,
+	"IS":                  is,
+	"INSERT":              insert,
+	"INT":                 intType,
+	"INT1":                int1Type,
+	"INT2":                int2Type,
+	"INT3":                int3Type,
+	"INT4":                int4Type,
+	"INT8":                int8Type,
+	"ITERATE":             iterate,
+	"JOIN":                join,
+	"KEY":                 key,
+	"KEYS":                keys,
+	"KILL":                kill,
+	"LAG":                 lag,
+	"LAST_VALUE":          lastValue,
+	"LEAD":                lead,
+	"LEADING":             leading,
+	"LEAVE":               leave,
+	"LEFT":                left,
+	"LIKE":                like,
+	"ILIKE":               ilike,
+	"LIMIT":               limit,
+	"LINES":               lines,
+	"LINEAR":              linear,
+	"LOAD":                load,
+	"LOCALTIME":           localTime,
+	"LOCALTIMESTAMP":      localTs,
+	"LOCK":                lock,
+	"LONGBLOB":            longblobType,
+	"LONGTEXT":            longtextType,
+	"LOW_PRIORITY":        lowPriority,
+	"MATCH":               match,
+	"MAXVALUE":            maxValue,
+	"MEDIUMBLOB":          mediumblobType,
+	"MEDIUMINT":           mediumIntType,
+	"MEDIUMTEXT":          mediumtextType,
+	"MINUTE_MICROSECOND":  minuteMicrosecond,
+	"MINUTE_SECOND":       minuteSecond,
+	"MOD":                 mod,
+	"NOT":                 not,
+	"NO_WRITE_TO_BINLOG":  noWriteToBinLog,
+	"NTH_VALUE":           nthValue,
+	"NTILE":               ntile,
+	"NULL":                null,
+	"NUMERIC":             numericType,
+	"OF":                  of,
+	"ON":                  on,
+	"OPTIMIZE":            optimize,
+	"OPTION":              option,
+	"OPTIONALLY":          optionally,
+	"OR":                  or,
+	"ORDER":               order,
+	"OUT":                 out,
+	"OUTER":               outer,
+	"OVER":                over,
+	"PARTITION":           partition,
+	"PERCENT_RANK":        percentRank,
+	"PRECISION":           precisionType,
+	"PRIMARY":             primary,
+	"PROCEDURE":           procedure,
+	"RANGE":               rangeKwd,
+	"RANK":                rank,
+	"READ":                read,
+	"REAL":                realType,
+	"RECURSIVE":           recursive,
+	"REFERENCES":          references,
+	"REGEXP":              regexpKwd,
+	"RELEASE":             release,
+	"RENAME":              rename,
+	"REPEAT":              repeat,
+	"REPLACE":             replace,
+	"REQUIRE":             require,
+	"RESTRICT":            restrict,
+	"REVOKE":              revoke,
+	"RIGHT":               right,
+	"RLIKE":               rlike,
+	"ROW":                 row,
+	"ROWS":                rows,
+	"ROW_NUMBER":          rowNumber,
+	"SECOND_MICROSECOND":  secondMicrosecond,
+	"SELECT":              selectKwd,
+	"SET":                 set,
+	"SHOW":                show,
+	"SMALLINT":            smallIntType,
+	"SPATIAL":             spatial,
+	"SQL":                 sql,
+	"SQL_BIG_RESULT":      sqlBigResult,
+	"SQL_CALC_FOUND_ROWS": sqlCalcFoundRows,
+	"SQL_SMALL_RESULT":    sqlSmallResult,
+	"SQLEXCEPTION":        sqlexception,
+	"SQLSTATE":            sqlstate,
+	"SQLWARNING":          sqlwarning,
+	"SSL":                 ssl,
+	"STARTING":            starting,
+	"STATS_EXTENDED":      statsExtended,
+	"STRAIGHT_JOIN":       straightJoin,
+	"TiDB_CURRENT_TSO":    tidbCurrentTSO,
+	"TABLE":               tableKwd,
+	"TABLESAMPLE":         tableSample,
+	"STORED":              stored,
+	"TERMINATED":          terminated,
+	"THEN":                then,
+	"TINYBLOB":            tinyblobType,
+	"TINYINT":             tinyIntType,
+	"TINYTEXT":            tinytextType,
+	"TO":                  to,
+	"TRAILING":            trailing,
+	"TRIGGER":             trigger,
+	"TRUE":                trueKwd,
+	"UNIQUE":              unique,
+	"UNION":               union,
+	"UNLOCK":              unlock,
+	"UNSIGNED":            unsigned,
+	"UNTIL":               until,
+	"UPDATE":              update,
+	"USAGE":               usage,
+	"USE":                 use,
+	"USING":               using,
+	"UTC_DATE":            utcDate,
+	"UTC_TIMESTAMP":       utcTimestamp,
+	"UTC_TIME":            utcTime,
+	"VALUES":              values,
+	"LONG":                long,
+	"VARCHAR":             varcharType,
+	"VARCHARACTER":        varcharacter,
+	"VARBINARY":           varbinaryType,
+	"VARYING":             varying,
+	"VIRTUAL":             virtual,
+	"WHEN":                when,
+	"WHERE":               where,
+	"WHILE":               while,
+	"WRITE":               write,
+	"WINDOW":              window,
+	"WITH":                with,
+	"XOR":                 xor,
+	"YEAR_MONTH":          yearMonth,
+	"ZEROFILL":            zerofill,
+	"NATURAL":             natural,
+}
+
 // tokenMap is a map of known identifiers to the parser token ID.
 // Please try to keep the map in alphabetical order.
 var tokenMap = map[string]int{
@@ -1066,20 +1304,10 @@ var hintTokenMap = map[string]int{
 func (s *Scanner) isTokenIdentifier(lit string, offset int) int {
 	// An identifier before or after '.' means it is part of a qualified identifier.
 	// We do not parse it as keyword.
-	if s.r.peek() == '.' {
+	if s.r.peek() == '.' || (offset != 0 && s.r.s[offset-1] == '.') {
 		return 0
 	}
 
-	for idx := offset - 1; idx >= 0; idx-- {
-		if s.r.s[idx] == ' ' {
-			continue
-		} else if s.r.s[idx] == '.' {
-			return 0
-		} else {
-			break
-		}
-	}
-
 	buf := &s.buf
 	buf.Reset()
 	buf.Grow(len(lit))
@@ -1094,6 +1322,20 @@ func (s *Scanner) isTokenIdentifier(lit string, offset int) int {
 		}
 	}
 
+	// select * from t where t. status = 1; -- parse ok, unreserved keyword
+	// select * from t where t. and = 1; -- parse failed, reserverd keyword.
+	if !isReservedKeyWord(string(data)) {
+		for idx := offset - 1; idx >= 0; idx-- {
+			if s.r.s[idx] == ' ' {
+				continue
+			} else if s.r.s[idx] == '.' {
+				return 0
+			} else {
+				break
+			}
+		}
+	}
+
 	checkBtFuncToken := s.r.peek() == '('
 	if !checkBtFuncToken && s.sqlMode.HasIgnoreSpaceMode() {
 		s.skipWhitespace()

diff --git a/parser/parser_test.go b/parser/parser_test.go
@@ -1059,6 +1059,8 @@ AAAAAAAAAAAA5gm5Mg==
 		{"select `t`.`1a`.1 from t;", true, "SELECT `t`.`1a`.`1` FROM `t`"},
 		{"select * from 1db.1table;", true, "SELECT * FROM `1db`.`1table`"},
 		{"select * from t where t. status = 1;", true, "SELECT * FROM `t` WHERE `t`.`status`=1"},
+		{"select * from t where t. and = 1;", false, ""},
+		{"select * from t where a between 1. and 2;", true, "SELECT * FROM `t` WHERE `a` BETWEEN 1. AND 2"},
 
 		// for show placement
 		{"SHOW PLACEMENT", true, "SHOW PLACEMENT"},

diff --git a/parser/test_driver/test_driver_mydecimal.go b/parser/test_driver/test_driver_mydecimal.go
@@ -124,7 +124,7 @@ func (d *MyDecimal) ToString() (str []byte) {
 	if d.negative {
 		length++
 	}
-	if digitsFrac > 0 {
+	if digitsFrac >= 0 {
 		length++
 	}
 	str = str[:length]
@@ -134,7 +134,7 @@ func (d *MyDecimal) ToString() (str []byte) {
 		strIdx++
 	}
 	var fill int
-	if digitsFrac > 0 {
+	if digitsFrac >= 0 {
 		fracIdx := strIdx + digitsIntLen
 		fill = digitsFracLen - digitsFrac
 		wordIdx := wordStartIdx + digitsToWords(digitsInt)