139-lex-refactoring

letheql/parser/lex.go

@@ -259,7 +259,7 @@ type Lexer struct {
 
 // next returns the next rune in the input.
 func (l *Lexer) next() rune {
-	if int(l.pos) >= len(l.input) {
+	if l.pos >= Pos(len(l.input)) {
 		l.width = 0
 		return eof
 	}
@@ -271,8 +271,10 @@ func (l *Lexer) next() rune {
 
 // peek returns but does not consume the next rune in the input.
 func (l *Lexer) peek() rune {
-	r := l.next()
-	l.backup()
+	if l.pos >= Pos(len(l.input)) {
+		return eof
+	}
+	r, _ := utf8.DecodeRuneInString(l.input[l.pos:])
 	return r
 }
 
@@ -313,9 +315,9 @@ func (l *Lexer) acceptRun(valid string) {
 // errorf returns an error token and terminates the scan by passing
 // back a nil pointer that will be the next state, terminating l.NextItem.
 func (l *Lexer) errorf(format string, args ...interface{}) stateFn {
-	*l.itemp = Item{ERROR, l.start, fmt.Sprintf(format, args...)}
+	message := fmt.Sprintf(format, args...)
+	*l.itemp = Item{ERROR, l.start, message}
 	l.scannedItem = true
-
 	return nil
 }
 
@@ -640,7 +642,11 @@ func digitVal(ch rune) int {
 
 // skipSpaces skips the spaces until a non-space is encountered.
 func skipSpaces(l *Lexer) {
-	for isSpace(l.peek()) {
+	for {
+		r := l.peek()
+		if !isSpace(r) {
+			break
+		}
 		l.next()
 	}
 	l.ignore()
@@ -804,24 +810,24 @@ func lexIdentifier(l *Lexer) stateFn {
 // a colon rune. If the identifier is a keyword the respective keyword Item
 // is scanned.
 func lexKeywordOrIdentifier(l *Lexer) stateFn {
-Loop:
 	for {
-		switch r := l.next(); {
-		case isAlphaNumeric(r) || r == ':':
-			// absorb.
-		default:
+		r := l.next()
+		if !isAlphaNumeric(r) && r != ':' {
 			l.backup()
-			word := l.input[l.start:l.pos]
-			if kw, ok := key[strings.ToLower(word)]; ok {
-				l.emit(kw)
-			} else if !strings.Contains(word, ":") {
-				l.emit(IDENTIFIER)
-			} else {
-				l.emit(METRIC_IDENTIFIER)
-			}
-			break Loop
+			break
 		}
 	}
+
+	word := l.input[l.start:l.pos]
+	switch {
+	case key[strings.ToLower(word)] != 0:
+		l.emit(key[strings.ToLower(word)])
+	case strings.Contains(word, ":"):
+		l.emit(METRIC_IDENTIFIER)
+	default:
+		l.emit(IDENTIFIER)
+	}
+
 	if l.seriesDesc && l.peek() != '{' {
 		return lexValueSequence
 	}