-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
d9674ff
commit 136d6ff
Showing
1 changed file
with
42 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
package token | ||
|
||
// Package token defines the tokens our lexer is going to output. | ||
|
||
// There is a limited number of different token types in the Monkey language. | ||
// That means we can define the possible TokenTypes as constants. | ||
const ( | ||
// special type that signifies a token/character we don't know about | ||
ILLEGAL = "ILLEGAL" | ||
// special type stands for "end of file", which tells parser that it can stop | ||
EOF = "EOF" | ||
|
||
// Identifiers + literals | ||
IDENT = "IDENT" // add, foobar, x, y, ... | ||
INT = "INT" // 1343456 | ||
|
||
// Operators | ||
ASSIGN = "=" | ||
PLUS = "+" | ||
|
||
// Delimiters | ||
COMMA = "," | ||
SEMICOLON = ";" | ||
|
||
LPAREN = "(" | ||
RPAREN = ")" | ||
LBRACE = "{" | ||
RBRACE = "}" | ||
|
||
// Keywords | ||
FUNCTION = "FUNCTION" | ||
LET = "LET" | ||
) | ||
|
||
// TokenType distinguishes between different types of tokens. | ||
type TokenType string | ||
|
||
// Token holds a single token type and its literal value. | ||
type Token struct { | ||
Type TokenType | ||
Literal string | ||
} |