Skip to content

Commit

Permalink
1.2 Define tokens
Browse files Browse the repository at this point in the history
  • Loading branch information
cedrickchee committed Mar 26, 2020
1 parent d9674ff commit 136d6ff
Showing 1 changed file with 42 additions and 0 deletions.
42 changes: 42 additions & 0 deletions token/token.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package token

// Package token defines the tokens our lexer is going to output.

// There is a limited number of different token types in the Monkey language.
// That means we can define the possible TokenTypes as constants.
const (
// special type that signifies a token/character we don't know about
ILLEGAL = "ILLEGAL"
// special type stands for "end of file", which tells parser that it can stop
EOF = "EOF"

// Identifiers + literals
IDENT = "IDENT" // add, foobar, x, y, ...
INT = "INT" // 1343456

// Operators
ASSIGN = "="
PLUS = "+"

// Delimiters
COMMA = ","
SEMICOLON = ";"

LPAREN = "("
RPAREN = ")"
LBRACE = "{"
RBRACE = "}"

// Keywords
FUNCTION = "FUNCTION"
LET = "LET"
)

// TokenType distinguishes between different types of tokens.
type TokenType string

// Token holds a single token type and its literal value.
type Token struct {
Type TokenType
Literal string
}

0 comments on commit 136d6ff

Please sign in to comment.