diff --git a/engine/lexer.go b/engine/lexer.go index 0a538146..5c0d7a65 100644 --- a/engine/lexer.go +++ b/engine/lexer.go @@ -701,7 +701,7 @@ func isLetterChar(r rune) bool { } func isSmallLetterChar(r rune) bool { - return unicode.In(r, unicode.Ll, unicode.Lo) + return unicode.In(r, unicode.Ll, unicode.Lo, unicode.Lm) } func isCapitalLetterChar(r rune) bool { diff --git a/engine/lexer_test.go b/engine/lexer_test.go index 85aa1520..a4bf81c9 100644 --- a/engine/lexer_test.go +++ b/engine/lexer_test.go @@ -39,6 +39,7 @@ func TestLexer_Token(t *testing.T) { {input: `프롤로그`, token: Token{Kind: TokenLetterDigit, Val: `프롤로그`}}, {input: `برولوغ`, token: Token{Kind: TokenLetterDigit, Val: `برولوغ`}}, {input: `פרולוג`, token: Token{Kind: TokenLetterDigit, Val: `פרולוג`}}, + {input: `ゴー`, token: Token{Kind: TokenLetterDigit, Val: `ゴー`}}, {input: `..`, token: Token{Kind: TokenGraphic, Val: `..`}}, {input: `#`, token: Token{Kind: TokenGraphic, Val: `#`}},