Merge pull request #240 from ichiban/add-letter-modifier-to-small-let…

…ter-char add Lm (Letter, modifier) to small letter char
ichiban · Aug 6, 2022 · 596218f · 596218f
2 parents 8bbdd2c + 7ffc3d8
commit 596218f
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/engine/lexer.go b/engine/lexer.go
@@ -701,7 +701,7 @@ func isLetterChar(r rune) bool {
 }
 
 func isSmallLetterChar(r rune) bool {
-	return unicode.In(r, unicode.Ll, unicode.Lo)
+	return unicode.In(r, unicode.Ll, unicode.Lo, unicode.Lm)
 }
 
 func isCapitalLetterChar(r rune) bool {

diff --git a/engine/lexer_test.go b/engine/lexer_test.go
@@ -39,6 +39,7 @@ func TestLexer_Token(t *testing.T) {
 		{input: `프롤로그`, token: Token{Kind: TokenLetterDigit, Val: `프롤로그`}},
 		{input: `برولوغ`, token: Token{Kind: TokenLetterDigit, Val: `برولوغ`}},
 		{input: `פרולוג`, token: Token{Kind: TokenLetterDigit, Val: `פרולוג`}},
+		{input: `ゴー`, token: Token{Kind: TokenLetterDigit, Val: `ゴー`}},
 
 		{input: `..`, token: Token{Kind: TokenGraphic, Val: `..`}},
 		{input: `#`, token: Token{Kind: TokenGraphic, Val: `#`}},