Merge pull request #236 from ichiban/generalize-small-letter-char

generalize small letter char
ichiban · Jul 31, 2022 · 8bbdd2c · 8bbdd2c
2 parents ee3a8cc + 56a4e16
commit 8bbdd2c
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 6 deletions.
diff --git a/engine/lexer.go b/engine/lexer.go
@@ -680,7 +680,12 @@ func (l *Lexer) doubleQuotedListToken() Token {
 // Characters
 
 func isGraphicChar(r rune) bool {
-	return strings.ContainsRune(`#$&*+-./:<=>?@^~`, r)
+	return strings.ContainsRune(`#$&*+-./:<=>?@^~`, r) || unicode.In(r, &unicode.RangeTable{
+		R16: []unicode.Range16{
+			{Lo: 0x2200, Hi: 0x22FF, Stride: 1}, // Mathematical Operators
+			{Lo: 0x2A00, Hi: 0x2AFF, Stride: 1}, // Supplemental Mathematical Operators
+		},
+	})
 }
 
 func isAlphanumericChar(r rune) bool {
@@ -696,11 +701,7 @@ func isLetterChar(r rune) bool {
 }
 
 func isSmallLetterChar(r rune) bool {
-	return unicode.IsLower(r) || unicode.In(r,
-		unicode.Unified_Ideograph,
-		unicode.Hiragana,
-		unicode.Katakana,
-	)
+	return unicode.In(r, unicode.Ll, unicode.Lo)
 }
 
 func isCapitalLetterChar(r rune) bool {

diff --git a/engine/lexer_test.go b/engine/lexer_test.go
@@ -33,9 +33,18 @@ func TestLexer_Token(t *testing.T) {
 		{input: "/* comment ", token: Token{Kind: TokenInsufficient}},
 		{input: `/ *`, token: Token{Kind: TokenGraphic, Val: `/`}},
 
+		{input: `改善`, token: Token{Kind: TokenLetterDigit, Val: `改善`}},
+		{input: `プロログ`, token: Token{Kind: TokenLetterDigit, Val: `プロログ`}},
+		{input: `ぷろろぐ`, token: Token{Kind: TokenLetterDigit, Val: `ぷろろぐ`}},
+		{input: `프롤로그`, token: Token{Kind: TokenLetterDigit, Val: `프롤로그`}},
+		{input: `برولوغ`, token: Token{Kind: TokenLetterDigit, Val: `برولوغ`}},
+		{input: `פרולוג`, token: Token{Kind: TokenLetterDigit, Val: `פרולוג`}},
+
 		{input: `..`, token: Token{Kind: TokenGraphic, Val: `..`}},
 		{input: `#`, token: Token{Kind: TokenGraphic, Val: `#`}},
 		{input: `\`, token: Token{Kind: TokenGraphic, Val: `\`}},
+		{input: `∀`, token: Token{Kind: TokenGraphic, Val: `∀`}},
+		{input: `⨀`, token: Token{Kind: TokenGraphic, Val: `⨀`}},
 
 		{input: `'abc'`, token: Token{Kind: TokenQuoted, Val: "'abc'"}},
 		{input: `'don''t panic'`, token: Token{Kind: TokenQuoted, Val: "'don''t panic'"}},