Merge pull request #1408 from nheir/language-cpp/add_universal

Add universal character names in identifiers to cpp parser
jplag · Dec 7, 2023 · 535e695 · 535e695
2 parents 2bba79f + 0d5fea4
commit 535e695
Showing 1 changed file with 34 additions and 1 deletion.
diff --git a/languages/cpp/src/main/javacc/CPP.jj b/languages/cpp/src/main/javacc/CPP.jj
@@ -9,6 +9,8 @@ options
   DEBUG_PARSER = false;
   DEBUG_LOOKAHEAD = false;
   DEBUG_TOKEN_MANAGER = false;
+  UNICODE_INPUT = true;
+  JAVA_UNICODE_ESCAPE = true;
 }
 
 PARSER_BEGIN(CPPScanner)
@@ -264,7 +266,38 @@ TOKEN :
 
 TOKEN :
 {
-  <  ID : ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* >
+  <  ID : (
+      ["a"-"z","A"-"Z", "_"]
+    | ["\u00A8", "\u00AA", "\u00AD", "\u00AF", "\u00B2"-"\u00B5", "\u00B7"-"\u00BA", "\u00BC"-"\u00BE", "\u00C0"-"\u00D6",
+    "\u00D8"-"\u00F6", "\u00F8"-"\u00FF"]
+    | ["\u0100"-"\u167F", "\u1681"-"\u180D", "\u180F"-"\u1FFF"]
+    | ["\u200B"-"\u200D", "\u202A"-"\u202E", "\u203F"-"\u2040", "\u2054", "\u2060"-"\u206F"]
+    | ["\u2070"-"\u218F", "\u2460"-"\u24FF", "\u2776"-"\u2793", "\u2C00"-"\u2DFF", "\u2E80"-"\u2FFF"]
+    | ["\u3004"-"\u3007", "\u3021"-"\u302F", "\u3031"-"\u303F"]
+    | ["\u3040"-"\uD7FF"]
+    | ["\uF900"-"\uFD3D", "\uFD40"-"\uFDCF", "\uFDF0"-"\uFE44", "\uFE47"-"\uFFFD"]
+    /* unsupported by javacc
+      | ["\u10000"-"\u1FFFD", "\u20000"-"\u2FFFD", "\u30000"-"\u3FFFD", "\u40000"-"\u4FFFD", "\u50000"-"\u5FFFD",
+      "\u60000"-"\u6FFFD", "\u70000"-"\u7FFFD", "\u80000"-"\u8FFFD", "\u90000"-"\u9FFFD", "\uA0000"-"\uAFFFD",
+      "\uB0000"-"\uBFFFD", "\uC0000"-"\uCFFFD", "\uD0000"-"\uDFFFD", "\uE0000"-"\uEFFFD"]
+    */
+   )
+   (
+    ["a"-"z","A"-"Z","0"-"9","_"]
+    | ["\u00A8", "\u00AA", "\u00AD", "\u00AF", "\u00B2"-"\u00B5", "\u00B7"-"\u00BA", "\u00BC"-"\u00BE", "\u00C0"-"\u00D6",
+    "\u00D8"-"\u00F6", "\u00F8"-"\u00FF"]
+    | ["\u0100"-"\u167F", "\u1681"-"\u180D", "\u180F"-"\u1FFF"]
+    | ["\u200B"-"\u200D", "\u202A"-"\u202E", "\u203F"-"\u2040", "\u2054", "\u2060"-"\u206F"]
+    | ["\u2070"-"\u218F", "\u2460"-"\u24FF", "\u2776"-"\u2793", "\u2C00"-"\u2DFF", "\u2E80"-"\u2FFF"]
+    | ["\u3004"-"\u3007", "\u3021"-"\u302F", "\u3031"-"\u303F"]
+    | ["\u3040"-"\uD7FF"]
+    | ["\uF900"-"\uFD3D", "\uFD40"-"\uFDCF", "\uFDF0"-"\uFE44", "\uFE47"-"\uFFFD"]
+    /* unsupported by javacc
+      | ["\u10000"-"\u1FFFD", "\u20000"-"\u2FFFD", "\u30000"-"\u3FFFD", "\u40000"-"\u4FFFD", "\u50000"-"\u5FFFD",
+      "\u60000"-"\u6FFFD", "\u70000"-"\u7FFFD", "\u80000"-"\u8FFFD", "\u90000"-"\u9FFFD", "\uA0000"-"\uAFFFD",
+      "\uB0000"-"\uBFFFD", "\uC0000"-"\uCFFFD", "\uD0000"-"\uDFFFD", "\uE0000"-"\uEFFFD"]
+    */
+   )* >
 }