Don't tokenizer ?. when followed by a digit

FIX: Don't consume `?.` tokens when followed by a digit. Closes #33
lezer-parser · Apr 23, 2024 · d31ea1a · d31ea1a
1 parent fca757b
commit d31ea1a
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 10 deletions.
diff --git a/src/javascript.grammar b/src/javascript.grammar
@@ -226,8 +226,8 @@ AmbientDeclaration {
 
 decoratorExpression {
   VariableName |
-  MemberExpression { decoratorExpression !member ("." | "?.") (PropertyName | PrivatePropertyName) } |
-  CallExpression { decoratorExpression !call TypeArgList? "?."? ArgList } |
+  MemberExpression { decoratorExpression !member ("." | questionDot) (PropertyName | PrivatePropertyName) } |
+  CallExpression { decoratorExpression !call TypeArgList? questionDot? ArgList } |
   ParenthesizedExpression
 }
 
@@ -299,7 +299,7 @@ expressionNoComma {
   ConditionalExpression { expressionNoComma !ternary questionOp expressionNoComma LogicOp<":"> expressionNoComma } |
   AssignmentExpression |
   PostfixExpression { expressionNoComma !postfix (incdec | LogicOp<"!">) } |
-  CallExpression { expressionNoComma !call TypeArgList? "?."? ArgList } |
+  CallExpression { expressionNoComma !call TypeArgList? questionDot? ArgList } |
   TaggedTemplateExpression { expressionNoComma !taggedTemplate TemplateString } |
   DynamicImport { kw<"import"> "(" expressionNoComma ")" } |
   ImportMeta { kw<"import"> "." PropertyName } |
@@ -378,7 +378,7 @@ AssignmentExpression {
 }
 
 MemberExpression {
-  expressionNoComma !member (("." | "?.") (PropertyName | PrivatePropertyName) | "?."? "[" expression "]")
+  expressionNoComma !member (("." | questionDot) (PropertyName | PrivatePropertyName) | questionDot? "[" expression "]")
 }
 
 ArgList {
@@ -400,7 +400,7 @@ TypeParamList {
 typeParam { TypeDefinition ~tsAngle (kw<"extends"> type)? ("=" type)? }
 
 typeofMemberExpression[@name=MemberExpression] {
-  VariableName !member (("." | "?.") PropertyName | "[" expression "]")
+  VariableName !member (("." | questionDot) PropertyName | "[" expression "]")
 }
 
 type[@isGroup=Type] {
@@ -600,9 +600,10 @@ intersectionOp[@name=LogicOp] { "&" }
 
 @external tokens noSemicolon from "./tokens" { noSemi }
 
-@external tokens incdecToken from "./tokens" {
+@external tokens operatorToken from "./tokens" {
  incdec[@name=ArithOp],
  incdecPrefix[@name=ArithOp]
+ questionDot[@name="?."]
 }
 
 @external tokens jsx from "./tokens" { JSXStartTag }
@@ -691,7 +692,7 @@ intersectionOp[@name=LogicOp] { "&" }
 
   "(" ")" "[" "]" "{" "}" "<" ">"
 
-  "?." "." "," ";" ":" "@"
+  "." "," ";" ":" "@"
 
   JSXIdentifier { $[A-Z_$\u{a1}-\u{10ffff}] (identifierChar | @digit | "-")* }
   JSXLowerIdentifier[@name=JSXIdentifier] { $[a-z] (identifierChar | @digit | "-")* }

diff --git a/src/tokens.js b/src/tokens.js
@@ -2,14 +2,15 @@
    expressed by lezer's built-in tokenizer. */
 
 import {ExternalTokenizer, ContextTracker} from "@lezer/lr"
-import {insertSemi, noSemi, incdec, incdecPrefix,
+import {insertSemi, noSemi, incdec, incdecPrefix, questionDot,
         spaces, newline, BlockComment, LineComment,
         JSXStartTag, Dialect_jsx} from "./parser.terms.js"
 
 const space = [9, 10, 11, 12, 13, 32, 133, 160, 5760, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200,
                8201, 8202, 8232, 8233, 8239, 8287, 12288]
 
-const braceR = 125, semicolon = 59, slash = 47, star = 42, plus = 43, minus = 45, lt = 60, comma = 44
+const braceR = 125, semicolon = 59, slash = 47, star = 42, plus = 43, minus = 45, lt = 60, comma = 44,
+      question = 63, dot = 46
 
 export const trackNewline = new ContextTracker({
   start: false,
@@ -33,7 +34,7 @@ export const noSemicolon = new ExternalTokenizer((input, stack) => {
     input.acceptToken(noSemi)
 }, {contextual: true})
 
-export const incdecToken = new ExternalTokenizer((input, stack) => {
+export const operatorToken = new ExternalTokenizer((input, stack) => {
   let {next} = input
   if (next == plus || next == minus) {
     input.advance()
@@ -42,6 +43,10 @@ export const incdecToken = new ExternalTokenizer((input, stack) => {
       let mayPostfix = !stack.context && stack.canShift(incdec)
       input.acceptToken(mayPostfix ? incdec : incdecPrefix)
     }
+  } else if (next == question && input.peek(1) == dot) {
+    input.advance(); input.advance()
+    if (input.next < 48 || input.next > 57) // No digit after
+      input.acceptToken(questionDot)
   }
 }, {contextual: true})
 

diff --git a/test/expression.txt b/test/expression.txt
@@ -676,3 +676,11 @@ Script(ExpressionStatement(ConditionalExpression(Number,LogicOp,Number,⚠)))
 ==>
 
 Script(ExpressionStatement(TemplateString(⚠)))
+
+# Ternary with leading-dot number
+
+a?.2:.3
+
+==>
+
+Script(ExpressionStatement(ConditionalExpression(VariableName,LogicOp,Number,LogicOp,Number)))