Wrap field/slot references in <var class="field">

Fixes tc39#95
gibson042 · Jan 24, 2023 · 95783a9 · 95783a9
1 parent f49b108
commit 95783a9
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 18 deletions.
diff --git a/src/emitter.ts b/src/emitter.ts
@@ -7,6 +7,7 @@ import type {
   TagNode,
   UnderscoreNode,
   StarNode,
+  DoubleBracketsNode,
   OrderedListItemNode,
   UnorderedListItemNode,
   OrderedListNode,
@@ -72,6 +73,9 @@ export class Emitter {
       case 'tilde':
         this.emitTilde(node);
         break;
+      case 'double-brackets':
+        this.emitFieldOrSlot(node);
+        break;
       case 'comment':
       case 'tag':
       case 'opaqueTag':
@@ -125,6 +129,10 @@ export class Emitter {
     this.wrapFragment('var', node.contents);
   }
 
+  emitFieldOrSlot(node: DoubleBracketsNode) {
+    this.wrapFragment('var', node.contents, ' class="field"');
+  }
+
   emitTag(tag: OpaqueTagNode | CommentNode | TagNode) {
     this.str += tag.contents;
   }
@@ -159,9 +167,9 @@ export class Emitter {
     this.str += '>' + pipe.nonTerminal + '</emu-nt>';
   }
 
-  wrapFragment(wrapping: string, fragment: Node[]) {
-    this.str += `<${wrapping}>`;
+  wrapFragment(tagName: string, fragment: Node[], attrs: string = '') {
+    this.str += `<${tagName}${attrs}>`;
     this.emitFragment(fragment);
-    this.str += `</${wrapping}>`;
+    this.str += `</${tagName}>`;
   }
 }
diff --git a/src/node-types.ts b/src/node-types.ts
@@ -21,7 +21,7 @@ export type EOFToken = {
   location: LocationRange;
 };
 
-export type Format = 'star' | 'underscore' | 'tick' | 'pipe' | 'tilde';
+export type Format = 'star' | 'underscore' | 'tick' | 'pipe' | 'tilde' | 'double-brackets';
 
 export type FormatToken = {
   name: Format;
@@ -168,7 +168,19 @@ export type PipeNode = {
   location: LocationRange;
 };
 
-export type FormatNode = StarNode | UnderscoreNode | TickNode | TildeNode | PipeNode;
+export type DoubleBracketsNode = {
+  name: 'double-brackets';
+  contents: FragmentNode[];
+  location: LocationRange;
+};
+
+export type FormatNode =
+  | StarNode
+  | UnderscoreNode
+  | TickNode
+  | TildeNode
+  | PipeNode
+  | DoubleBracketsNode;
 
 export type UnorderedListNode = {
   name: 'ul';
@@ -213,6 +225,7 @@ export type Node =
   | TextNode
   | StarNode
   | UnderscoreNode
+  | DoubleBracketsNode
   | TickNode
   | TildeNode
   | PipeNode

diff --git a/src/parser.ts b/src/parser.ts
@@ -247,8 +247,8 @@ export class Parser {
       if (isFormatToken(tok)) {
         // check if format token is valid
         //
-        // tick is always valid
-        if (tok.name === 'tick') {
+        // tick and field/slot are always valid
+        if (tok.name === 'tick' || tok.name === 'double-brackets') {
           break;
         }
 
@@ -285,9 +285,15 @@ export class Parser {
 
   parseFormat(format: Format, opts: ParseFragmentOpts) {
     const startTok = this._t.next() as FormatToken;
+    const start = this.getPos(startTok);
     let contents: (TextNode | CommentNode | TagNode)[] = [];
 
-    if (startTok.name === 'underscore') {
+    if (format === 'double-brackets') {
+      // the tokenizer emits fields/slots as complete `[[...]]` tokens, which are preserved here
+      const location = { start, end: this.getPos() };
+      const text = { name: 'text', contents: startTok.contents, location };
+      return [{ name: format, contents: [text as TextNode], location }];
+    } else if (format === 'underscore') {
       if (this._t.peek().name === 'text') {
         contents = [this._t.next() as TextNode];
       }
@@ -296,7 +302,6 @@ export class Parser {
     }
 
     const nextTok = this._t.peek();
-    const start = this.getPos(startTok);
 
     // fragment ended but we don't have a close format. Convert this node into a text node.
     if (nextTok.name !== format) {
@@ -382,7 +387,8 @@ function isFormatToken(tok: Token): tok is FormatToken {
     tok.name === 'underscore' ||
     tok.name === 'tilde' ||
     tok.name === 'tick' ||
-    tok.name === 'pipe'
+    tok.name === 'pipe' ||
+    tok.name === 'double-brackets'
   );
 }
 
@@ -404,15 +410,15 @@ function isList(tok: Token): tok is OrderedListToken | UnorderedListToken {
 // Backtick can work anywhere, other format tokens have more stringent requirements.
 // This aligns with gmd semantics.
 function isValidStartFormat(prev: NotEOFToken, cur: Token, next: Token) {
-  if (cur.name === 'tick') {
+  if (cur.name === 'tick' || cur.name === 'double-brackets') {
     return true;
   }
 
   return !isAlphaNumeric(prev.contents[prev.contents.length - 1]) && !isWhitespace(next);
 }
 
 function isValidEndFormat(prev: Token, cur: Token) {
-  if (cur.name === 'tick') {
+  if (cur.name === 'tick' || cur.name === 'double-brackets') {
     return true;
   }
 

diff --git a/src/tokenizer.ts b/src/tokenizer.ts
@@ -1,5 +1,6 @@
 import type { Unlocated, Token, AttrToken, Position } from './node-types';
 
+const fieldOrSlotRegexp = /^\[\[(?:[^\\\]]|\\.)+\]\]/;
 const tagRegexp = /^<[/!]?(\w[\w-]*)(\s+\w[\w-]*(\s*=\s*("[^"]*"|'[^']*'|[^><"'=`]+))?)*\s*>/;
 const commentRegexp = /^<!--[\w\W]*?-->/;
 const attrRegexp = /^\[ *[\w-]+ *= *"(?:[^"\\\x00-\x1F]|\\["\\/bfnrt]|\\u[a-fA-F]{4})*" *(?:, *[\w-]+ *= *"(?:[^"\\\x00-\x1F]|\\["\\/bfnrt]|\\u[a-fA-F]{4})*" *)*] /;
@@ -79,6 +80,13 @@ export class Tokenizer {
       if (chr === '\\') {
         out += this.scanEscape();
       } else if (isChars(chr)) {
+        out += chr;
+        this.pos++;
+      } else if (chr === '[') {
+        if (this.tryScanFieldOrSlot()) {
+          break;
+        }
+
         out += chr;
         this.pos++;
       } else if (chr === '<') {
@@ -115,13 +123,20 @@ export class Tokenizer {
     return this.str.slice(start, this.pos);
   }
 
-  // does not actually consume the tag
-  // you should manually `this.pos += tag[0].length;` if you end up consuming it
-  tryScanTag() {
-    if (this.str[this.pos] !== '<') {
+  // does not actually consume the field/slot
+  // you should manually `this.pos += result.length;` if you end up consuming it
+  tryScanFieldOrSlot() {
+    const match = this.str.slice(this.pos).match(fieldOrSlotRegexp);
+    if (!match) {
       return;
     }
 
+    return match[0];
+  }
+
+  // does not actually consume the tag
+  // you should manually `this.pos += tag[0].length;` if you end up consuming it
+  tryScanTag() {
     const match = this.str.slice(this.pos).match(tagRegexp);
     if (!match) {
       return;
@@ -297,6 +312,16 @@ export class Tokenizer {
           } else if (isChars(chr)) {
             this.enqueue({ name: 'text', contents: this.scanChars() }, start);
             return;
+          } else if (chr === '[') {
+            const fieldOrSlot = this.tryScanFieldOrSlot();
+            if (fieldOrSlot) {
+              this.pos += fieldOrSlot.length;
+              this.enqueue({ name: 'double-brackets', contents: fieldOrSlot }, start);
+              return;
+            }
+
+            // didn't find a valid field/slot, so fall back to text.
+            this.enqueue({ name: 'text', contents: this.scanChars() }, start);
           } else if (chr === '<') {
             if (
               this.str[this.pos + 1] === '!' &&
@@ -436,5 +461,13 @@ function isChars(chr: string) {
 }
 
 function isFormat(chr: string) {
-  return chr === '*' || chr === '_' || chr === '`' || chr === '<' || chr === '|' || chr === '~';
+  return (
+    chr === '*' ||
+    chr === '_' ||
+    chr === '`' ||
+    chr === '[' ||
+    chr === '<' ||
+    chr === '|' ||
+    chr === '~'
+  );
 }
diff --git a/src/visitor.ts b/src/visitor.ts
@@ -11,6 +11,7 @@ const childKeys = {
   tick: ['contents'],
   tilde: ['contents'],
   pipe: [],
+  'double-brackets': ['contents'],
   ul: ['contents'],
   ol: ['contents'],
   'ordered-list-item': ['contents', 'sublist'],

diff --git a/test/cases/iterator-close.html b/test/cases/iterator-close.html
@@ -5,7 +5,7 @@
   <li>ReturnIfAbrupt(<var>hasReturn</var>).<ol>
       <li>If <var>hasReturn</var> is <emu-val>true</emu-val>, then<ol>
           <li>Let <var>innerResult</var> be Invoke(<var>iterator</var>, <code>"return"</code>, ( )).</li>
-          <li>If <var>completion</var>.[[type]] is not <emu-const>throw</emu-const> and <var>innerResult</var>.[[type]] is <emu-const>throw</emu-const>, then<ol>
+          <li>If <var>completion</var>.<var class="field">[[type]]</var> is not <emu-const>throw</emu-const> and <var>innerResult</var>.<var class="field">[[type]]</var> is <emu-const>throw</emu-const>, then<ol>
               <li>Return <var>innerResult</var>.</li>
             </ol>
           </li>