From 4b5ad865110fad8e07e13a063088a1bf98630ad9 Mon Sep 17 00:00:00 2001
From: Marijn Haverbeke <marijn@haverbeke.nl>
Date: Mon, 9 Jan 2023 14:08:33 +0100
Subject: [PATCH] Allow use of token rules from other tokenizer blocks in local
 token blocks

---
 src/build.ts | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/build.ts b/src/build.ts
index 7143af3..d74b662 100644
--- a/src/build.ts
+++ b/src/build.ts
@@ -1415,8 +1415,12 @@ class TokenSet {
     if (expr instanceof NameExpression) {
       let name = expr.id.name, arg = args.find(a => a.name == name)
       if (arg) return this.build(arg.expr, from, to, arg.scope)
-      let rule = this.rules.find(r => r.id.name == name)
-      if (!rule) return this.b.raise(`Reference to rule '${expr.id.name}', which isn't found in this token group`, expr.start)
+      let rule
+      for (let i = 0, lt = this.b.localTokens; i <= lt.length; i++) {
+        let set = i == lt.length ? this.b.tokens : lt[i]
+        rule = set.rules.find(r => r.id.name == name)
+      }
+      if (!rule) return this.b.raise(`Reference to token rule '${expr.id.name}', which isn't found`, expr.start)
       this.buildRule(rule, expr, from, to, args)
     } else if (expr instanceof CharClass) {
       for (let [a, b] of CharClasses[expr.type]) from.edge(a, b, to)
@@ -1744,7 +1748,7 @@ class LocalTokenSet extends TokenSet {
       groupID: id,
       create: () => new LocalTokenGroup(fullData, precOffset, this.fallback ? this.fallback.id : undefined),
       createSource: importName =>
-        `new ${importName("LocalTokenGroup", "@lezer/lr")}(${JSON.stringify(encodeArray(fullData))}, ${precOffset}${
+        `new ${importName("LocalTokenGroup", "@lezer/lr")}(${encodeArray(fullData)}, ${precOffset}${
           this.fallback ? `, ${this.fallback.id}` : ''})`
     }
   }