Debug JavaScript/Python in Enso Source

engine/runtime-language-epb/src/main/java/org/enso/interpreter/epb/ForeignEvalNode.java

@@ -31,23 +31,40 @@ static ForeignEvalNode parse(EpbLanguage epb, Source langAndCode, List<String> a
 
   private String truffleId(Source langAndCode) {
     var seq = langAndCode.getCharacters();
-    return seq.subSequence(0, splitAt(seq)).toString().toLowerCase();
+    var langAndLine = seq.subSequence(0, splitAt(seq, '#'));
+    var lang = langAndLine.subSequence(0, splitAt(langAndLine, ':'));
+    return lang.toString().toLowerCase();
   }
 
-  private int splitAt(CharSequence seq) {
+  private int startLine(Source langAndCode) {
+    var seq = langAndCode.getCharacters();
+    var langAndLine = seq.subSequence(0, splitAt(seq, '#')).toString();
+    var at = splitAt(langAndLine, ':') + 1;
+    var line = langAndLine.substring(at);
+    return Integer.parseInt(line);
+  }
+
+  private String foreignSource(Source langAndCode) {
+    var seq = langAndCode.getCharacters();
+    var code = new StringBuilder();
+    var line = startLine(langAndCode);
+    while (line-- > 0) {
+      code.append("\n");
+    }
+    var realCode = seq.toString().substring(splitAt(seq, '#') + 1);
+    code.append(realCode);
+    return code.toString();
+  }
+
+  private int splitAt(CharSequence seq, char ch) {
     var at = 0;
     while (at < seq.length()) {
-      if (seq.charAt(at) == '#') {
+      if (seq.charAt(at) == ch) {
         return at;
       }
       at++;
     }
-    throw new ForeignParsingException("No # found", this);
-  }
-
-  private String foreignSource(Source langAndCode) {
-    var seq = langAndCode.getCharacters();
-    return seq.toString().substring(splitAt(seq) + 1);
+    throw new ForeignParsingException("No " + ch + " found", this);
   }
 
   @Override
@@ -91,18 +108,23 @@ private ForeignFunctionCallNode parseJs() {
     var inner = context.getInnerContext();
     var code = foreignSource(langAndCode);
     var args = Arrays.stream(argNames).skip(1).collect(Collectors.joining(","));
-    var wrappedSrc = "var poly_enso_eval=function(" + args + "){\n" + code + "\n};poly_enso_eval";
-    Source source = Source.newBuilder("js", wrappedSrc, "").build();
+    var wrappedSrc = "var poly_enso_eval=function(" + args + "){" + code + "\n};poly_enso_eval";
+    Source source = newSource("js", wrappedSrc);
     var fn = inner.evalPublic(this, source);
     return JsForeignNode.build(fn);
   }
 
   private ForeignFunctionCallNode parseGeneric(
       String language, Function<CallTarget, ForeignFunctionCallNode> nodeFactory) {
     var ctx = EpbContext.get(this);
-    Source source =
-        Source.newBuilder(language, foreignSource(langAndCode), langAndCode.getName()).build();
+    Source source = newSource(language, foreignSource(langAndCode));
     CallTarget ct = ctx.getEnv().parsePublic(source, argNames);
     return nodeFactory.apply(ct);
   }
+
+  private Source newSource(String language, String code) {
+    return Source.newBuilder(language, code, langAndCode.getName())
+        .uri(langAndCode.getURI())
+        .build();
+  }
 }

engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala

@@ -1916,6 +1916,7 @@ class IrToTruffle(
           case Foreign.Definition(lang, code, _, _, _) =>
             buildForeignBody(
               lang,
+              body.location,
               code,
               arguments.map(_.name.name),
               argSlotIdxs
@@ -1977,12 +1978,18 @@ class IrToTruffle(
 
     private def buildForeignBody(
       language: String,
+      location: Option[IdentifiedLocation],
       code: String,
       argumentNames: List[String],
       argumentSlotIdxs: List[Int]
     ): RuntimeExpression = {
-      val src =
-        Source.newBuilder("epb", language + "#" + code, scopeName).build()
+      val line = location
+        .map(l => source.createSection(l.start, l.length).getStartLine())
+        .getOrElse(0)
+      val name = scopeName.replace('.', '_') + "." + language
+      val b    = Source.newBuilder("epb", language + ":" + line + "#" + code, name)
+      b.uri(source.getURI())
+      val src       = b.build()
       val foreignCt = context.parseInternal(src, argumentNames: _*)
       val argumentReaders = argumentSlotIdxs
         .map(slotIdx =>

tools/enso4igv/pom.xml

@@ -231,6 +231,13 @@
             <scope>test</scope>
             <type>jar</type>
         </dependency>
+        <dependency>
+            <groupId>org.netbeans.modules</groupId>
+            <artifactId>org-netbeans-modules-lexer-nbbridge</artifactId>
+            <version>${netbeans.version}</version>
+            <scope>test</scope>
+            <type>jar</type>
+        </dependency>
         <dependency>
             <groupId>org.netbeans.api</groupId>
             <artifactId>org-netbeans-modules-extexecution-base</artifactId>

tools/enso4igv/src/main/resources/org/enso/tools/enso4igv/enso.tmLanguage.json

@@ -85,6 +85,30 @@
 						"matches": "\\\""
 					}]
 				},
+				{
+					"name": "entity.name.function",
+					"contentName": "foreign.js",
+					"begin": "^(\\s*)(foreign)\\s+js\\s*([^\\s]+).*=(.*$)",
+					"end": "^(?!\\1\\s+)(?!\\s*$)",
+					"beginCaptures": {
+						"2" : { "name": "keyword.other" },
+						"3" : { "name": "entity.name.function"},
+						"4" : { "name": "header"}
+					},
+					"patterns": [ { "include": "source.js" }]
+				},
+				{
+					"name": "entity.name.function",
+					"contentName": "foreign.python",
+					"begin": "^(\\s*)(foreign)\\s+python\\s*([^\\s]+).*=(.*$)",
+					"end": "^(?!\\1\\s+)(?!\\s*$)",
+					"beginCaptures": {
+						"2" : { "name": "keyword.other" },
+						"3" : { "name": "entity.name.function"},
+						"4" : { "name": "header"}
+					},
+					"patterns": [ { "include": "source.python" }]
+				},
 				{
 					"contentName": "string.quoted.triple.begin",
 					"begin": "^(\\s*)(\"\"\"|''')",

tools/enso4igv/src/test/java/org/enso/tools/enso4igv/EnsoTextMateTest.java

@@ -0,0 +1,88 @@
+package org.enso.tools.enso4igv;
+
+import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import org.netbeans.api.lexer.Language;
+import org.netbeans.api.lexer.TokenId;
+import org.netbeans.api.lexer.TokenSequence;
+
+public class EnsoTextMateTest {
+
+    private static Language<? extends TokenId> ensoLanguage;
+
+    @BeforeClass
+    public static void findEnsoLanguage() {
+        ensoLanguage = org.netbeans.api.lexer.Language.find("application/x-enso");
+        assertNotNull("Needs org-netbeans-modules-lexer-nbbridge dependency", ensoLanguage);
+    }
+
+    @Test
+    public void testSimpleMain() {
+        var code = """
+                  main = 42
+                  """;
+        var hier = org.netbeans.api.lexer.TokenHierarchy.create(code, ensoLanguage);
+        assertNotNull(hier);
+        var seq = hier.tokenSequence();
+        assertEquals(6, seq.tokenCount());
+        assertNextToken(seq, "main", "entity.name.function");
+        assertNextToken(seq, " ", null);
+        assertNextToken(seq, "=", "keyword.operator");
+        assertNextToken(seq, " ", null);
+        assertNextToken(seq, "42", "constant.character.numeric");
+    }
+
+    @Test
+    public void testForeignJavaScriptFunction() {
+        var code = """
+                  foreign js dbg = '''
+                      debugger;
+                  """;
+        var hier = org.netbeans.api.lexer.TokenHierarchy.create(code, ensoLanguage);
+        assertNotNull(hier);
+        var seq = hier.tokenSequence();
+        assertEquals(4, seq.tokenCount());
+        assertNextToken(seq, "foreign js dbg = ", null);
+        assertNextToken(seq, "'''", "string.quoted.triple.begin");
+        assertNextToken(seq, "\n", null);
+        assertNextToken(seq, "    debugger;\n", null);
+        assertFalse("EOF", seq.moveNext());
+    }
+
+    private static void assertNextToken(TokenSequence<?> seq, String expectedText, String expectedCategory) {
+        try {
+            assertNextTokenImpl(seq, expectedText, expectedCategory);
+        } catch (AssertionError err) {
+            var sb = new StringBuilder();
+            sb.append(err.getMessage()).append("\n");
+            sb.append("Remaining tokens:\n");
+            seq.movePrevious();
+            while (seq.moveNext()) {
+                final Object categories = seq.token().getProperty("categories");
+                sb.append(seq.token().text()).append(" categories: ").append(categories).append("\n");
+            }
+            throw new AssertionError(sb.toString(), err);
+        }
+    }
+
+    private static void assertNextTokenImpl(TokenSequence<?> seq, String expectedText, String expectedCategory) {
+        assertTrue("Can move next", seq.moveNext());
+        var tok = seq.token();
+        assertNotNull("Token found", tok);
+        if (expectedText != null) {
+            assertEquals(expectedText, tok.text().toString());
+        }
+        if (expectedCategory != null) {
+            var categories = (List<String>) tok.getProperty("categories");
+            assertNotNull("Categories found", categories);
+            var at = categories.indexOf(expectedCategory);
+            assertNotEquals("Category " + expectedCategory + " found in " + categories, -1, at);
+        }
+    }
+}