feat: java parser ffi

app/gui2/vite.config.ts

@@ -2,7 +2,7 @@
 
 import vue from '@vitejs/plugin-vue'
 import { getDefines, readEnvironmentFromFile } from 'enso-common/src/appConfig'
-import { spawn, spawnSync } from 'node:child_process'
+import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
 import { existsSync } from 'node:fs'
 import { fileURLToPath } from 'node:url'
 import postcssNesting from 'postcss-nesting'
@@ -97,6 +97,7 @@ export default defineConfig({
   },
 })
 
+let ydocServer: ChildProcessWithoutNullStreams | null;
 function gatewayServer(): Plugin {
   return {
     name: 'gateway-server',
@@ -111,10 +112,13 @@ function gatewayServer(): Plugin {
           ),
         )
         const runYdocServer = () => {
-          const ydocServer = spawn('java', ['-jar', ydocServerJar])
+          ydocServer = spawn('java', ['-jar', ydocServerJar])
           ydocServer.stdout.on('data', (data) => {
             console.log(`ydoc: ${data}`)
           })
+          ydocServer.stderr.on('data', (data) => {
+            console.log(`ydoc: ${data}`)
+          })
         }
         if (!existsSync(ydocServerJar)) {
           const cwd = fileURLToPath(new URL('../..', import.meta.url))
@@ -130,6 +134,11 @@ function gatewayServer(): Plugin {
         createGatewayServer(server.httpServer, undefined)
       }
     },
+    buildEnd() {
+      if (ydocServer == null) return;
+
+      ydocServer.kill(9)
+    }
   }
 }
 

build.sbt

@@ -667,12 +667,12 @@ def generateRustParser(
 
 lazy val `syntax-rust-definition` = project
   .in(file("lib/rust/parser"))
+  .enablePlugins(JPMSPlugin)
   .configs(Test)
   .settings(
     Compile / sourceGenerators += generateParserJavaSources,
     Compile / resourceGenerators += generateRustParserLib,
-    Compile / javaSource := baseDirectory.value / "generate-java" / "java",
-    frgaalJavaCompilerSetting
+    Compile / javaSource := baseDirectory.value / "generate-java" / "java"
   )
 
 lazy val pkg = (project in file("lib/scala/pkg"))
@@ -1136,21 +1136,21 @@ lazy val `ydoc-server` = project
         }
       )
       .evaluated,
-      assembly / assemblyMergeStrategy := {
-        case PathList("META-INF", file, xs @ _*) if file.endsWith(".DSA") =>
-          MergeStrategy.discard
-        case PathList("META-INF", file, xs @ _*) if file.endsWith(".SF") =>
-          MergeStrategy.discard
-        case PathList("META-INF", "MANIFEST.MF", xs @ _*) =>
-          MergeStrategy.discard
-        case PathList("META-INF", "services", xs @ _*) =>
-          MergeStrategy.concat
-        case PathList("module-info.class") =>
-          MergeStrategy.preferProject
-        case PathList(xs @ _*) if xs.last.contains("module-info.class") =>
-          MergeStrategy.discard
-        case _ => MergeStrategy.first
-      },
+    assembly / assemblyMergeStrategy := {
+      case PathList("META-INF", file, xs @ _*) if file.endsWith(".DSA") =>
+        MergeStrategy.discard
+      case PathList("META-INF", file, xs @ _*) if file.endsWith(".SF") =>
+        MergeStrategy.discard
+      case PathList("META-INF", "MANIFEST.MF", xs @ _*) =>
+        MergeStrategy.discard
+      case PathList("META-INF", "services", xs @ _*) =>
+        MergeStrategy.concat
+      case PathList("module-info.class") =>
+        MergeStrategy.preferProject
+      case PathList(xs @ _*) if xs.last.contains("module-info.class") =>
+        MergeStrategy.discard
+      case _ => MergeStrategy.first
+    },
     commands += WithDebugCommand.withDebug,
     modulePath := {
       JPMSUtils.filterModulesFromUpdate(
@@ -1185,6 +1185,7 @@ lazy val `ydoc-server` = project
         shouldContainAll = true
       )
     },
+    modulePath += (`syntax-rust-definition` / Compile / productDirectories).value.head,
     libraryDependencies ++= Seq(
       "org.graalvm.polyglot" % "polyglot"                    % graalMavenPackagesVersion,
       "org.graalvm.polyglot" % "inspect"                     % graalMavenPackagesVersion % "runtime",
@@ -1195,6 +1196,7 @@ lazy val `ydoc-server` = project
       "com.github.sbt"       % "junit-interface"             % junitIfVersion            % Test
     )
   )
+  .dependsOn(`syntax-rust-definition`)
 
 lazy val `persistance` = (project in file("lib/java/persistance"))
   .settings(

lib/java/ydoc-server/src/main/java/module-info.java

@@ -1,4 +1,5 @@
-module YdocServer {
+module org.enso.ydoc {
+  requires org.enso.syntax;
   requires org.graalvm.polyglot;
   requires io.helidon.webclient;
   requires io.helidon.webclient.websocket;

lib/java/ydoc-server/src/main/java/org/enso/ydoc/polyfill/ParserPolyfill.java

@@ -0,0 +1,73 @@
+package org.enso.ydoc.polyfill;
+
+import java.util.Arrays;
+import org.enso.syntax2.Parser;
+import org.enso.ydoc.Polyfill;
+import org.graalvm.polyglot.Context;
+import org.graalvm.polyglot.Source;
+import org.graalvm.polyglot.Value;
+import org.graalvm.polyglot.proxy.ProxyExecutable;
+
+public final class ParserPolyfill implements AutoCloseable, ProxyExecutable, Polyfill {
+
+  private static final String PARSE_TREE = "parse-tree";
+  private static final String XX_HASH_128 = "xx-hash-128";
+  private static final String IS_IDENT_OR_OPERATOR = "is-ident-or-operator";
+
+  private static final String PARSER_JS = "parser.js";
+
+  private final Parser parser;
+
+  ParserPolyfill() {
+    Parser p;
+    try {
+      p = Parser.create();
+    } catch (LinkageError err) {
+      err.printStackTrace();
+      throw err;
+    }
+    this.parser = p;
+
+  }
+
+  @Override
+  public void initialize(Context ctx) {
+    Source parserJs =
+        Source.newBuilder("js", ParserPolyfill.class.getResource(PARSER_JS)).buildLiteral();
+
+    ctx.eval(parserJs).execute(this);
+  }
+
+  @Override
+  public Object execute(Value... arguments) {
+    var command = arguments[0].asString();
+    System.err.println(command + " " + Arrays.toString(arguments));
+
+    return switch (command) {
+      case PARSE_TREE -> {
+        var input = arguments[1].asString();
+
+        yield parser.parseInput(input);
+      }
+
+      case XX_HASH_128 -> {
+        var input = arguments[1].asString();
+
+        yield Integer.toString(input.hashCode());
+      }
+
+      case IS_IDENT_OR_OPERATOR -> {
+        var input = arguments[1].asString();
+
+        yield parser.isIdentOrOperator(input);
+      }
+
+      default -> throw new IllegalStateException(command);
+    };
+  }
+
+  @Override
+  public void close() {
+    parser.close();
+  }
+}

lib/java/ydoc-server/src/main/resources/org/enso/ydoc/polyfill/parser.js

@@ -0,0 +1,15 @@
+(function (jvm) {
+
+    globalThis.parse_tree = function(code) {
+        const byteBuffer = jvm('parse-tree', code);
+        return new Uint8Array(new ArrayBuffer(byteBuffer));
+    };
+
+    globalThis.xxHash128 = function(input) {
+        return jvm('xx-hash-128', input);
+    };
+
+    globalThis.is_ident_or_operator = function(code) {
+        return jvm('is-ident-or-operator', code);
+    };
+})

lib/java/ydoc-server/src/test/java/org/enso/ydoc/polyfill/EncodingTest.java

@@ -21,7 +21,7 @@ public void setup() throws Exception {
     executor = Executors.newSingleThreadExecutor();
     var encoding = new Encoding();
 
-    var b = Context.newBuilder("js");
+    var b = Context.newBuilder("js").allowExperimentalOptions(true);
 
     var chromePort = Integer.getInteger("inspectPort", -1);
     if (chromePort > 0) {

lib/java/ydoc-server/src/test/java/org/enso/ydoc/polyfill/ParserPolyfillTest.java

@@ -0,0 +1,93 @@
+package org.enso.ydoc.polyfill;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.graalvm.polyglot.Context;
+import org.graalvm.polyglot.HostAccess;
+import org.graalvm.polyglot.io.ByteSequence;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ParserPolyfillTest {
+
+  private Context context;
+  private ExecutorService executor;
+  private ParserPolyfill parser;
+
+  public ParserPolyfillTest() {}
+
+  @Before
+  public void setup() throws Exception {
+    executor = Executors.newSingleThreadExecutor();
+    parser = new ParserPolyfill();
+
+    var hostAccess =
+        HostAccess.newBuilder(HostAccess.EXPLICIT)
+            .allowArrayAccess(true)
+            .allowBufferAccess(true)
+            .build();
+    var b = Context.newBuilder("js").allowHostAccess(hostAccess).allowExperimentalOptions(true);
+
+    var chromePort = Integer.getInteger("inspectPort", -1);
+    if (chromePort > 0) {
+      b.option("inspect", ":" + chromePort);
+    }
+
+    context =
+        CompletableFuture.supplyAsync(
+                () -> {
+                  var ctx = b.build();
+                  parser.initialize(ctx);
+                  return ctx;
+                },
+                executor)
+            .get();
+  }
+
+  @After
+  public void tearDown() {
+    executor.close();
+    context.close();
+    parser.close();
+  }
+
+  @Test
+  public void parseTree() throws Exception {
+    var code =
+        """
+        const arr = parse_tree(`main = 1 + 2`)
+        arr.buffer
+        """;
+
+    var result = CompletableFuture.supplyAsync(() -> context.eval("js", code), executor).get();
+
+    Assert.assertTrue(result.as(ByteSequence.class).length() > 0);
+  }
+
+  @Test
+  public void xxHash128() throws Exception {
+    var code =
+        """
+        xxHash128(`main = 1 + 2`)
+        """;
+
+    var result = CompletableFuture.supplyAsync(() -> context.eval("js", code), executor).get();
+
+    Assert.assertEquals("1764801540", result.asString());
+  }
+
+  @Test
+  public void isIdentOrOperator() throws Exception {
+    var code =
+        """
+        is_ident_or_operator(`ident`)
+        """;
+
+    var result = CompletableFuture.supplyAsync(() -> context.eval("js", code), executor).get();
+
+    Assert.assertEquals(1, result.asLong());
+  }
+}

lib/rust/parser/generate-java/java/module-info.java

@@ -0,0 +1,3 @@
+module org.enso.syntax {
+  exports org.enso.syntax2;
+}

lib/rust/parser/generate-java/java/org/enso/syntax2/Parser.java

@@ -64,6 +64,8 @@ private Parser(long stateIn) {
 
   private static native ByteBuffer parseInput(long state, ByteBuffer input);
 
+  private static native long isIdentOrOperator(ByteBuffer input);
+
   private static native long getLastInputBase(long state);
 
   private static native long getMetadata(long state);
@@ -77,11 +79,23 @@ public static Parser create() {
     return new Parser(state);
   }
 
-  public Tree parse(CharSequence input) {
+  public long isIdentOrOperator(CharSequence input) {
     byte[] inputBytes = input.toString().getBytes(StandardCharsets.UTF_8);
     ByteBuffer inputBuf = ByteBuffer.allocateDirect(inputBytes.length);
     inputBuf.put(inputBytes);
-    var serializedTree = parseInput(state, inputBuf);
+
+    return isIdentOrOperator(inputBuf);
+  }
+
+  public ByteBuffer parseInput(CharSequence input) {
+    byte[] inputBytes = input.toString().getBytes(StandardCharsets.UTF_8);
+    ByteBuffer inputBuf = ByteBuffer.allocateDirect(inputBytes.length);
+    inputBuf.put(inputBytes);
+    return parseInput(state, inputBuf);
+  }
+
+  public Tree parse(CharSequence input) {
+    var serializedTree = parseInput(input);
     var base = getLastInputBase(state);
     var metadata = getMetadata(state);
     serializedTree.order(ByteOrder.LITTLE_ENDIAN);