Use Rust Parser from Java instead of AST.scala

LoadParser.java

@@ -0,0 +1,139 @@
+package org.enso.checkparser;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import org.enso.compiler.EnsoCompiler;
+import org.enso.compiler.core.IR;
+import org.enso.syntax2.Parser;
+import org.enso.syntax2.Tree;
+import org.enso.syntax2.UnsupportedSyntaxException;
+import org.graalvm.polyglot.Source;
+
+class LoadParser implements FileVisitor<Path>, AutoCloseable {
+    private final File root;
+    private final Parser parser;
+    private final EnsoCompiler compiler;
+    private final Set<Path> visited = new LinkedHashSet<>();
+    private final Set<Path> failed = new LinkedHashSet<>();
+    private final Set<Path> irComment = new LinkedHashSet<>();
+    private final Set<Path> irTested = new LinkedHashSet<>();
+    private final Map<Path,Exception> irFailed = new LinkedHashMap<>();
+
+    private LoadParser(File root) {
+        this.parser = Parser.create();
+        this.compiler = new EnsoCompiler();
+        this.root = root;
+    }
+
+    @Override
+    public void close() throws Exception {
+        parser.close();
+    }
+
+    public static void main(String[] args) throws Exception {
+        var root = new File(".").getAbsoluteFile();
+        try (LoadParser checker = new LoadParser(root)) {
+            checker.scan("distribution");
+            checker.scan("tests");
+
+            checker.printSummary(true);
+        }
+    }
+
+    private void scan(String path) throws IOException {
+        var dir = root.toPath().resolve(path);
+        assert Files.isDirectory(dir) : "isDirectory: " + dir;
+
+        Files.walkFileTree(dir, this);
+    }
+
+    @Override
+    public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+        return FileVisitResult.CONTINUE;
+    }
+
+    @Override
+    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+        if (!file.getFileName().toString().endsWith(".enso")) {
+            return FileVisitResult.CONTINUE;
+        }
+        visited.add(file);
+
+        System.err.println("processing " + file);
+        Source src = Source.newBuilder("enso", file.toFile()).build();
+        TEST: try {
+            Tree tree = parser.parse(src.getCharacters().toString());
+            if (tree == null) {
+                failed.add(file);
+            } else {
+                if (skipIrTest(file)) {
+                    return FileVisitResult.CONTINUE;
+                }
+                try {
+                    irTested.add(file);
+                    IR.Module m = compiler.generateIR(tree);
+                    if (m == null) {
+                        throw new NullPointerException();
+                    }
+                } catch (Exception ex) {
+                    if (ex.getClass().getSimpleName().equals("UnhandledEntity") && ex.getMessage().contains("Comment[")) {
+                        irTested.remove(file);
+                        irComment.add(file);
+                        break TEST;
+                    }
+                    irFailed.put(file, ex);
+                }
+            }
+        } catch (UnsupportedSyntaxException ex) {
+            failed.add(file);
+        }
+
+        return FileVisitResult.CONTINUE;
+    }
+
+    @Override
+    public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+        visited.add(file);
+        failed.add(file);
+        return FileVisitResult.CONTINUE;
+    }
+
+    @Override
+    public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+        return FileVisitResult.CONTINUE;
+    }
+
+    private void printSummary(boolean verbose) {
+        if (verbose) {
+            for (var en : irFailed.entrySet()) {
+                var key = en.getKey();
+                var value = en.getValue();
+                System.err.println("File " + key);
+                value.printStackTrace();
+            }
+        }
+        System.out.println("Found " + visited.size() + " files. " + failed.size() + " failed to parse");
+        System.out.println(irComment.size() + " files failed because they have comments");
+        System.out.println("From " + irTested.size() + " files " + irFailed.size() + " failed to produce IR");
+    }
+
+    private static boolean skipIrTest(Path file) throws IOException {
+        var caseOf = Pattern.compile("case.*of");
+        for (var line : Files.readAllLines(file)) {
+            if (caseOf.matcher(line).find()) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

LoadParser.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -x
+
+set -e
+
+GRAALVM=$HOME/bin/graalvm/
+
+# build runtime.jar including the new parser classes
+sbt --java-home $GRAALVM bootstrap
+sbt --java-home $GRAALVM syntax-rust-definition/compile
+
+cargo build -p enso-parser-jni
+
+sbt --java-home $GRAALVM buildEngineDistribution
+
+# run test: parser all .enso files in the repository
+$GRAALVM/bin/java -cp runtime.jar LoadParser.java

build.sbt

@@ -250,6 +250,7 @@ lazy val enso = (project in file("."))
     `polyglot-api`,
     `project-manager`,
     `syntax-definition`.jvm,
+    `syntax-rust-definition`,
     `text-buffer`,
     flexer.jvm,
     graph,
@@ -657,6 +658,32 @@ lazy val `text-buffer` = project
     )
   )
 
+val generateRustParserLib = TaskKey[Unit]("generateRustParserLib", "Generates parser native library")
+val generateRustParserLibSettings = generateRustParserLib := {
+    import sys.process._
+    Seq("cargo", "build", "-p", "enso-parser-jni") !
+}
+
+val generateRustParser = TaskKey[Unit]("generateRustParser", "Generates parser sources")
+val generateRustParserSettings = generateRustParser := {
+    import sys.process._
+    Seq("cargo", "run", "-p", "enso-parser-generate-java", "--bin", "enso-parser-generate-java", "lib/rust/parser/generate-java/java/org/enso/syntax2/") !
+}
+
+lazy val `syntax-rust-definition` = project
+  .in(file("lib/rust/parser"))
+  .configs(Test)
+  .settings(
+    compile := ((Compile / compile) dependsOn generateRustParser).value,
+    compile := ((Compile / compile) dependsOn generateRustParserLib).value,
+    Compile / javaSource := baseDirectory.value / "generate-java" / "java",
+    frgaalJavaCompilerSetting,
+    generateRustParserSettings,
+    generateRustParserLibSettings,
+    libraryDependencies ++= Seq(
+    ),
+  )
+
 lazy val graph = (project in file("lib/scala/graph/"))
   .dependsOn(logger.jvm)
   .configs(Test)
@@ -1334,6 +1361,7 @@ lazy val runtime = (project in file("engine/runtime"))
   .dependsOn(`library-manager`)
   .dependsOn(`connected-lock-manager`)
   .dependsOn(syntax.jvm)
+  .dependsOn(`syntax-rust-definition`)
   .dependsOn(`docs-generator`)
   .dependsOn(testkit % Test)
 

engine/runtime/src/main/java/org/enso/compiler/EnsoCompiler.java

@@ -0,0 +1,33 @@
+package org.enso.compiler;
+
+import com.oracle.truffle.api.source.Source;
+import org.enso.compiler.core.IR;
+import org.enso.syntax2.Parser;
+import org.enso.syntax2.Tree;
+import org.enso.syntax2.UnsupportedSyntaxException;
+
+public final class EnsoCompiler implements AutoCloseable {
+  private final Parser parser;
+
+  public EnsoCompiler() {
+    this.parser = Parser.create();
+  }
+
+  @Override
+  public void close() throws Exception {
+    this.parser.close();
+  }
+
+  IR.Module compile(Source src) throws UnsupportedSyntaxException {
+    var tree = parse(src);
+    return generateIR(tree);
+  }
+
+  Tree parse(Source src) throws UnsupportedSyntaxException {
+    return parser.parse(src.getCharacters());
+  }
+
+  public IR.Module generateIR(Tree t) {
+    return TreeToIr.MODULE.translate(t);
+  }
+}