Skip to content

Commit

Permalink
EnsoCompilerTest to verify compatibility of parsers (#3723)
Browse files Browse the repository at this point in the history
Adding new _compatibility test_ `EnsoCompilerTest` to verify the new Rust based parser can produce the same `IR` as the original `AST` based one. The simplest way to execute the test from an empty repository is:
```bash
enso$ sbt bootstrap
enso$ sbt "testOnly *EnsoCompilerTest"
```

There are [GitHub Actions run](https://github.com/enso-org/enso/actions/runs/3087664644/jobs/4993266212#step:9:5187) on Linux as well as [run on Windows](https://github.com/enso-org/enso/actions/runs/3087664644/jobs/4993266370#step:9:5254) that show `EnsoCompilerTest` is being executed by the CI (good, as that means `.so` was properly built and linked to the JVM running the test). The [linux](https://github.com/enso-org/enso/actions/runs/3087664644/jobs/4993266212#step:9:5187) as well as [windows](https://github.com/enso-org/enso/actions/runs/3087664644/jobs/4993266370#step:9:5254) runs also demonstrate that failures in the `EnsoCompilerTest` suite fail the CI.

# Important Notes
Right now [there are five test failures](https://github.com/enso-org/enso/actions/runs/3087664644/jobs/4993266212#step:9:5187) - waiting for @kazcw to make sure `codeRepr()` doesn't contain spaces. However, as this PR is more about the infrastructure, I am disabling the currently failing tests in [031169b](031169b)
  • Loading branch information
JaroslavTulach authored Sep 20, 2022
1 parent 97a28ee commit 9134f9b
Show file tree
Hide file tree
Showing 12 changed files with 2,347 additions and 148 deletions.
54 changes: 54 additions & 0 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
import LibraryManifestGenerator.BundledLibrary
import org.enso.build.BenchTasks._
import org.enso.build.WithDebugCommand
import org.apache.commons.io.FileUtils
import sbt.Keys.{libraryDependencies, scalacOptions}
import sbt.addCompilerPlugin
import sbt.complete.DefaultParsers._
import sbt.complete.Parser
import sbt.nio.FileStamper.LastModified
import sbt.nio.file.FileTreeView
import sbtcrossproject.CrossPlugin.autoImport.{CrossType, crossProject}
import src.main.scala.licenses.{DistributionDescription, SBTDistributionComponent}
import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType}
import src.main.scala.licenses.{
DistributionDescription,
Expand Down Expand Up @@ -253,6 +258,7 @@ lazy val enso = (project in file("."))
`polyglot-api`,
`project-manager`,
`syntax-definition`.jvm,
`syntax-rust-definition`,
`text-buffer`,
flexer.jvm,
graph,
Expand Down Expand Up @@ -660,6 +666,53 @@ lazy val `text-buffer` = project
)
)

val generateRustParserLib = TaskKey[Seq[File]]("generateRustParserLib", "Generates parser native library")
`syntax-rust-definition` / generateRustParserLib := {
import sys.process._
val libGlob = target.value.toGlob / "rust" / * / "libenso_parser.so"
val allLibs = FileTreeView.default.list(Seq(libGlob)).map(_._1)
if (allLibs.isEmpty || (`syntax-rust-definition` / generateRustParserLib).inputFileChanges.hasChanges) {
Seq("cargo", "build", "-p", "enso-parser-jni") !
}
FileTreeView.default.list(Seq(libGlob)).map(_._1.toFile)
}

`syntax-rust-definition` / generateRustParserLib / fileInputs +=
(`syntax-rust-definition` / baseDirectory).value.toGlob / "jni" / "src" / "*.rs"

val generateParserJavaSources = TaskKey[Seq[File]]("generateParserJavaSources", "Generates Java sources for Rust parser")
`syntax-rust-definition` / generateParserJavaSources := {
generateRustParser((`syntax-rust-definition` / Compile / sourceManaged).value, (`syntax-rust-definition` / generateParserJavaSources).inputFileChanges)
}
`syntax-rust-definition` / generateParserJavaSources / fileInputs +=
(`syntax-rust-definition` / baseDirectory).value.toGlob / "generate-java" / "src" / ** / "*.rs"

def generateRustParser(base: File, changes: sbt.nio.FileChanges): Seq[File] = {
import scala.jdk.CollectionConverters._
import java.nio.file.Paths

import sys.process._
val syntaxPkgs = Paths.get("org", "enso", "syntax2").toString
val fullPkg = Paths.get(base.toString, syntaxPkgs).toFile
if (!fullPkg.exists()) {
fullPkg.mkdirs()
}
if (changes.hasChanges) {
Seq("cargo", "run", "-p", "enso-parser-generate-java", "--bin", "enso-parser-generate-java", fullPkg.toString) !
}
FileUtils.listFiles(fullPkg, Array("scala", "java"), true).asScala.toSeq
}

lazy val `syntax-rust-definition` = project
.in(file("lib/rust/parser"))
.configs(Test)
.settings(
Compile / sourceGenerators += generateParserJavaSources,
Compile / resourceGenerators += generateRustParserLib,
Compile / javaSource := baseDirectory.value / "generate-java" / "java",
frgaalJavaCompilerSetting,
)

lazy val graph = (project in file("lib/scala/graph/"))
.dependsOn(logger.jvm)
.configs(Test)
Expand Down Expand Up @@ -1336,6 +1389,7 @@ lazy val runtime = (project in file("engine/runtime"))
.dependsOn(`library-manager`)
.dependsOn(`connected-lock-manager`)
.dependsOn(syntax.jvm)
.dependsOn(`syntax-rust-definition`)
.dependsOn(`docs-generator`)
.dependsOn(testkit % Test)

Expand Down
33 changes: 33 additions & 0 deletions engine/runtime/src/main/java/org/enso/compiler/EnsoCompiler.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package org.enso.compiler;

import com.oracle.truffle.api.source.Source;
import org.enso.compiler.core.IR;
import org.enso.syntax2.Parser;
import org.enso.syntax2.Tree;
import org.enso.syntax2.UnsupportedSyntaxException;

public final class EnsoCompiler implements AutoCloseable {
private final Parser parser;

public EnsoCompiler() {
this.parser = Parser.create();
}

@Override
public void close() throws Exception {
this.parser.close();
}

IR.Module compile(Source src) throws UnsupportedSyntaxException {
var tree = parse(src);
return generateIR(tree);
}

Tree parse(Source src) throws UnsupportedSyntaxException {
return parser.parse(src.getCharacters());
}

public IR.Module generateIR(Tree t) {
return TreeToIr.MODULE.translate(t);
}
}
Loading

0 comments on commit 9134f9b

Please sign in to comment.