Stub Out the Parser Interface

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 
 members = [
+    "lib/rust/ast",
     "lib/rust/enso-data",
     "lib/rust/enso-generics",
     "lib/rust/enso-logger",
@@ -10,11 +11,11 @@ members = [
     "lib/rust/enso-shapely/impl",
     "lib/rust/enso-shapely/macros",
     "lib/rust/flexer",
+    "lib/rust/lazy-reader",
     "lib/rust/lexer/definition",
     "lib/rust/lexer/generation",
     "lib/rust/lexer/tests",
-    "lib/rust/lazy-reader",
-    "lib/rust/ast",
+    "lib/rust/parser",
 ]
 
 [profile.dev]

build.sbt

@@ -798,10 +798,32 @@ lazy val `language-server` = (project in file("engine/language-server"))
 lazy val ast = (project in file("lib/scala/ast"))
   .settings(
     version := ensoVersion,
-    GenerateAST.rustVersion := rustVersion,
+    Cargo.rustVersion := rustVersion,
     Compile / sourceGenerators += GenerateAST.task
   )
 
+lazy val parser = (project in file("lib/scala/parser"))
+  .settings(
+    fork := true,
+    Cargo.rustVersion := rustVersion,
+    Compile / compile / compileInputs := (Compile / compile / compileInputs)
+      .dependsOn(Cargo("build -p parser"))
+      .value,
+    javaOptions += {
+      val root = baseDirectory.value.getParentFile.getParentFile.getParentFile
+      s"-Djava.library.path=$root/target/rust/debug"
+    },
+    libraryDependencies ++= Seq(
+      "com.storm-enroute" %% "scalameter" % scalameterVersion % "bench",
+      "org.scalatest" %%% "scalatest"     % scalatestVersion % Test,
+    ),
+    testFrameworks := List(
+      new TestFramework("org.scalatest.tools.Framework"),
+      new TestFramework("org.scalameter.ScalaMeterFramework")
+    ),
+  )
+  .dependsOn(ast)
+
 lazy val runtime = (project in file("engine/runtime"))
   .configs(Benchmark)
   .settings(

docs/distribution/distribution.md

@@ -47,8 +47,8 @@ directories, as explained below.
 
 ### Portable Enso Distribution Layout
 
-All files in the directory structure, except for configuration, can be safely
-removed and the launcher will re-download them if needed.
+All files in the directory structure, except for the configuration, can be
+safely removed, and the launcher will re-download them if needed.
 
 The directory structure is as follows:
 
@@ -162,6 +162,9 @@ enso-1.0.0
 ├── component        # Contains all the executable tools and their dependencies.
 │   ├── runner.jar   # The main executable of the distribution. CLI entry point.
 │   └── runtime.jar  # The language runtime. It is loaded by other JVM components, like the runner.
+├── native-libraries # Contains all shared libraries that are used by JVM components.
+│   └── parser.so    # The language parser. It is loaded by the runtime component.
+│                    # Alternative extensions are .dll Windows and .dylib on Mac.
 └── std-lib          # Contains all the libraries that are pre-installed within that compiler version.
     ├── Http         # Every version sub-directory is just an Enso package containing the library.
     │     ├── package.yaml

docs/infrastructure/rust.md

@@ -0,0 +1,42 @@
+---
+layout: developer-doc
+title: Rust
+category: infrastructure
+tags: [infrastructure, build]
+order: 1
+---
+
+# Rust
+
+The Rust project is built using Cargo which manages dependencies between the
+projects as well as external dependencies and allows for incremental
+compilation. The build configuration is defined in
+[`Cargo.toml`](../../Cargo.toml).
+
+<!-- MarkdownTOC levels="2,3" autolink="true" -->
+
+- [Shared Libraries](#shared-libraries)
+
+<!-- /MarkdownTOC -->
+
+# Java Native Interface
+
+Although the entry point of the Enso project is a Java archive, it can still
+make use of native libraries built in Rust trough the JVM foreign function
+interface (FFI) named
+[Java Native Interface](https://en.wikipedia.org/wiki/Java_Native_Interface)
+(JNI).
+
+In order to generate a shared library, the `Cargo.toml` needs to enable
+compilation into a dynamic system library:
+
+```
+crate-type = ["cdylib"]
+```
+
+Invoking `cargo build` will then output the library into `target/rust/debug`
+with the extension `.so` on Linux, `.dll` on Windows and `.dylib` on macOS.
+
+Then, if the Java application is started with the option
+`-Djava.library.path=path/to/lib_folder`, it can load the shared library by
+`System.loadLibrary("lib_name")`.

docs/parser/jvm-object-generation.md

@@ -14,9 +14,18 @@ the compiler and runtime to work with the AST.
 
 <!-- MarkdownTOC levels="2,3" autolink="true" -->
 
+- [Overall Architecture](#overall-architecture)
+
 <!-- /MarkdownTOC -->
 
-> The actionables for this section are:
->
-> - Work out how on earth this is going to work.
-> - Produce a detailed design for this functionality.
+# Overall Architecture
+
+The JVM foreign function interface (FFI), named
+[Java Native Interface](https://en.wikipedia.org/wiki/Java_Native_Interface)
+(JNI), enables the Rust parser library to call and be called by the parser
+library implemented in Scala.
+
+Specifically, in our architecture, the JNI is used for:
+
+- Invoking Rust parser methods from Scala.
+- Invoking Scala AST constructors from Rust.

lib/rust/parser/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name    = "parser"
+version = "0.1.0"
+authors = ["Enso Team <[email protected]>"]
+edition = "2018"
+
+description  = "A parser for the Enso language"
+readme       = "README.md"
+homepage     = "https://github.com/enso-org/enso/lib/rust/parser"
+repository   = "https://github.com/enso-org/enso"
+license-file = "../../../LICENSE"
+
+keywords   = ["parser"]
+categories = ["parsing"]
+
+publish = false
+
+[lib]
+name       = "parser"
+crate-type = ["cdylib", "rlib"]
+test       = true
+bench      = true
+
+[dependencies]
+jni = { version = "0.17.0"                  }
+ast = { version = "0.1.0",  path = "../ast" }

lib/rust/parser/src/jni.rs

@@ -0,0 +1,116 @@
+//! This module exports JNI interface for parser methods implemented in Rust.
+//!
+//! The basics steps to add a new method are following:
+//! 1. Add the new method in Scala (in `org.enso.parser.Parser`).
+//! 2. (Optional) Run `scalac Parser.scala; javah Parser` to generate the C API in `Parser.h`.
+//!    Note that you can skip this step. It is merely a guidance for you, as it generates
+//!    the correct function names and type signatures of all `Parser` native methods.
+//!    Generally, the method interface is going to have the following shape:
+//!    ```c
+//!    JNIEXPORT $returnType JNICALL Java_$package_$className_$methodName
+//!      (JNIEnv* env, jobject this, $argType1 $arg1, $argType2 $arg2)
+//!    ```
+//!    For example if the definition is:
+//!    ```scala
+//!    package org.enso.parser
+//!
+//!    class Parser {
+//!      @native def newMethod(string: String, array: Array[Int])
+//!    }
+//!    ```
+//!    Then the JNI API is going to be:
+//!    ```c
+//!    JNIEXPORT jobject JNICALL Java_org_enso_parser_Parser_newMethod
+//!      (JNIEnv* env, jobject this, jstring string, jintArray array)
+//!    ```
+//!    The list of all available types can be found in
+//!    [oracle documentation](https://docs.oracle.com/javase/7/docs/technotes/guides/jni/spec/types.html).
+//! 3. Implement the new parser method in this file.
+//!    For the above definition the implementation is going to be:
+//!    ```rust
+//!    use jni::JNIEnv;
+//!    use jni::objects::*;
+//!    use jni::sys::*;
+//!
+//!    #[no_mangle]
+//!    pub extern "system" fn Java_org_enso_parser_Parser_newMethod(
+//!        env    : JNIEnv,    // the JVM enviroment, used for calling methods and constructors
+//!        this   : JClass,    // the instance of `Parser`
+//!        string : JString,
+//!        array  : jintArray,
+//!    ) -> jweak { unimplemented!() }
+//!    ```
+//! 4. (Optional) Generate a shared library from the Rust definition by `cargo build`.
+//!    It will be generated into `target/rust/debug/`.
+//!    This step is done automatically by `sbt`.
+
+use jni::JNIEnv;
+use jni::objects::*;
+use jni::sys::*;
+
+
+
+// ======================
+// === Parser JNI API ===
+// ======================
+
+/// Parses a content a of single source file.
+#[no_mangle]
+pub extern "system" fn Java_org_enso_parser_Parser_parseStr(
+    env   : JNIEnv,
+    _this : JClass,
+    input : JString,
+) -> jweak {
+    let txt = env.new_object(
+        env.find_class("org/enso/ast/Ast$Txt$Text").unwrap(),
+        "(Ljava/lang/String;)V",
+        &[input.into()],
+    ).unwrap();
+
+    let non = env.get_static_field(
+        env.find_class("scala/None$").unwrap(),
+        "MODULE$",
+        "Lscala/None$;",
+    ).unwrap().l().unwrap();
+
+    let ast = env.new_object(
+        env.find_class("org/enso/ast/Ast$Ast").unwrap(),
+        "(Lscala/Option;JJLjava/lang/Object;)V",
+        &[non.into(), 0i64.into(), 0i64.into(), txt.into()],
+    ).unwrap();
+
+    ast.into_inner()
+}
+
+/// Parses a single source file.
+#[no_mangle]
+pub extern "system" fn Java_org_enso_parser_Parser_parseFile(
+    env      : JNIEnv,
+    this     : JClass,
+    filename : JString,
+) -> jweak {
+    Java_org_enso_parser_Parser_parseStr(env, this, filename)
+}
+
+
+// === Tokens ===
+
+/// Parses a content of a single source file into a stream of tokens.
+#[no_mangle]
+pub extern "system" fn Java_org_enso_parser_Parser_lexStr(
+    env   : JNIEnv,
+    this  : JClass,
+    input : JString,
+) -> jweak {
+    Java_org_enso_parser_Parser_parseStr(env, this, input)
+}
+
+/// Parses a single source file into a stream of tokens.
+#[no_mangle]
+pub extern "system" fn Java_org_enso_parser_Parser_lexFile(
+    env      : JNIEnv,
+    this     : JClass,
+    filename : JString,
+) -> jweak {
+    Java_org_enso_parser_Parser_parseStr(env, this, filename)
+}

lib/rust/parser/src/lib.rs

@@ -0,0 +1,47 @@
+#![feature(test)]
+#![deny(unconditional_recursion)]
+#![warn(missing_copy_implementations)]
+#![warn(missing_debug_implementations)]
+#![warn(missing_docs)]
+#![warn(trivial_casts)]
+#![warn(trivial_numeric_casts)]
+#![warn(unsafe_code)]
+#![warn(unused_import_braces)]
+
+//! This module exports the implementation of parser for the Enso language.
+
+mod jni;
+
+pub use crate::jni::*;
+
+use ast::AnyAst;
+use ast::Ast;
+
+
+
+// =======================
+// === Parser Rust API ===
+// =======================
+
+/// Parse a content of a single source file.
+pub fn parse_str(input:String) -> AnyAst {
+    Ast::new(ast::txt::Text{text:input})
+}
+
+/// Parse a single source file.
+pub fn parse_file(filename:String) -> AnyAst {
+    parse_str(filename)
+}
+
+
+// === Tokens ===
+
+/// Parse a content of single source file.
+pub fn lexe_str(input:String) -> AnyAst {
+    parse_str(input)
+}
+
+/// Parse a single source file.
+pub fn lexe_file(filename:String) -> AnyAst {
+    parse_str(filename)
+}

lib/scala/parser/src/main/scala/org/enso/parser/Parser.scala

@@ -0,0 +1,41 @@
+package org.enso.parser
+
+import org.enso.ast.Ast
+
+import scala.annotation.unused
+
+
+/** This is the Enso language parser.
+  *
+  * It is a wrapper of parser written in Rust that uses JNI to efficiently
+  * construct scala AST directly without any serialization overhead.
+  *
+  * The methods are loaded from a native shared library `parser` that is located
+  * in a directory specified by `-Djava.library.path` and has one of the extensions
+  * `.dll`, `.so` or `dylib` depending on the platform (windows, linux or mac).
+  *
+  * The shared library itself is generated into `target/rust/debug` by executing
+  * `cargo build -p parser`. Each method marked by `@native` must have a
+  * corresponding counterpart in rust, otherwise the loading of the shared library
+  * is going to fail at runtime with `UnsatisfiedLinkingError`.
+  */
+class Parser private () {
+  /**  Parses a content of a single source file. */
+  @native def parseStr(@unused input: String): Ast.AnyAst
+
+  /**  Parses a single source file. */
+  @native def parseFile(@unused filename: String): Ast.AnyAst
+
+  /**  Parses a content of a single source file into a stream of tokens. */
+  @native def lexStr(@unused input: String): Ast.AnyAst
+
+  /**  Parses a single source file into a stream of tokens. */
+  @native def lexFile(@unused filename: String): Ast.AnyAst
+}
+
+object Parser {
+  System.loadLibrary("parser")
+
+  /** Constructs a new parser */
+  def apply(): Parser = new Parser()
+}

lib/scala/parser/src/test/scala/org/enso/parser/ParserTest.scala

@@ -0,0 +1,20 @@
+package org.enso.parser
+
+import org.enso.ast.Ast
+
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+
+
+class ParserTest extends AnyFlatSpec with Matchers {
+  val parser: Parser = Parser()
+
+  it should "parse file" in {
+    val expected = Ast.Ast(uid=None, len=0, off=0, ast=Ast.Txt.Text("Hello!"))
+    assert(expected == parser.parseStr("Hello!"))
+    assert(expected == parser.parseFile("Hello!"))
+    assert(expected == parser.lexStr("Hello!"))
+    assert(expected == parser.lexFile("Hello!"))
+  }
+}