diff --git a/Cargo.toml b/Cargo.toml index feb84d1f200e..5f3c112fcc33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "lib/rust/flexer-testing/definition", "lib/rust/flexer-testing/generation", "lib/rust/lazy-reader", + "lib/rust/parser", ] [profile.dev] diff --git a/build.sbt b/build.sbt index c902f1809b71..00179090d383 100644 --- a/build.sbt +++ b/build.sbt @@ -798,10 +798,32 @@ lazy val `language-server` = (project in file("engine/language-server")) lazy val ast = (project in file("lib/scala/ast")) .settings( version := ensoVersion, - GenerateAST.rustVersion := rustVersion, + Cargo.rustVersion := rustVersion, Compile / sourceGenerators += GenerateAST.task ) +lazy val parser = (project in file("lib/scala/parser")) + .settings( + fork := true, + Cargo.rustVersion := rustVersion, + Compile / compile / compileInputs := (Compile / compile / compileInputs) + .dependsOn(Cargo("build --project parser")) + .value, + javaOptions += { + val root = baseDirectory.value.getParentFile.getParentFile.getParentFile + s"-Djava.library.path=$root/target/rust/debug" + }, + libraryDependencies ++= Seq( + "com.storm-enroute" %% "scalameter" % scalameterVersion % "bench", + "org.scalatest" %%% "scalatest" % scalatestVersion % Test, + ), + testFrameworks := List( + new TestFramework("org.scalatest.tools.Framework"), + new TestFramework("org.scalameter.ScalaMeterFramework") + ), + ) + .dependsOn(ast) + lazy val runtime = (project in file("engine/runtime")) .configs(Benchmark) .settings( diff --git a/docs/distribution/distribution.md b/docs/distribution/distribution.md index 0e584d7bf4b4..385db09c1967 100644 --- a/docs/distribution/distribution.md +++ b/docs/distribution/distribution.md @@ -47,8 +47,8 @@ directories, as explained below. ### Portable Enso Distribution Layout -All files in the directory structure, except for configuration, can be safely -removed and the launcher will re-download them if needed. +All files in the directory structure, except for the configuration, can be +safely removed, and the launcher will re-download them if needed. The directory structure is as follows: @@ -162,6 +162,9 @@ enso-1.0.0 ├── component # Contains all the executable tools and their dependencies. │ ├── runner.jar # The main executable of the distribution. CLI entry point. │ └── runtime.jar # The language runtime. It is loaded by other JVM components, like the runner. +├── native-libraries # Contains all shared libraries that are used by JVM components. +│ └── parser.so # The language parser. It is loaded by the runtime component. +│ # Alternative extensions are .dll Windows and .dylib on Mac. └── std-lib # Contains all the libraries that are pre-installed within that compiler version. ├── Http # Every version sub-directory is just an Enso package containing the library. │ ├── package.yaml diff --git a/docs/infrastructure/rust.md b/docs/infrastructure/rust.md new file mode 100644 index 000000000000..9c0e7d2cb13f --- /dev/null +++ b/docs/infrastructure/rust.md @@ -0,0 +1,42 @@ +--- +layout: developer-doc +title: Rust +category: infrastructure +tags: [infrastructure, build] +order: 1 +--- + +# Rust + +The Rust project is built using Cargo which manages dependencies between the +projects as well as external dependencies and allows for incremental +compilation. The build configuration is defined in +[`Cargo.toml`](../../Cargo.toml). + + + +- [Shared Libraries](#shared-libraries) + + + +# Java Native Interface + +Although the entry point of the Enso project is a Java archive, it can still +make use of native libraries built in Rust trough the JVM foreign function +interface (FFI) named +[Java Native Interface](https://en.wikipedia.org/wiki/Java_Native_Interface) +(JNI). + +In order to generate a shared library, the `Cargo.toml` needs to enable +compilation into a dynamic system library: + +``` +crate-type = ["cdylib"] +``` + +Invoking `cargo build` will then output the library into `target/rust/debug` +with the extension `.so` on Linux, `.dll` on Windows and `.dylib` on macOS. + +Then, to be able to load the library by `System.loadLibrary("lib_name")`, +the Java application should be started with the option +`-Djava.library.path=path/to/lib_folder`. diff --git a/docs/parser/jvm-object-generation.md b/docs/parser/jvm-object-generation.md index 91c89ec5f124..df32d53db30d 100644 --- a/docs/parser/jvm-object-generation.md +++ b/docs/parser/jvm-object-generation.md @@ -14,9 +14,18 @@ the compiler and runtime to work with the AST. +- [Overall Architecture](#overall-architecture) + -> The actionables for this section are: -> -> - Work out how on earth this is going to work. -> - Produce a detailed design for this functionality. +# Overall Architecture + +The JVM foreign function interface (FFI), named +[Java Native Interface](https://en.wikipedia.org/wiki/Java_Native_Interface) +(JNI), enables the Rust parser library to call and be called by the parser +library implemented in Scala. + +Specifically, in our architecture, the JNI is used for: + +- Invoking Rust parser methods from Scala. +- Invoking Scala AST constructors from Rust. diff --git a/lib/rust/parser/Cargo.toml b/lib/rust/parser/Cargo.toml new file mode 100644 index 000000000000..6b811ef6354a --- /dev/null +++ b/lib/rust/parser/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "parser" +version = "0.1.0" +authors = ["Enso Team "] +edition = "2018" + +description = "A parser for the Enso language" +readme = "README.md" +homepage = "https://github.com/enso-org/enso/lib/rust/parser" +repository = "https://github.com/enso-org/enso" +license-file = "../../../LICENSE" + +keywords = ["parser"] +categories = ["parsing"] + +publish = false + +[lib] +name = "parser" +crate-type = ["cdylib", "rlib"] +test = true +bench = true + +[dependencies] +jni = { version = "0.17.0" } +ast = { version = "0.1.0", path = "../ast" } diff --git a/lib/rust/parser/src/jni.rs b/lib/rust/parser/src/jni.rs new file mode 100644 index 000000000000..bba0a177f102 --- /dev/null +++ b/lib/rust/parser/src/jni.rs @@ -0,0 +1,116 @@ +//! This module exports JNI interface for parser methods implemented in Rust. +//! +//! The basics steps to add a new method are following: +//! 1. Add the new method in Scala (in `org.enso.parser.Parser`). +//! 2. (Optional) Run `scalac Parser.scala; javah Parser` to generate the C API in `Parser.h`. +//! Note that you can skip this step. It is merely a guidance for you, as it generates +//! the correct function names and type signatures of all `Parser` native methods. +//! Generally, the method interface is going to have the following shape: +//! ```c +//! JNIEXPORT $returnType JNICALL Java_$package_$className_$methodName +//! (JNIEnv* env, jobject this, $argType1 $arg1, $argType2 $arg2) +//! ``` +//! For example if the definition is: +//! ```scala +//! package org.enso.parser +//! +//! class Parser { +//! @native def newMethod(string: String, array: Array[Int]) +//! } +//! ``` +//! Then the JNI API is going to be: +//! ```c +//! JNIEXPORT jobject JNICALL Java_org_enso_parser_Parser_newMethod +//! (JNIEnv* env, jobject this, jstring string, jintArray array) +//! ``` +//! The list of all available types can be found in +//! [oracle documentation](https://docs.oracle.com/javase/7/docs/technotes/guides/jni/spec/types.html). +//! 3. Implement the new parser method in this file. +//! For the above definition the implementation is going to be: +//! ```rust +//! use jni::JNIEnv; +//! use jni::objects::*; +//! use jni::sys::*; +//! +//! #[no_mangle] +//! pub extern "system" fn Java_org_enso_parser_Parser_newMethod( +//! env : JNIEnv, // the JVM enviroment, used for calling methods and constructors +//! this : JClass, // the instance of `Parser` +//! string : JString, +//! array : jintArray, +//! ) -> jweak { unimplemented!() } +//! ``` +//! 4. (Optional) Generate a shared library from the Rust definition by `cargo build`. +//! It will be generated into `target/rust/debug/`. +//! This step is done automatically by `sbt`. + +use jni::JNIEnv; +use jni::objects::*; +use jni::sys::*; + + + +// ====================== +// === Parser JNI API === +// ====================== + +/// Parses a content a of single source file. +#[no_mangle] +pub extern "system" fn Java_org_enso_parser_Parser_parseStr( + env : JNIEnv, + _this : JClass, + input : JString, +) -> jweak { + let txt = env.new_object( + env.find_class("org/enso/ast/Ast$Txt$Text").unwrap(), + "(Ljava/lang/String;)V", + &[input.into()], + ).unwrap(); + + let non = env.get_static_field( + env.find_class("scala/None$").unwrap(), + "MODULE$", + "Lscala/None$;", + ).unwrap().l().unwrap(); + + let ast = env.new_object( + env.find_class("org/enso/ast/Ast$Ast").unwrap(), + "(Lscala/Option;JJLjava/lang/Object;)V", + &[non.into(), 0i64.into(), 0i64.into(), txt.into()], + ).unwrap(); + + ast.into_inner() +} + +/// Parses a single source file. +#[no_mangle] +pub extern "system" fn Java_org_enso_parser_Parser_parseFile( + env : JNIEnv, + this : JClass, + filename : JString, +) -> jweak { + Java_org_enso_parser_Parser_parseStr(env, this, filename) +} + + +// === Tokens === + +/// Parses a content of a single source file into a stream of tokens. +#[no_mangle] +pub extern "system" fn Java_org_enso_parser_Parser_lexStr( + env : JNIEnv, + this : JClass, + input : JString, +) -> jweak { + Java_org_enso_parser_Parser_parseStr(env, this, input) +} + +/// Parses a single source file into a stream of tokens. +#[no_mangle] +pub extern "system" fn Java_org_enso_parser_Parser_lexFile( + env : JNIEnv, + this : JClass, + filename : JString, +) -> jweak { + Java_org_enso_parser_Parser_parseStr(env, this, filename) +} diff --git a/lib/rust/parser/src/lib.rs b/lib/rust/parser/src/lib.rs new file mode 100644 index 000000000000..a3dcd74630a7 --- /dev/null +++ b/lib/rust/parser/src/lib.rs @@ -0,0 +1,47 @@ +#![feature(test)] +#![deny(unconditional_recursion)] +#![warn(missing_copy_implementations)] +#![warn(missing_debug_implementations)] +#![warn(missing_docs)] +#![warn(trivial_casts)] +#![warn(trivial_numeric_casts)] +#![warn(unsafe_code)] +#![warn(unused_import_braces)] + +//! This module exports the implementation of parser for the Enso language. + +mod jni; + +pub use crate::jni::*; + +use ast::AnyAst; +use ast::Ast; + + + +// ======================= +// === Parser Rust API === +// ======================= + +/// Parse a content of a single source file. +pub fn parse_str(input:String) -> AnyAst { + Ast::new(ast::txt::Text{text:input}) +} + +/// Parse a single source file. +pub fn parse_file(filename:String) -> AnyAst { + parse_str(filename) +} + + +// === Tokens === + +/// Parse a content of single source file. +pub fn lexe_str(input:String) -> AnyAst { + parse_str(input) +} + +/// Parse a single source file. +pub fn lexe_file(filename:String) -> AnyAst { + parse_str(filename) +} diff --git a/lib/scala/parser/src/main/scala/org/enso/parser/Parser.scala b/lib/scala/parser/src/main/scala/org/enso/parser/Parser.scala new file mode 100644 index 000000000000..d2eaa2cddb9c --- /dev/null +++ b/lib/scala/parser/src/main/scala/org/enso/parser/Parser.scala @@ -0,0 +1,41 @@ +package org.enso.parser + +import org.enso.ast.Ast + +import scala.annotation.unused + + +/** This is the Enso language parser. + * + * It is a wrapper of parser written in Rust that uses JNI to efficiently + * construct scala AST directly without any serialization overhead. + * + * The methods are loaded from a native shared library `parser` that is located + * in a directory specified by `-Djava.library.path` and has one of the extensions + * `.dll`, `.so` or `dylib` depending on the platform (windows, linux or mac). + * + * The shared library itself is generated into `target/rust/debug` by executing + * `cargo build -p parser`. Each method marked by `@native` must have a + * corresponding counterpart in rust, otherwise the loading of the shared library + * is going to fail at runtime with `UnsatisfiedLinkingError`. + */ +class Parser private () { + /** Parses a content of a single source file. */ + @native def parseStr(@unused input: String): Ast.AnyAst + + /** Parses a single source file. */ + @native def parseFile(@unused filename: String): Ast.AnyAst + + /** Parses a content of a single source file into a stream of tokens. */ + @native def lexStr(@unused input: String): Ast.AnyAst + + /** Parses a single source file into a stream of tokens. */ + @native def lexFile(@unused filename: String): Ast.AnyAst +} + +object Parser { + System.loadLibrary("parser") + + /** Constructs a new parser */ + def apply(): Parser = new Parser() +} diff --git a/lib/scala/parser/src/test/scala/org/enso/parser/ParserTest.scala b/lib/scala/parser/src/test/scala/org/enso/parser/ParserTest.scala new file mode 100644 index 000000000000..9474362b7457 --- /dev/null +++ b/lib/scala/parser/src/test/scala/org/enso/parser/ParserTest.scala @@ -0,0 +1,20 @@ +package org.enso.parser + +import org.enso.ast.Ast + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + + + +class ParserTest extends AnyFlatSpec with Matchers { + val parser: Parser = Parser() + + it should "parse file" in { + val expected = Ast.Ast(uid=None, len=0, off=0, ast=Ast.Txt.Text("Hello!")) + assert(expected == parser.parseStr("Hello!")) + assert(expected == parser.parseFile("Hello!")) + assert(expected == parser.lexStr("Hello!")) + assert(expected == parser.lexFile("Hello!")) + } +} diff --git a/project/Cargo.scala b/project/Cargo.scala new file mode 100644 index 000000000000..aa7f46eb5e3c --- /dev/null +++ b/project/Cargo.scala @@ -0,0 +1,49 @@ +import sbt.Keys._ +import sbt._ +import sbt.internal.util.ManagedLogger + +import scala.sys.process._ + + + +/** A wrapper for executing the command `cargo`. */ +object Cargo { + + /** The version of rust that needs to be installed. */ + val rustVersion = settingKey[String]("rustc version used in the project") + + private val cargoCmd = "cargo" + + /** Checks rust version and executes the command `cargo $args`. */ + def apply(args: String): Def.Initialize[Task[Unit]] = Def.task { + run(args, rustVersion.value, state.value.log) + } + + /** Checks rust version and executes the command `cargo $args`. */ + def run(args: String, rustVersion: String, log: ManagedLogger): Unit = { + val cmd = s"$cargoCmd $args" + + if (!cargoOk(log)) + throw new RuntimeException("Cargo isn't installed!") + + if (!EnvironmentCheck.rustVersionOk(rustVersion, log)) + throw new RuntimeException("Rust version mismatch!") + + log.info(cmd) + + try cmd.!! catch { + case _: RuntimeException => + throw new RuntimeException("Cargo command failed.") + } + } + + /** Checks that cargo is installed. Logs an error and returns false if not. */ + def cargoOk(log: ManagedLogger): Boolean = { + try s"$cargoCmd version".!! catch { + case _: RuntimeException => + log.error(s"The command `cargo` isn't on path. Did you install cargo?") + return false + } + true + } +} diff --git a/project/EnvironmentCheck.scala b/project/EnvironmentCheck.scala index 2ed87cf3df8d..258300abb903 100644 --- a/project/EnvironmentCheck.scala +++ b/project/EnvironmentCheck.scala @@ -1,6 +1,5 @@ import java.io.IOException -import GenerateAST.cargoCmd import sbt._ import sbt.internal.util.ManagedLogger diff --git a/project/GenerateAST.scala b/project/GenerateAST.scala index 8fe0a2fe1ccc..3e6a4eca6a71 100644 --- a/project/GenerateAST.scala +++ b/project/GenerateAST.scala @@ -1,31 +1,24 @@ -import java.io.IOException - import sbt.Keys._ import sbt._ import sbt.internal.util.ManagedLogger -import scala.sys.process._ -object GenerateAST { - val rustVersion = settingKey[String]("rustc version used in the project") - private val cargoCmd = "cargo" +object GenerateAST { lazy val task = Def.task { val log = state.value.log - val lib = baseDirectory.value.getParentFile.getParentFile / "lib" + val lib = baseDirectory.value.getParentFile.getParentFile val source = lib / "rust/ast/src/ast.rs" val output = sourceManaged.value / "main/org/enso/ast/Ast.scala" val cache = streams.value.cacheStoreFactory.make("ast_source") - if (!EnvironmentCheck.rustVersionOk(rustVersion.value, log)) - throw new RuntimeException("Rust version mismatch!") - Tracked.diffInputs(cache, FileInfo.lastModified)(Set(source)) { source: ChangeReport[File] => + val rustVersion = Cargo.rustVersion.value if (source.modified.nonEmpty) { output.getParentFile.mkdirs - generateAST(output, log) + generateAST(rustVersion, output, log) } } @@ -33,17 +26,18 @@ object GenerateAST { } /** - * Generates the Scala AST in the specified file. All errors are reported in + * Generates the Scala AST in the specified file. All errors are reported in * stderr and raise a runtime exception. * * @param out the file where the generated AST is going to be placed */ - private def generateAST(out: File, log: ManagedLogger): Unit = { - val command = s"$cargoCmd run -p ast -- --generate-scala-ast $out" + def generateAST + (rustVersion: String, out: File, log: ManagedLogger): Unit = { + val args = s"run -p ast -- --generate-scala-ast $out" - log.info(s"Generating AST with the command: $command:") + log.info(s"Generating Scala AST from Rust definitions.") - try {command.!!} catch { + try Cargo.run(args, rustVersion, log) catch { case ex: RuntimeException => log.error(s"Generation of the Scala AST failed.") throw ex