Optional Espresso support with ENSO_JAVA=espresso env variable

.vscode/settings.json

@@ -5,5 +5,8 @@
   "vue.complete.casing.tags": "pascal",
   "auto-snippets.snippets": [
     { "language": "vue", "snippet": "Vue single-file component" }
-  ]
+  ],
+  "files.watcherExclude": {
+    "**/target": true
+  }
 }

CHANGELOG.md

@@ -925,6 +925,7 @@
 - [Warning.get_all returns only unique warnings][6372]
 - [Reimplement `enso_project` as a proper builtin][6352]
 - [Limit number of reported warnings per value][6577]
+- [Experimental support for Espresso Java interpreter][6966]
 - [Suggestions are updated only when the type of the expression changes][6755]
 - [Add project creation time to project metadata][6780]
 - [Upgrade GraalVM to 22.3.1 JDK17][6750]
@@ -1066,6 +1067,7 @@
 [6372]: https://github.com/enso-org/enso/pull/6372
 [6352]: https://github.com/enso-org/enso/pull/6352
 [6577]: https://github.com/enso-org/enso/pull/6577
+[6966]: https://github.com/enso-org/enso/pull/6966
 [6750]: https://github.com/enso-org/enso/pull/6750
 [6755]: https://github.com/enso-org/enso/pull/6755
 [6780]: https://github.com/enso-org/enso/pull/6780

build.sbt

@@ -1699,18 +1699,33 @@ lazy val `engine-runner` = project
           "-H:IncludeResources=.*Main.enso$",
           "--macro:truffle",
           "--language:js",
-          //          "-g",
+          // "-g",
           //          "-H:+DashboardAll",
           //          "-H:DashboardDump=runner.bgv"
           "-Dnic=nic"
-        ),
+        ) ++ (if (
+                org.graalvm.polyglot.Engine
+                  .create()
+                  .getLanguages()
+                  .containsKey("java")
+              ) {
+                Seq(
+                  "-Dorg.graalvm.launcher.home=" + System.getProperty(
+                    "java.home"
+                  ),
+                  "--language:java"
+                )
+              } else {
+                Seq()
+              }),
         mainClass = Option("org.enso.runner.Main"),
         cp        = Option("runtime.jar"),
         initializeAtRuntime = Seq(
           "org.jline.nativ.JLineLibrary",
           "io.methvin.watchservice.jna.CarbonAPI",
           "org.enso.syntax2.Parser",
           "zio.internal.ZScheduler$$anon$4",
+          "org.enso.runner.Main$",
           "sun.awt",
           "sun.java2d",
           "sun.font",

docs/infrastructure/native-image.md

@@ -201,20 +201,71 @@ safely.
 ### Engine runner Configuration
 
 The Native Image generation for the Engine Runner is currently in a preview
-state. Limitations are currently mostly due to
-[Java interop](https://www.pivotaltracker.com/story/show/183260380) and loading
-of stdlib components. To generate the Native Image for runner simply execute
+state. To generate the Native Image for runner simply execute
 
-```
+```bash
 sbt> engine-runner/buildNativeImage
 ```
 
 and execute the binary on a sample factorial test program
 
-```
+```bash
 > runner --run engine/runner-native/src/test/resources/Factorial.enso 6
 ```
 
 The task that generates the Native Image, along with all the necessary
 configuration, reside in a separate project due to a bug in the currently used
-GraalVM version.
+GraalVM version. As September 2023 it can execute all Enso code, but cannot
+invoke `IO.println` or other library functions that require
+[polyglot java import](../../docs/polyglot/java.md), but read on...
+
+### Engine with Espresso
+
+Since [PR-6966](https://github.com/enso-org/enso/pull/6966) there is an
+experimental support for including
+[Espresso Java interpreter](https://www.graalvm.org/jdk17/reference-manual/java-on-truffle/)
+to allow use of some library functions (like `IO.println`) in the _Native Image_
+built runner.
+
+The support can be enabled by setting environment variable `ENSO_JAVA=espresso`
+and making sure Espresso is installed in GraalVM executing the Enso engine -
+e.g. by running `graalvm/bin/gu install espresso`. Then execute:
+
+```bash
+$ cat >hello.enso
+import Standard.Base.IO
+
+main = IO.println <| "Hello World!"
+
+$ ENSO_JAVA=espresso ./enso-x.y.z-dev/bin/enso --run hello.enso
+```
+
+Unless you see a warning containing _"No language for id java found."_ your code
+has just successfully been executed by
+[Espresso](https://www.graalvm.org/jdk17/reference-manual/java-on-truffle/)! To
+debug just add `JAVA_OPTS` environment variable set to your IDE favorite value:
+
+```bash
+$ JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,address=5005 ENSO_JAVA=espresso enso --run hello.enso
+```
+
+Espresso support works also with
+[native image support](#engine-runner-configuration). Just make sure Espresso is
+installed in your GraalVM (via `gu install espresso`) and then rebuild the
+`runner` executable:
+
+```bash
+enso$ rm runner
+enso$ sbt --java-home /graalvm
+sbt> engine-runner/buildNativeImage
+```
+
+as suggested in the [native image support](#engine-runner-configuration). The
+build script detects presence of Espresso and automatically adds
+`--language:java` when creating the image. Then you can use
+
+```bash
+$ ENSO_JAVA=espresso ./runner --run hello.enso
+```
+
+to execute native image `runner` build of Enso together with Espresso.

engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala

@@ -50,6 +50,7 @@ import org.enso.logger.akka.AkkaConverter
 import org.enso.polyglot.{HostAccessFactory, RuntimeOptions, RuntimeServerInfo}
 import org.enso.searcher.sql.{SqlDatabase, SqlSuggestionsRepo}
 import org.enso.text.{ContentBasedVersioning, Sha3_224VersionCalculator}
+import org.graalvm.polyglot.Engine
 import org.graalvm.polyglot.Context
 import org.graalvm.polyglot.io.MessageEndpoint
 import org.slf4j.event.Level
@@ -286,7 +287,7 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
   val stdInSink = new ObservableOutputStream
   val stdIn     = new ObservablePipedInputStream(stdInSink)
 
-  val context = Context
+  val builder = Context
     .newBuilder()
     .allowAllAccess(true)
     .allowHostAccess(new HostAccessFactory().allWithTypeMapping())
@@ -319,7 +320,22 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
         connection
       } else null
     })
-    .build()
+  if (
+    Engine
+      .newBuilder()
+      .allowExperimentalOptions(true)
+      .build
+      .getLanguages()
+      .containsKey("java")
+  ) {
+    builder
+      .option("java.ExposeNativeJavaVM", "true")
+      .option("java.Polyglot", "true")
+      .option("java.UseBindingsLoader", "true")
+      .allowCreateThread(true)
+  }
+
+  val context = builder.build()
   log.trace("Created Runtime context [{}].", context)
 
   system.eventStream.setLogLevel(AkkaConverter.toAkka(logLevel))

engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala

@@ -7,6 +7,7 @@ import org.enso.polyglot.debugger.{
   DebuggerSessionManagerEndpoint
 }
 import org.enso.polyglot.{HostAccessFactory, PolyglotContext, RuntimeOptions}
+import org.graalvm.polyglot.Engine
 import org.graalvm.polyglot.Context
 import org.slf4j.event.Level
 
@@ -51,12 +52,22 @@ class ContextFactory {
     executionEnvironment.foreach { name =>
       options.put("enso.ExecutionEnvironment", name)
     }
+    var javaHome = System.getenv("JAVA_HOME");
+    if (javaHome == null) {
+      javaHome = System.getProperty("java.home");
+    }
+    if (javaHome == null) {
+      throw new IllegalStateException("Specify JAVA_HOME environment property");
+    }
     val logLevelName = Converter.toJavaLevel(logLevel).getName
     val builder = Context
       .newBuilder()
       .allowExperimentalOptions(true)
       .allowAllAccess(true)
-      .allowHostAccess(new HostAccessFactory().allWithTypeMapping())
+      .allowHostAccess(
+        new HostAccessFactory()
+          .allWithTypeMapping()
+      )
       .option(RuntimeOptions.PROJECT_ROOT, projectRoot)
       .option(RuntimeOptions.STRICT_ERRORS, strictErrors.toString)
       .option(RuntimeOptions.WAIT_FOR_PENDING_SERIALIZATION_JOBS, "true")
@@ -95,10 +106,25 @@ class ContextFactory {
         "bin"
       ),
       "graalpy"
-    );
+    )
     if (graalpy.exists()) {
       builder.option("python.Executable", graalpy.getAbsolutePath());
     }
+    if (
+      Engine
+        .newBuilder()
+        .allowExperimentalOptions(true)
+        .build()
+        .getLanguages()
+        .containsKey("java")
+    ) {
+      builder
+        .option("java.ExposeNativeJavaVM", "true")
+        .option("java.Polyglot", "true")
+        .option("java.UseBindingsLoader", "true")
+        .option("java.JavaHome", javaHome)
+        .allowCreateThread(true)
+    }
     new PolyglotContext(builder.build)
   }
 }

engine/runtime/src/main/java/org/enso/interpreter/node/ExpressionNode.java

@@ -79,10 +79,19 @@ public void setSourceLocation(int sourceStartIndex, int sourceLength) {
    */
   @Override
   public SourceSection getSourceSection() {
+    var bounds = getSourceSectionBounds();
+    return bounds == null ? null : EnsoRootNode.findSourceSection(getRootNode(), bounds[0], bounds[1]);
+  }
+
+  public int[] getSourceSectionBounds() {
     if (this instanceof ExpressionNodeWrapper wrapper) {
-      return wrapper.getDelegateNode().getSourceSection();
+      return wrapper.getDelegateNode().getSourceSectionBounds();
     } else {
-      return EnsoRootNode.findSourceSection(getRootNode(), sourceStartIndex, sourceLength);
+      if (sourceStartIndex == EnsoRootNode.NO_SOURCE && sourceLength == EnsoRootNode.NO_SOURCE) {
+        return null;
+      } else {
+        return new int[] { sourceStartIndex, sourceLength };
+      }
     }
   }
 

engine/runtime/src/main/java/org/enso/interpreter/node/callable/function/StatementNode.java

@@ -3,7 +3,6 @@
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.instrumentation.StandardTags;
 import com.oracle.truffle.api.instrumentation.Tag;
-import com.oracle.truffle.api.source.SourceSection;
 import org.enso.interpreter.node.ClosureRootNode;
 import org.enso.interpreter.node.ExpressionNode;
 import org.enso.interpreter.runtime.tag.AvoidIdInstrumentationTag;
@@ -28,13 +27,13 @@ static StatementNode wrap(ExpressionNode node) {
   }
 
   @Override
-  public SourceSection getSourceSection() {
-    return node.getSourceSection();
+  public int[] getSourceSectionBounds() {
+    return node.getSourceSectionBounds();
   }
 
   @Override
   public boolean isInstrumentable() {
-    return getSourceSection() != null && node.isInstrumentable();
+    return getSourceSectionBounds() != null && node.isInstrumentable();
   }
 
   @Override

engine/runtime/src/main/java/org/enso/interpreter/node/callable/resolver/HostMethodCallNode.java

@@ -1,5 +1,6 @@
 package org.enso.interpreter.node.callable.resolver;
 
+import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.Cached.Shared;
 import com.oracle.truffle.api.dsl.GenerateUncached;
@@ -213,25 +214,25 @@ Object resolveHostMethod(
       Object[] args,
       @Shared("interop") @CachedLibrary(limit = "LIB_LIMIT") InteropLibrary members,
       @Shared("hostValueToEnsoNode") @Cached HostValueToEnsoNode hostValueToEnsoNode) {
+    var ctx = EnsoContext.get(this);
     try {
       return hostValueToEnsoNode.execute(members.invokeMember(self, symbol, args));
     } catch (UnsupportedMessageException | UnknownIdentifierException e) {
-      throw new IllegalStateException(
-          "Impossible to reach here. The member is checked to be invocable.");
+      CompilerDirectives.transferToInterpreter();
+      var err = ctx.getBuiltins().error().makeNotInvokable(self);
+      throw new PanicException(err, e, this);
     } catch (ArityException e) {
-      throw new PanicException(
-          EnsoContext.get(this)
-              .getBuiltins()
+      var err =
+          ctx.getBuiltins()
               .error()
-              .makeArityError(e.getExpectedMinArity(), e.getExpectedMaxArity(), e.getActualArity()),
-          this);
+              .makeArityError(e.getExpectedMinArity(), e.getExpectedMaxArity(), e.getActualArity());
+      throw new PanicException(err, this);
     } catch (UnsupportedTypeException e) {
-      throw new PanicException(
-          EnsoContext.get(this)
-              .getBuiltins()
+      var err =
+          ctx.getBuiltins()
               .error()
-              .makeUnsupportedArgumentsError(e.getSuppliedValues(), e.getMessage()),
-          this);
+              .makeUnsupportedArgumentsError(e.getSuppliedValues(), e.getMessage());
+      throw new PanicException(err, this);
     }
   }
 
@@ -268,8 +269,10 @@ Object resolveHostConstructor(
     try {
       return hostValueToEnsoNode.execute(instances.instantiate(self, args));
     } catch (UnsupportedMessageException e) {
-      throw new IllegalStateException(
-          "Impossible to reach here. The member is checked to be instantiable.");
+      CompilerDirectives.transferToInterpreter();
+      var ctx = EnsoContext.get(this);
+      var err = ctx.getBuiltins().error().makeNotInvokable(self);
+      throw new PanicException(err, e, this);
     } catch (ArityException e) {
       throw new PanicException(
           EnsoContext.get(this)

engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java

@@ -25,10 +25,9 @@ static AddToClassPathNode build() {
   @CompilerDirectives.TruffleBoundary
   @Specialization
   Object doExecute(Object path, @Cached ExpectStringNode expectStringNode) {
-    EnsoContext context = EnsoContext.get(this);
-    context
-        .getEnvironment()
-        .addToHostClassPath(context.getTruffleFile(new File(expectStringNode.execute(path))));
-    return context.getBuiltins().nothing();
+    var ctx = EnsoContext.get(this);
+    var file = ctx.getTruffleFile(new File(expectStringNode.execute(path)));
+    ctx.addToClassPath(file);
+    return ctx.getBuiltins().nothing();
   }
 }

engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/LookupClassNode.java

@@ -1,5 +1,6 @@
 package org.enso.interpreter.node.expression.builtin.interop.java;
 
+import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.Specialization;
 import com.oracle.truffle.api.nodes.Node;
@@ -18,6 +19,7 @@ static LookupClassNode build() {
   }
 
   @Specialization
+  @CompilerDirectives.TruffleBoundary
   Object doExecute(Object name, @Cached("build()") ExpectStringNode expectStringNode) {
     return EnsoContext.get(this).lookupJavaClass(expectStringNode.execute(name));
   }

engine/runtime/src/main/java/org/enso/interpreter/node/expression/literal/PatchableLiteralNode.java

@@ -3,7 +3,6 @@
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.nodes.Node;
 import com.oracle.truffle.api.nodes.NodeInfo;
-import com.oracle.truffle.api.source.SourceSection;
 import java.util.function.Predicate;
 import org.enso.compiler.core.IR;
 import org.enso.compiler.core.ir.Expression;
@@ -51,8 +50,8 @@ public boolean test(Expression ir) {
   }
 
   @Override
-  public SourceSection getSourceSection() {
-    return node.getSourceSection();
+  public int[] getSourceSectionBounds() {
+    return node.getSourceSectionBounds();
   }
 
   @Override