From 542357addcc415311b519be3e0c64adce6adeaaf Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Fri, 5 Jan 2024 10:18:39 +0100 Subject: [PATCH] Instructions to build Enso with Espresso for GraalVM for JDK21 (#8641) --- docs/infrastructure/native-image.md | 38 +++- .../java/org/enso/runner/ContextFactory.java | 210 ++++++++++++++++++ .../org/enso/runner/ContextFactory.scala | 164 -------------- .../src/main/scala/org/enso/runner/Main.scala | 96 ++++---- .../enso/interpreter/runtime/EnsoContext.java | 3 +- project/GraalVM.scala | 12 +- 6 files changed, 305 insertions(+), 218 deletions(-) create mode 100644 engine/runner/src/main/java/org/enso/runner/ContextFactory.java delete mode 100644 engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala diff --git a/docs/infrastructure/native-image.md b/docs/infrastructure/native-image.md index 541bb54853f9..e4112463b5a1 100644 --- a/docs/infrastructure/native-image.md +++ b/docs/infrastructure/native-image.md @@ -228,8 +228,33 @@ to allow use of some library functions (like `IO.println`) in the _Native Image_ built runner. The support can be enabled by setting environment variable `ENSO_JAVA=espresso` -and making sure Espresso is installed in GraalVM executing the Enso engine - -e.g. by running `graalvm/bin/gu install espresso`. Then execute: +and making sure Espresso is installed in the Enso engine `component` directory: + +```bash +enso$ built-distribution/enso-engine-*/enso-*/component/ +``` + +e.g. next to `js-language-*.jar` and other JARs. Download following these two +JARs (tested for version 23.1.1) and copy them into the directory: + +```bash +enso$ ls built-distribution/enso-engine-*/enso-*/component/espresso-* +built-distribution/enso-engine-0.0.0-dev-linux-amd64/enso-0.0.0-dev/component/espresso-language-23.1.1.jar +built-distribution/enso-engine-0.0.0-dev-linux-amd64/enso-0.0.0-dev/component/espresso-libs-resources-linux-amd64-23.1.1.jar +built-distribution/enso-engine-0.0.0-dev-linux-amd64/enso-0.0.0-dev/component/espresso-runtime-resources-linux-amd64-23.1.1.jar +``` + +the libraries can be found at +[Maven Central](https://repo1.maven.org/maven2/org/graalvm/espresso/). Version +`23.1.1` is known to work. + +Alternatively just build the Enso code with `ENSO_JAVA=espresso` specified + +```bash +enso$ ENSO_JAVA=espresso sbt --java-home /graalvm buildEngineDistribution +``` + +Then you can verify the support works: ```bash $ cat >hello.enso @@ -237,7 +262,7 @@ import Standard.Base.IO main = IO.println <| "Hello World!" -$ ENSO_JAVA=espresso ./enso-x.y.z-dev/bin/enso --run hello.enso +$ ENSO_JAVA=espresso ./built-distribution/enso-engine-*/enso-*/bin/enso --run hello.enso ``` Unless you see a warning containing _"No language for id java found."_ your code @@ -250,13 +275,12 @@ $ JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,address=5005 ENSO_JAVA=espresso e ``` Espresso support works also with -[native image support](#engine-runner-configuration). Just make sure Espresso is -installed in your GraalVM (via `gu install espresso`) and then rebuild the -`runner` executable: +[native image support](#engine-runner-configuration). Just make sure +`ENSO_JAVA=espresso` is specified when building the `runner` executable: ```bash enso$ rm runner -enso$ sbt --java-home /graalvm +enso$ ENSO_JAVA=espresso sbt --java-home /graalvm sbt> engine-runner/buildNativeImage ``` diff --git a/engine/runner/src/main/java/org/enso/runner/ContextFactory.java b/engine/runner/src/main/java/org/enso/runner/ContextFactory.java new file mode 100644 index 000000000000..c22f79d0a272 --- /dev/null +++ b/engine/runner/src/main/java/org/enso/runner/ContextFactory.java @@ -0,0 +1,210 @@ +package org.enso.runner; + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Map; +import org.enso.logger.Converter; +import org.enso.logger.JulHandler; +import org.enso.logger.LoggerSetup; +import org.enso.polyglot.HostAccessFactory; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.PolyglotContext; +import org.enso.polyglot.RuntimeOptions; +import org.enso.polyglot.debugger.DebugServerInfo; +import org.enso.polyglot.debugger.DebuggerSessionManagerEndpoint; +import org.graalvm.polyglot.Context; +import org.slf4j.event.Level; + +/** + * Builder to create a new Graal polyglot context. + * + * @param projectRoot root of the project the interpreter is being run in (or empty if ran outside + * of any projects) + * @param in the input stream for standard in + * @param out the output stream for standard out + * @param repl the Repl manager to use for this context + * @param logLevel the log level for this context + * @param enableIrCaches whether or not IR caching should be enabled + * @param disablePrivateCheck If `private` keyword should be disabled. + * @param strictErrors whether or not to use strict errors + * @param useGlobalIrCacheLocation whether or not to use the global IR cache location + * @param options additional options for the Context + * @param executionEnvironment optional name of the execution environment to use during execution + * @param warningsLimit maximal number of warnings reported to the user + */ +final class ContextFactory { + private String projectRoot; + private InputStream in; + private OutputStream out; + private Repl repl; + private Level logLevel; + private boolean logMasking; + private boolean enableIrCaches; + private boolean disablePrivateCheck; + private boolean strictErrors; + private boolean useGlobalIrCacheLocation = true; + private boolean enableAutoParallelism; + private String executionEnvironment; + private int warningsLimit = 100; + private java.util.Map options = java.util.Collections.emptyMap(); + + private ContextFactory() {} + + public static ContextFactory create() { + return new ContextFactory(); + } + + public ContextFactory projectRoot(String projectRoot) { + this.projectRoot = projectRoot; + return this; + } + + public ContextFactory in(InputStream in) { + this.in = in; + return this; + } + + public ContextFactory out(OutputStream out) { + this.out = out; + return this; + } + + public ContextFactory repl(Repl repl) { + this.repl = repl; + return this; + } + + public ContextFactory logLevel(Level logLevel) { + this.logLevel = logLevel; + return this; + } + + public ContextFactory logMasking(boolean logMasking) { + this.logMasking = logMasking; + return this; + } + + public ContextFactory enableIrCaches(boolean enableIrCaches) { + this.enableIrCaches = enableIrCaches; + return this; + } + + public ContextFactory disablePrivateCheck(boolean disablePrivateCheck) { + this.disablePrivateCheck = disablePrivateCheck; + return this; + } + + public ContextFactory strictErrors(boolean strictErrors) { + this.strictErrors = strictErrors; + return this; + } + + public ContextFactory useGlobalIrCacheLocation(boolean useGlobalIrCacheLocation) { + this.useGlobalIrCacheLocation = useGlobalIrCacheLocation; + return this; + } + + public ContextFactory enableAutoParallelism(boolean enableAutoParallelism) { + this.enableAutoParallelism = enableAutoParallelism; + return this; + } + + public ContextFactory executionEnvironment(String executionEnvironment) { + this.executionEnvironment = executionEnvironment; + return this; + } + + public ContextFactory warningsLimit(int warningsLimit) { + this.warningsLimit = warningsLimit; + return this; + } + + public ContextFactory options(Map options) { + this.options = options; + return this; + } + + PolyglotContext build() { + if (executionEnvironment != null) { + options.put("enso.ExecutionEnvironment", executionEnvironment); + } + var julLogLevel = Converter.toJavaLevel(logLevel); + var logLevelName = julLogLevel.getName(); + var builder = + Context.newBuilder() + .allowExperimentalOptions(true) + .allowAllAccess(true) + .allowHostAccess(new HostAccessFactory().allWithTypeMapping()) + .option(RuntimeOptions.PROJECT_ROOT, projectRoot) + .option(RuntimeOptions.STRICT_ERRORS, Boolean.toString(strictErrors)) + .option(RuntimeOptions.WAIT_FOR_PENDING_SERIALIZATION_JOBS, "true") + .option( + RuntimeOptions.USE_GLOBAL_IR_CACHE_LOCATION, + Boolean.toString(useGlobalIrCacheLocation)) + .option(RuntimeOptions.DISABLE_IR_CACHES, Boolean.toString(!enableIrCaches)) + .option(RuntimeOptions.DISABLE_PRIVATE_CHECK, Boolean.toString(disablePrivateCheck)) + .option(DebugServerInfo.ENABLE_OPTION, "true") + .option(RuntimeOptions.LOG_MASKING, Boolean.toString(logMasking)) + .options(options) + .option(RuntimeOptions.ENABLE_AUTO_PARALLELISM, Boolean.toString(enableAutoParallelism)) + .option(RuntimeOptions.WARNINGS_LIMIT, Integer.toString(warningsLimit)) + .option("js.foreign-object-prototype", "true") + .out(out) + .in(in) + .serverTransport( + (uri, peer) -> + DebugServerInfo.URI.equals(uri.toString()) + ? new DebuggerSessionManagerEndpoint(repl, peer) + : null); + + builder.option(RuntimeOptions.LOG_LEVEL, logLevelName); + var logHandler = JulHandler.get(); + var logLevels = LoggerSetup.get().getConfig().getLoggers(); + if (logLevels.hasEnsoLoggers()) { + logLevels + .entrySet() + .forEach( + (entry) -> + builder.option( + "log." + LanguageInfo.ID + "." + entry.getKey() + ".level", + Converter.toJavaLevel(entry.getValue()).getName())); + } + builder.logHandler(logHandler); + + var graalpy = + new File( + new File(new File(new File(new File(projectRoot), "polyglot"), "python"), "bin"), + "graalpy"); + if (graalpy.exists()) { + builder.option("python.Executable", graalpy.getAbsolutePath()); + } + if (ENGINE_HAS_JAVA) { + var javaHome = System.getProperty("java.home"); + if (javaHome == null) { + javaHome = System.getenv("JAVA_HOME"); + } + if (javaHome == null) { + throw new IllegalStateException("Specify JAVA_HOME environment property"); + } + builder + .option("java.ExposeNativeJavaVM", "true") + .option("java.Polyglot", "true") + .option("java.UseBindingsLoader", "true") + .option("java.JavaHome", javaHome) + .allowCreateThread(true); + } + return new PolyglotContext(builder.build()); + } + + /** + * Checks whether the polyglot engine has Espresso. Recorded as static constant to be remembered + * in AOT mode. + */ + private static final boolean ENGINE_HAS_JAVA; + + static { + var modules = ModuleLayer.boot().modules().stream(); + ENGINE_HAS_JAVA = modules.anyMatch(m -> "org.graalvm.espresso".equals(m.getName())); + } +} diff --git a/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala b/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala deleted file mode 100644 index 4f7baedd7bf1..000000000000 --- a/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala +++ /dev/null @@ -1,164 +0,0 @@ -package org.enso.runner - -import org.enso.logger.{Converter, JulHandler, LoggerSetup} -import org.enso.polyglot.debugger.{ - DebugServerInfo, - DebuggerSessionManagerEndpoint -} -import org.enso.polyglot.{ - HostAccessFactory, - LanguageInfo, - PolyglotContext, - RuntimeOptions -} -import org.graalvm.polyglot.{Context, Engine} -import org.slf4j.event.Level - -import java.io.{ByteArrayOutputStream, File, InputStream, OutputStream} -import scala.util.{Failure, Success, Using} - -/** Utility class for creating Graal polyglot contexts. - */ -class ContextFactory { - - /** Creates a new Graal polyglot context. - * - * @param projectRoot root of the project the interpreter is being run in - * (or empty if ran outside of any projects) - * @param in the input stream for standard in - * @param out the output stream for standard out - * @param repl the Repl manager to use for this context - * @param logLevel the log level for this context - * @param enableIrCaches whether or not IR caching should be enabled - * @param disablePrivateCheck If `private` keyword should be disabled. - * @param strictErrors whether or not to use strict errors - * @param useGlobalIrCacheLocation whether or not to use the global IR cache - * location - * @param options additional options for the Context - * @param executionEnvironment optional name of the execution environment to use during execution - * @param warningsLimit maximal number of warnings reported to the user - * @return configured Context instance - */ - def create( - projectRoot: String = "", - in: InputStream, - out: OutputStream, - repl: Repl, - logLevel: Level, - logMasking: Boolean, - enableIrCaches: Boolean, - disablePrivateCheck: Boolean = false, - strictErrors: Boolean = false, - useGlobalIrCacheLocation: Boolean = true, - enableAutoParallelism: Boolean = false, - executionEnvironment: Option[String] = None, - warningsLimit: Int = 100, - options: java.util.Map[String, String] = java.util.Collections.emptyMap - ): PolyglotContext = { - executionEnvironment.foreach { name => - options.put("enso.ExecutionEnvironment", name) - } - var javaHome = System.getenv("JAVA_HOME"); - if (javaHome == null) { - javaHome = System.getProperty("java.home"); - } - if (javaHome == null) { - throw new IllegalStateException("Specify JAVA_HOME environment property"); - } - val julLogLevel = Converter.toJavaLevel(logLevel) - val logLevelName = julLogLevel.getName - val builder = Context - .newBuilder() - .allowExperimentalOptions(true) - .allowAllAccess(true) - .allowHostAccess( - new HostAccessFactory() - .allWithTypeMapping() - ) - .option(RuntimeOptions.PROJECT_ROOT, projectRoot) - .option(RuntimeOptions.STRICT_ERRORS, strictErrors.toString) - .option(RuntimeOptions.WAIT_FOR_PENDING_SERIALIZATION_JOBS, "true") - .option( - RuntimeOptions.USE_GLOBAL_IR_CACHE_LOCATION, - useGlobalIrCacheLocation.toString - ) - .option(RuntimeOptions.DISABLE_IR_CACHES, (!enableIrCaches).toString) - .option( - RuntimeOptions.DISABLE_PRIVATE_CHECK, - disablePrivateCheck.toString - ) - .option(DebugServerInfo.ENABLE_OPTION, "true") - .option(RuntimeOptions.LOG_MASKING, logMasking.toString) - .options(options) - .option( - RuntimeOptions.ENABLE_AUTO_PARALLELISM, - enableAutoParallelism.toString - ) - .option( - RuntimeOptions.WARNINGS_LIMIT, - warningsLimit.toString - ) - .option("js.foreign-object-prototype", "true") - .out(out) - .in(in) - .serverTransport { (uri, peer) => - if (uri.toString == DebugServerInfo.URI) { - new DebuggerSessionManagerEndpoint(repl, peer) - } else null - } - - builder.option(RuntimeOptions.LOG_LEVEL, logLevelName) - val logHandler = JulHandler.get() - val logLevels = LoggerSetup.get().getConfig.getLoggers - if (logLevels.hasEnsoLoggers()) { - logLevels.entrySet().forEach { entry => - builder.option( - s"log.${LanguageInfo.ID}.${entry.getKey}.level", - Converter.toJavaLevel(entry.getValue).getName - ) - } - } - builder - .logHandler(logHandler) - - val graalpy = new File( - new File( - new File(new File(new File(projectRoot), "polyglot"), "python"), - "bin" - ), - "graalpy" - ) - if (graalpy.exists()) { - builder.option("python.Executable", graalpy.getAbsolutePath()); - } - if (engineHasJava()) { - builder - .option("java.ExposeNativeJavaVM", "true") - .option("java.Polyglot", "true") - .option("java.UseBindingsLoader", "true") - .option("java.JavaHome", javaHome) - .allowCreateThread(true) - } - new PolyglotContext(builder.build) - } - - /** Checks whether the polyglot engine has Espresso. - * - * Creates a temporary polyglot engine for that and makes sure that it is closed. - */ - private def engineHasJava(): Boolean = { - Using( - Engine - .newBuilder() - .allowExperimentalOptions(true) - .out(new ByteArrayOutputStream()) - .err(new ByteArrayOutputStream()) - .build() - ) { engine => - engine.getLanguages.containsKey("java") - } match { - case Success(ret) => ret - case Failure(ex) => throw new IllegalStateException("unreachable", ex) - } - } -} diff --git a/engine/runner/src/main/scala/org/enso/runner/Main.scala b/engine/runner/src/main/scala/org/enso/runner/Main.scala index 88474caf8888..2e0ac2e3f97f 100644 --- a/engine/runner/src/main/scala/org/enso/runner/Main.scala +++ b/engine/runner/src/main/scala/org/enso/runner/Main.scala @@ -546,17 +546,19 @@ object Main { exitFail() } - val context = new ContextFactory().create( - packagePath, - System.in, - System.out, - Repl(makeTerminalForRepl()), - logLevel, - logMasking, - enableIrCaches = true, - strictErrors = true, - useGlobalIrCacheLocation = shouldUseGlobalCache - ) + val context = ContextFactory + .create() + .projectRoot(packagePath) + .in(System.in) + .out(System.out) + .repl(Repl(makeTerminalForRepl())) + .logLevel(logLevel) + .logMasking(logMasking) + .enableIrCaches(true) + .strictErrors(true) + .useGlobalIrCacheLocation(shouldUseGlobalCache) + .build + val topScope = context.getTopScope try { topScope.compile(shouldCompileDependencies) @@ -628,21 +630,25 @@ object Main { if (inspect) { options.put("inspect", "") } - val context = new ContextFactory().create( - projectRoot, - System.in, - System.out, - Repl(makeTerminalForRepl()), - logLevel, - logMasking, - enableIrCaches, - disablePrivateCheck, - strictErrors = true, - enableAutoParallelism = enableAutoParallelism, - executionEnvironment = executionEnvironment, - warningsLimit = warningsLimit, - options = options - ) + val context = ContextFactory + .create() + .projectRoot(projectRoot) + .in(System.in) + .out(System.out) + .repl(Repl(makeTerminalForRepl())) + .logLevel(logLevel) + .logMasking(logMasking) + .enableIrCaches(enableIrCaches) + .disablePrivateCheck(disablePrivateCheck) + .strictErrors(true) + .enableAutoParallelism(enableAutoParallelism) + .executionEnvironment( + if (executionEnvironment.isDefined) executionEnvironment.get else null + ) + .warningsLimit(warningsLimit) + .options(options) + .build + if (projectMode) { PackageManager.Default.loadPackage(file) match { case Success(pkg) => @@ -703,15 +709,16 @@ object Main { logMasking: Boolean, enableIrCaches: Boolean ): Unit = { - val executionContext = new ContextFactory().create( - path, - System.in, - System.out, - Repl(makeTerminalForRepl()), - logLevel, - logMasking, - enableIrCaches - ) + val executionContext = ContextFactory + .create() + .projectRoot(path) + .in(System.in) + .out(System.out) + .repl(Repl(makeTerminalForRepl())) + .logLevel(logLevel) + .logMasking(logMasking) + .enableIrCaches(enableIrCaches) + .build val file = new File(path) val pkg = PackageManager.Default.fromDirectory(file) @@ -911,15 +918,16 @@ object Main { val replModuleName = "Internal_Repl_Module___" val projectRoot = projectPath.getOrElse("") val context = - new ContextFactory().create( - projectRoot, - System.in, - System.out, - Repl(makeTerminalForRepl()), - logLevel, - logMasking, - enableIrCaches - ) + ContextFactory + .create() + .projectRoot(projectRoot) + .in(System.in) + .out(System.out) + .repl(Repl(makeTerminalForRepl())) + .logLevel(logLevel) + .logMasking(logMasking) + .enableIrCaches(enableIrCaches) + .build val mainModule = context.evalModule(dummySourceToTriggerRepl, replModuleName) runMain( diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index 3e393aa3b6a8..d959c9d88f3f 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -551,8 +551,7 @@ private Object findGuestJava() throws IllegalStateException { logger.log( Level.SEVERE, "Environment variable ENSO_JAVA=" + envJava + ", but " + ex.getMessage()); - logger.log( - Level.SEVERE, "Use " + System.getProperty("java.home") + "/bin/gu install espresso"); + logger.log(Level.SEVERE, "Copy missing libraries to components directory"); logger.log(Level.SEVERE, "Continuing in regular Java mode"); } else { var ise = new IllegalStateException(ex.getMessage()); diff --git a/project/GraalVM.scala b/project/GraalVM.scala index 1220d2a31373..57a227e9c136 100644 --- a/project/GraalVM.scala +++ b/project/GraalVM.scala @@ -87,9 +87,19 @@ object GraalVM { "org.graalvm.tools" % "insight-tool" % version ) + val espressoPkgs = if ("espresso".equals(System.getenv("ENSO_JAVA"))) { + Seq( + "org.graalvm.espresso" % "espresso-language" % version, + "org.graalvm.espresso" % "espresso-libs-resources-linux-amd64" % version, + "org.graalvm.espresso" % "espresso-runtime-resources-linux-amd64" % version + ) + } else { + Seq() + } + val toolsPkgs = chromeInspectorPkgs ++ debugAdapterProtocolPkgs ++ insightPkgs - val langsPkgs = jsPkgs ++ pythonPkgs + val langsPkgs = jsPkgs ++ pythonPkgs ++ espressoPkgs /** Augments a state transition to do GraalVM version check. *