Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Using latest GraalVM bits in Enso #8885

Closed
wants to merge 4 commits into from
Closed

Using latest GraalVM bits in Enso #8885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2a92040
Using latest GraalVM bits in Enso
JaroslavTulach Jan 29, 2024
88fd039
Print out missing Module IDs
JaroslavTulach Jan 30, 2024
253ded5
Adding DSL dependency
JaroslavTulach Jan 30, 2024
160f2b8
Print out what's happening in ForeignMethodInvokeTest
JaroslavTulach Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 29 additions & 18 deletions build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ val graalVersion = "21.0.1"
// Version used for the Graal/Truffle related Maven packages
// Keep in sync with GraalVM.version. Do not change the name of this variable,
// it is used by the Rust build script via regex matching.
val graalMavenPackagesVersion = "23.1.0"
val graalMavenPackagesVersion = "24.1.0-SNAPSHOT"
val targetJavaVersion = "17"
val defaultDevEnsoVersion = "0.0.0-dev"
val ensoVersion = sys.env.getOrElse(
Expand Down Expand Up @@ -1228,6 +1228,7 @@ lazy val `polyglot-api` = project
"runtime-fat-jar"
) / Compile / fullClasspath).value,
libraryDependencies ++= Seq(
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided",
"org.graalvm.sdk" % "polyglot-tck" % graalMavenPackagesVersion % "provided",
"org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % "provided",
"com.google.flatbuffers" % "flatbuffers-java" % flatbuffersVersion,
Expand Down Expand Up @@ -1261,6 +1262,7 @@ lazy val `language-server` = (project in file("engine/language-server"))
"com.typesafe.akka" %% "akka-http-testkit" % akkaHTTPVersion % Test,
"org.scalatest" %% "scalatest" % scalatestVersion % Test,
"org.scalacheck" %% "scalacheck" % scalacheckVersion % Test,
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided",
"org.graalvm.sdk" % "polyglot-tck" % graalMavenPackagesVersion % "provided",
"org.eclipse.jgit" % "org.eclipse.jgit" % jgitVersion,
"org.bouncycastle" % "bcutil-jdk18on" % "1.76" % Test,
Expand Down Expand Up @@ -1432,6 +1434,7 @@ lazy val frgaalJavaCompilerSetting =
customFrgaalJavaCompilerSettings(targetJavaVersion)

def customFrgaalJavaCompilerSettings(targetJdk: String) = Seq(
resolvers += Resolver.mavenLocal,
Copy link
Member Author

@JaroslavTulach JaroslavTulach Jan 29, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This enables usage of JARs from ~/.m2/repository. To install them update GraalVM to: oracle/graal#8266 in particular commit 853702c407c8fb31116decfefddef87a71203154 and do:

graal/sdk$ mx maven-deploy
graal/truffle$ mx maven-deploy

these two commands install all Truffle & SDK JARs into local repository with version 24.1.0-SNAPSHOT. The sbt build can then find them.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As of 253ded5 I am able to execute:

enso$ sbt 
sbt> project runtime
sbt:runtime> testOnly *ForeignMethodInvokeTest -- -z org.enso.interpreter.test.ForeignMethodInvokeTest.testParallelInteropWithJavaScript

and that is enough for testing oracle/graal#8266

Compile / compile / compilers := FrgaalJavaCompiler.compilers(
(Compile / dependencyClasspath).value,
compilers.value,
Expand Down Expand Up @@ -1482,10 +1485,11 @@ lazy val `runtime-language-epb` =
Test / javaOptions ++= Seq(),
instrumentationSettings,
libraryDependencies ++= Seq(
"junit" % "junit" % junitVersion % Test,
"com.github.sbt" % "junit-interface" % junitIfVersion % Test,
"org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % "provided",
"org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion % "provided"
"junit" % "junit" % junitVersion % Test,
"com.github.sbt" % "junit-interface" % junitIfVersion % Test,
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided",
"org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % "provided",
"org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion % "provided"
)
)

Expand Down Expand Up @@ -1562,6 +1566,7 @@ lazy val `runtime-test-instruments` =
libraryDependencies ++= GraalVM.modules,
libraryDependencies ++= Seq(
"org.graalvm.sdk" % "polyglot-tck" % graalMavenPackagesVersion,
"org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion,
"org.graalvm.truffle" % "truffle-tck" % graalMavenPackagesVersion,
"org.graalvm.truffle" % "truffle-tck-common" % graalMavenPackagesVersion,
"org.graalvm.truffle" % "truffle-tck-tests" % graalMavenPackagesVersion,
Expand Down Expand Up @@ -1620,6 +1625,7 @@ lazy val runtime = (project in file("engine/runtime"))
"org.apache.tika" % "tika-core" % tikaVersion,
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided",
"org.graalvm.sdk" % "polyglot-tck" % graalMavenPackagesVersion % "provided",
"org.graalvm.sdk" % "collections" % graalMavenPackagesVersion % "provided",
"org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % "provided",
"org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion % "provided",
"org.graalvm.truffle" % "truffle-tck" % graalMavenPackagesVersion % Test,
Expand Down Expand Up @@ -1866,11 +1872,12 @@ lazy val `runtime-compiler` =
frgaalJavaCompilerSetting,
instrumentationSettings,
libraryDependencies ++= Seq(
"junit" % "junit" % junitVersion % Test,
"com.github.sbt" % "junit-interface" % junitIfVersion % Test,
"org.scalatest" %% "scalatest" % scalatestVersion % Test,
"org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided",
"com.lihaoyi" %% "fansi" % fansiVersion
"junit" % "junit" % junitVersion % Test,
"com.github.sbt" % "junit-interface" % junitIfVersion % Test,
"org.scalatest" %% "scalatest" % scalatestVersion % Test,
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided",
"org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided",
"com.lihaoyi" %% "fansi" % fansiVersion
)
)
.dependsOn(`runtime-parser`)
Expand Down Expand Up @@ -2085,14 +2092,15 @@ lazy val `engine-runner` = project
commands += WithDebugCommand.withDebug,
inConfig(Compile)(truffleRunOptionsSettings),
libraryDependencies ++= Seq(
"org.graalvm.sdk" % "polyglot-tck" % graalMavenPackagesVersion % Provided,
"org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Provided,
"commons-cli" % "commons-cli" % commonsCliVersion,
"com.monovore" %% "decline" % declineVersion,
"org.jline" % "jline" % jlineVersion,
"org.typelevel" %% "cats-core" % catsVersion,
"junit" % "junit" % junitVersion % Test,
"com.github.sbt" % "junit-interface" % junitIfVersion % Test
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided",
"org.graalvm.sdk" % "polyglot-tck" % graalMavenPackagesVersion % Provided,
"org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Provided,
"commons-cli" % "commons-cli" % commonsCliVersion,
"com.monovore" %% "decline" % declineVersion,
"org.jline" % "jline" % jlineVersion,
"org.typelevel" %% "cats-core" % catsVersion,
"junit" % "junit" % junitVersion % Test,
"com.github.sbt" % "junit-interface" % junitIfVersion % Test
),
run / connectInput := true
)
Expand Down Expand Up @@ -2592,6 +2600,7 @@ lazy val `std-base` = project
`base-polyglot-root` / "std-base.jar",
libraryDependencies ++= Seq(
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion,
"org.graalvm.sdk" % "collections" % graalMavenPackagesVersion,
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Collections are now separated into its own JAR. As the group ID suggests (org.graalvm.sdk) they are not part of the org.graalvm.polyglot ecosystem. We should probably stop using them and not expose their classes to libraries. CCing @radeusgd

"org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided"
),
Compile / packageBin := Def.task {
Expand Down Expand Up @@ -2708,6 +2717,7 @@ lazy val `std-table` = project
},
libraryDependencies ++= Seq(
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided",
"org.graalvm.sdk" % "collections" % graalMavenPackagesVersion,
"org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided",
"com.univocity" % "univocity-parsers" % univocityParsersVersion,
"org.apache.poi" % "poi-ooxml" % poiOoxmlVersion,
Expand Down Expand Up @@ -2797,6 +2807,7 @@ lazy val `std-database` = project
`database-polyglot-root` / "std-database.jar",
libraryDependencies ++= Seq(
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided",
"org.graalvm.sdk" % "collections" % graalMavenPackagesVersion % "provided",
"org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided",
"org.xerial" % "sqlite-jdbc" % sqliteVersion,
"org.postgresql" % "postgresql" % "42.4.0"
Expand Down
56 changes: 51 additions & 5 deletions engine/runtime/src/test/java/org/enso/interpreter/test/ForeignMethodInvokeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,44 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.util.Random;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Value;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;

public class ForeignMethodInvokeTest extends TestBase {
private static Context ctx;

@BeforeClass
public static void prepareCtx() {
ctx = defaultContextBuilder("enso", "js").build();
var r = new Random();
var counter = new long[1];
var b =
defaultContextBuilder("enso", "js")
.onDeniedThreadAccess(
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With onDeniedThreadAccess as proposed by oracle/graal#8266 we have a way to act like an Ethernet on collision and successfully orchestrate the two threads accessing the same context to run into completion.

More info on Ethernet collision detection and behavior - most important "Calculate and wait the random backoff".

(ex) -> {
System.err.println(
"Denied thread access: "
+ ex.getMessage()
+ " in "
+ Thread.currentThread().getName());
if (counter[0] > 500) {
throw ex;
}
try {
long waitTime = r.nextInt(100);
System.err.println("Waiting " + waitTime + " ms like Ethernet");
Thread.sleep(waitTime);
counter[0] += waitTime;
} catch (InterruptedException ignore) {
}
System.err.println("Trying again");
});
ctx = b.build();
}

@AfterClass
Expand Down Expand Up @@ -85,7 +109,6 @@ public void testInteropWithJavaScript() throws Exception {
assertEquals(12, res2.getArrayElement(2).asInt());
}

@Ignore
@Test
public void testParallelInteropWithJavaScript() throws Exception {
var source =
Expand All @@ -95,7 +118,9 @@ public void testParallelInteropWithJavaScript() throws Exception {
polyglot java import java.lang.Thread

foreign js js_array t f = \"\"\"
print("In JavaScript...")
f(300)
print("In JavaScript after waiting...")
return [1, 2, t]

third t = js_array t (delay-> Thread.sleep delay)
Expand All @@ -108,16 +133,37 @@ public void testParallelInteropWithJavaScript() throws Exception {
Executors.newSingleThreadExecutor()
.submit(
() -> {
return third.execute(12);
try {
Thread.sleep(100);
System.err.println(
"Entering context in executor thread in "
+ Thread.currentThread().getName());
ctx.enter();
System.err.println("Entered context in executor thread");
return third.execute(12);
} catch (InterruptedException ex) {
ex.printStackTrace();
return null;
} finally {
ctx.leave();
System.err.println("Left context in executor thread");
}
});
ctx.enter();
System.err.println("Entered context on main thread in " + Thread.currentThread().getName());
var res = third.execute(13);
System.err.println("Got result on main thread: " + res);
assertTrue("It is an array", res.hasArrayElements());
assertEquals(3, res.getArraySize());
assertEquals(1, res.getArrayElement(0).asInt());
assertEquals(2, res.getArrayElement(1).asInt());
assertEquals(13, res.getArrayElement(2).asInt());
ctx.leave();
System.err.println("Context left on main thread");

var res2 = future.get(10, TimeUnit.SECONDS);

var res2 = future.get();
System.err.println("Future result obtained: " + res2);

assertTrue("It is an array2", res2.hasArrayElements());
assertEquals(12, res2.getArrayElement(2).asInt());
Expand Down
31 changes: 18 additions & 13 deletions project/GraalVM.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,12 @@ import scala.collection.immutable.Seq
*/
object GraalVM {
// Keep in sync with graalMavenPackagesVersion in build.sbt
val version: String = "23.1.0"
val version: String = "24.1.0-SNAPSHOT"
private val pythonVersion = "23.1.0"
private val jsVersion = "23.1.0"
private val llvmVersion = "23.1.0"
private val regexVersion = "23.1.0"
private val toolVersion = "23.1.0"

/** The list of modules that are included in the `component` directory in engine distribution.
* When invoking the `java` command, these modules need to be put on the module-path.
Expand Down Expand Up @@ -52,39 +57,39 @@ object GraalVM {
*/

val pythonPkgs = Seq(
"org.graalvm.python" % "python-language" % version,
"org.graalvm.python" % "python-resources" % version,
"org.graalvm.python" % "python-language" % pythonVersion,
"org.graalvm.python" % "python-resources" % pythonVersion,
"org.bouncycastle" % "bcutil-jdk18on" % "1.76",
"org.bouncycastle" % "bcpkix-jdk18on" % "1.76",
"org.bouncycastle" % "bcprov-jdk18on" % "1.76",
"org.graalvm.llvm" % "llvm-api" % version,
"org.graalvm.llvm" % "llvm-api" % llvmVersion,
"org.graalvm.truffle" % "truffle-nfi" % version,
"org.graalvm.truffle" % "truffle-nfi-libffi" % version,
"org.graalvm.regex" % "regex" % version,
"org.graalvm.tools" % "profiler-tool" % version,
"org.graalvm.regex" % "regex" % regexVersion,
"org.graalvm.tools" % "profiler-tool" % toolVersion,
"org.graalvm.shadowed" % "json" % version,
"org.graalvm.shadowed" % "icu4j" % version,
"org.tukaani" % "xz" % "1.9"
)

val jsPkgs = Seq(
"org.graalvm.js" % "js-language" % version,
"org.graalvm.regex" % "regex" % version,
"org.graalvm.shadowed" % "icu4j" % version
"org.graalvm.js" % "js-language" % jsVersion,
"org.graalvm.regex" % "regex" % regexVersion,
"org.graalvm.shadowed" % "icu4j" % jsVersion
)

val chromeInspectorPkgs = Seq(
"org.graalvm.tools" % "chromeinspector-tool" % version,
"org.graalvm.tools" % "chromeinspector-tool" % toolVersion,
"org.graalvm.shadowed" % "json" % version,
"org.graalvm.tools" % "profiler-tool" % version
"org.graalvm.tools" % "profiler-tool" % toolVersion
)

val debugAdapterProtocolPkgs = Seq(
"org.graalvm.tools" % "dap-tool" % version
"org.graalvm.tools" % "dap-tool" % toolVersion
)

val insightPkgs = Seq(
"org.graalvm.tools" % "insight-tool" % version
"org.graalvm.tools" % "insight-tool" % toolVersion
)

val espressoPkgs = if ("espresso".equals(System.getenv("ENSO_JAVA"))) {
Expand Down
11 changes: 10 additions & 1 deletion project/JPMSUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,24 @@ object JPMSUtils {
)
}

val found = mutable.Seq()
val ret = cp.filter(dep => {
val moduleID = dep.metadata.get(AttributeKey[ModuleID]("moduleID")).get
shouldFilterModule(moduleID)
if (shouldFilterModule(moduleID)) {
found :+ moduleID
true
} else {
false
}
})
if (shouldContainAll) {
if (ret.size < distinctModules.size) {
log.error("Not all modules from classpath were found")
log.error(s"Returned (${ret.size}): $ret")
log.error(s"Expected: (${distinctModules.size}): $distinctModules")

val missing = distinctModules.diff(found)
log.error(s"Missing IDs: $missing")
}
}
ret
Expand Down
Loading