Add graalpy packages to the component directory

app/ide-desktop/lib/client/tasks/signArchivesMacOs.ts

@@ -80,6 +80,17 @@ async function ensoPackageSignables(resourcesDir: string): Promise<Signable[]> {
                 'com/sun/jna/darwin-x86-64/libjnidispatch.jnilib',
             ],
         ],
+        [
+            'component/python-resources-23.1.0.jar',
+            [
+                'META-INF/resources/darwin/*/lib/graalpy23.1/*.dylib',
+                'META-INF/resources/darwin/*/lib/graalpy23.1/modules/*.so',
+            ],
+        ],
+        [
+            `component/truffle-nfi-libffi-23.1.0.jar`,
+            ['META-INF/resources/nfi-native/libnfi/darwin/*/bin/libtrufflenfi.dylib'],
+        ],
         [
             `component/truffle-runtime-23.1.0.jar`,
             [

build.sbt

@@ -1628,6 +1628,15 @@ lazy val `runtime-with-instruments` =
         "org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion % Test,
         "org.slf4j"           % "slf4j-nop"             % slf4jVersion              % Benchmark
       ),
+      // Add all GraalVM packages with Runtime scope - we don't need them for compilation,
+      // just provide them at runtime (in module-path).
+      libraryDependencies ++= {
+        val necessaryModules =
+          GraalVM.modules.map(_.withConfigurations(Some(Runtime.name)))
+        val langs =
+          GraalVM.langsPkgs.map(_.withConfigurations(Some(Runtime.name)))
+        necessaryModules ++ langs
+      },
       // Note [Unmanaged Classpath]
       Test / unmanagedClasspath += (baseDirectory.value / ".." / ".." / "app" / "gui" / "view" / "graph-editor" / "src" / "builtin" / "visualization" / "native" / "inc"),
       // Filter module-info.java from the compilation
@@ -1659,9 +1668,15 @@ lazy val `runtime-with-instruments` =
       assembly / assemblyJarName := "runtime.jar",
       assembly / test := {},
       assembly / assemblyOutputPath := file("runtime.jar"),
-      // Exclude all the Truffle/Graal related artifacts from the Uber jar
       assembly / assemblyExcludedJars := {
-        val pkgsToExclude = JPMSUtils.componentModules
+        // bouncycastle jdk5 needs to be excluded from the Uber jar, as otherwise its packages would
+        // clash with packages in org.bouncycastle.jdk18 modules
+        val bouncyCastleJdk5 = Seq(
+          "org.bouncycastle" % "bcutil-jdk15on" % bcpkixJdk15Version,
+          "org.bouncycastle" % "bcpkix-jdk15on" % bcpkixJdk15Version,
+          "org.bouncycastle" % "bcprov-jdk15on" % bcpkixJdk15Version
+        )
+        val pkgsToExclude = JPMSUtils.componentModules ++ bouncyCastleJdk5
         val ourFullCp     = (Runtime / fullClasspath).value
         JPMSUtils.filterModulesFromClasspath(
           ourFullCp,

build/build/src/engine/context.rs

@@ -193,8 +193,22 @@ impl RunContext {
 
         // Setup GraalVM
         let graalvm =
-            crate::engine::deduce_graal(self.octocrab.clone(), &self.repo_root.build_sbt).await?;
+            engine::deduce_graal(self.octocrab.clone(), &self.repo_root.build_sbt).await?;
         graalvm.install_if_missing(&self.cache).await?;
+        let graal_version = engine::deduce_graal_bundle(&self.repo_root.build_sbt).await?;
+        let graalpy_version = graal_version.packages;
+
+        // Install GraalPy standalone distribution
+        // GraalPy has a version that corresponds to the `graalMavenPackagesVersion` variable in
+        // build.sbt
+        let graalpy = cache::goodie::graalpy::GraalPy {
+            client:  self.octocrab.clone(),
+            version: graalpy_version,
+            os:      self.paths.triple.os,
+            arch:    self.paths.triple.arch,
+        };
+        graalpy.install_if_missing(&self.cache).await?;
+        ide_ci::programs::graalpy::GraalPy.require_present().await?;
 
         prepare_simple_library_server.await??;
         Ok(())

build/ci_utils/src/cache/goodie.rs

@@ -9,6 +9,7 @@ use crate::cache::Cache;
 // ==============
 
 pub mod binaryen;
+pub mod graalpy;
 pub mod graalvm;
 pub mod sbt;
 

build/ci_utils/src/cache/goodie/graalpy.rs

@@ -0,0 +1,128 @@
+use crate::prelude::*;
+use regex::Regex;
+
+use crate::cache::goodie;
+use crate::cache::goodie::Goodie;
+use crate::cache::Cache;
+use crate::env::known::PATH;
+use crate::github::RepoRef;
+use crate::programs::graalpy::GraalPy as GraalPyProgram;
+
+
+pub const CE_BUILDS_REPOSITORY: RepoRef = RepoRef { owner: "oracle", name: "graalpython" };
+
+#[derive(Clone, Debug)]
+pub struct GraalPy {
+    pub client:  Octocrab,
+    pub version: Version,
+    pub os:      OS,
+    pub arch:    Arch,
+}
+
+fn graalpy_version_from_str(version_string: &str) -> Result<Version> {
+    let line = version_string.lines().find(|line| line.contains("GraalVM CE")).context(
+        "There is a Java environment available but it is not recognizable as GraalVM one.",
+    )?;
+    let re = Regex::new(r"GraalPy.*\((.+)\)").unwrap();
+    let caps = re.captures(line).unwrap();
+    Version::find_in_text(caps.get(1).context("graalpy wrong version text").unwrap().as_str())
+}
+
+async fn find_graalpy_version() -> Result<Version> {
+    let text = GraalPyProgram.version_string().await?;
+    graalpy_version_from_str(&text)
+}
+
+impl Goodie for GraalPy {
+    fn get(&self, cache: &Cache) -> BoxFuture<'static, Result<PathBuf>> {
+        goodie::download_try_future_url(self.url(), cache)
+    }
+
+    fn is_active(&self) -> BoxFuture<'static, Result<bool>> {
+        let expected_graalpy_version = self.version.clone();
+        async move {
+            let found_version = find_graalpy_version().await?;
+            ensure!(found_version == expected_graalpy_version, "GraalPy version mismatch. Expected {expected_graalpy_version}, found {found_version}.");
+            Ok(true)
+        }
+            .boxed()
+    }
+
+    fn activation_env_changes(&self, package_path: &Path) -> Result<Vec<crate::env::Modification>> {
+        let dir_entries = package_path
+            .read_dir()
+            .context("Failed to read GraalPy cache directory")?
+            .collect_vec();
+        let [graalpy_dir] = dir_entries.as_slice() else {
+            bail!("GraalPy cache directory should contain exactly one directory");
+        };
+        let graalpy_dir = match graalpy_dir {
+            Ok(dir_entry) => dir_entry,
+            Err(err) => bail!("Failed to read GraalPy cache directory: {}", err),
+        };
+        let dir_name = graalpy_dir.file_name();
+        let dir_name = dir_name.as_str();
+        ensure!(dir_name.contains("graalpy"));
+        ensure!(dir_name.contains(self.version.to_string_core().as_str()));
+        Ok(vec![crate::env::Modification::prepend_path(&PATH, graalpy_dir.path().join("bin"))])
+    }
+}
+
+impl GraalPy {
+    pub fn url(&self) -> BoxFuture<'static, Result<Url>> {
+        let this = self.clone();
+        let client = self.client.clone();
+        let arch_name = match self.arch {
+            Arch::X86_64 => "amd64",
+            Arch::AArch64 => "aarch64",
+            _ => unimplemented!("Unsupported architecture: {}", self.arch.to_string()),
+        };
+        async move {
+            let repo = CE_BUILDS_REPOSITORY.handle(&client);
+            let tag = format!("graal-{}", this.version);
+            let release = repo.find_release_by_tag(tag.as_str()).await?;
+            let asset_name =
+                format!("graalpy-community-{}-{}-{}", this.version, this.os, arch_name);
+            let asset =
+                crate::github::find_asset_url_by_text(&release, asset_name.as_str()).cloned();
+            asset
+        }
+        .boxed()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::cache::goodie::graalpy::graalpy_version_from_str;
+    use crate::cache::goodie::graalpy::GraalPy;
+    use octocrab::Octocrab;
+    use platforms::Arch;
+    use platforms::OS;
+    use semver::Version;
+
+    #[test]
+    fn version_recognize() {
+        let expected_version = Version::new(23, 1, 0);
+        let version_string = "GraalPy 3.10.8 (GraalVM CE Native 23.1.0)";
+        let found_version = graalpy_version_from_str(version_string).unwrap();
+        assert_eq!(found_version, expected_version);
+    }
+
+    #[test]
+    fn fetch_correct_url() {
+        let version = Version::new(23, 1, 0);
+        let client = Octocrab::builder().build().unwrap();
+        let graalpy = GraalPy { client, version, os: OS::Linux, arch: Arch::X86_64 };
+        let found_url_opt =
+            tokio::runtime::Runtime::new().unwrap().block_on(async { graalpy.url().await });
+        let found_url = match found_url_opt {
+            Ok(url) => url,
+            Err(err) => {
+                unreachable!("URL not found: {}", err);
+            }
+        };
+
+        let expected_url = "https://github.com/oracle/graalpython/releases/download/graal-23.1.0/graalpy-community-23.1.0-linux-amd64.tar.gz";
+        assert_eq!(found_url.as_str(), expected_url);
+    }
+}

build/ci_utils/src/programs.rs

@@ -13,6 +13,7 @@ pub mod docker;
 pub mod flatc;
 pub mod git;
 pub mod go;
+pub mod graalpy;
 pub mod java;
 pub mod javac;
 pub mod node;

build/ci_utils/src/programs/graalpy.rs

@@ -0,0 +1,10 @@
+use crate::prelude::*;
+
+#[derive(Clone, Copy, Debug)]
+pub struct GraalPy;
+
+impl Program for GraalPy {
+    fn executable_name(&self) -> &'static str {
+        "graalpy"
+    }
+}

distribution/engine/THIRD-PARTY/NOTICE

@@ -291,16 +291,31 @@ The license file can be found at `licenses/Bouncy_Castle_Licence.txt`.
 Copyright notices related to this dependency can be found in the directory `org.bouncycastle.bcpkix-jdk15on-1.70`.
 
 
+'bcpkix-jdk18on', licensed under the Bouncy Castle Licence, is distributed with the engine.
+The license file can be found at `licenses/Bouncy_Castle_Licence.txt`.
+Copyright notices related to this dependency can be found in the directory `org.bouncycastle.bcpkix-jdk18on-1.76`.
+
+
 'bcprov-jdk15on', licensed under the Bouncy Castle Licence, is distributed with the engine.
 The license file can be found at `licenses/Bouncy_Castle_Licence.txt`.
 Copyright notices related to this dependency can be found in the directory `org.bouncycastle.bcprov-jdk15on-1.70`.
 
 
+'bcprov-jdk18on', licensed under the Bouncy Castle Licence, is distributed with the engine.
+The license file can be found at `licenses/Bouncy_Castle_Licence.txt`.
+Copyright notices related to this dependency can be found in the directory `org.bouncycastle.bcprov-jdk18on-1.76`.
+
+
 'bcutil-jdk15on', licensed under the Bouncy Castle Licence, is distributed with the engine.
 The license file can be found at `licenses/Bouncy_Castle_Licence.txt`.
 Copyright notices related to this dependency can be found in the directory `org.bouncycastle.bcutil-jdk15on-1.70`.
 
 
+'bcutil-jdk18on', licensed under the Bouncy Castle Licence, is distributed with the engine.
+The license file can be found at `licenses/Bouncy_Castle_Licence.txt`.
+Copyright notices related to this dependency can be found in the directory `org.bouncycastle.bcutil-jdk18on-1.76`.
+
+
 'checker-qual', licensed under the The MIT License, is distributed with the engine.
 The license information can be found along with the copyright notices.
 Copyright notices related to this dependency can be found in the directory `org.checkerframework.checker-qual-3.33.0`.
@@ -316,11 +331,26 @@ The license file can be found at `licenses/MIT`.
 Copyright notices related to this dependency can be found in the directory `org.graalvm.js.js-language-23.1.0`.
 
 
+'llvm-api', licensed under the New BSD License (3-clause BSD license), is distributed with the engine.
+The license file can be found at `licenses/BSD-3-Clause`.
+Copyright notices related to this dependency can be found in the directory `org.graalvm.llvm.llvm-api-23.1.0`.
+
+
 'polyglot', licensed under the Universal Permissive License, Version 1.0, is distributed with the engine.
 The license file can be found at `licenses/Universal_Permissive_License__Version_1.0`.
 Copyright notices related to this dependency can be found in the directory `org.graalvm.polyglot.polyglot-23.1.0`.
 
 
+'python-language', licensed under the MIT License, is distributed with the engine.
+The license file can be found at `licenses/MIT`.
+Copyright notices related to this dependency can be found in the directory `org.graalvm.python.python-language-23.1.0`.
+
+
+'python-resources', licensed under the MIT License, is distributed with the engine.
+The license file can be found at `licenses/MIT`.
+Copyright notices related to this dependency can be found in the directory `org.graalvm.python.python-resources-23.1.0`.
+
+
 'regex', licensed under the Universal Permissive License, Version 1.0, is distributed with the engine.
 The license file can be found at `licenses/Universal_Permissive_License__Version_1.0`.
 Copyright notices related to this dependency can be found in the directory `org.graalvm.regex.regex-23.1.0`.
@@ -346,11 +376,31 @@ The license information can be found along with the copyright notices.
 Copyright notices related to this dependency can be found in the directory `org.graalvm.shadowed.icu4j-23.1.0`.
 
 
+'json', licensed under the Universal Permissive License, Version 1.0, is distributed with the engine.
+The license file can be found at `licenses/Universal_Permissive_License__Version_1.0`.
+Copyright notices related to this dependency can be found in the directory `org.graalvm.shadowed.json-23.1.0`.
+
+
+'profiler-tool', licensed under the GNU General Public License, version 2, with the Classpath Exception, is distributed with the engine.
+The license file can be found at `licenses/GNU_General_Public_License__version_2__with_the_Classpath_Exception`.
+Copyright notices related to this dependency can be found in the directory `org.graalvm.tools.profiler-tool-23.1.0`.
+
+
 'truffle-api', licensed under the Universal Permissive License, Version 1.0, is distributed with the engine.
 The license file can be found at `licenses/Universal_Permissive_License__Version_1.0`.
 Copyright notices related to this dependency can be found in the directory `org.graalvm.truffle.truffle-api-23.1.0`.
 
 
+'truffle-nfi', licensed under the Universal Permissive License, Version 1.0, is distributed with the engine.
+The license file can be found at `licenses/Universal_Permissive_License__Version_1.0`.
+Copyright notices related to this dependency can be found in the directory `org.graalvm.truffle.truffle-nfi-23.1.0`.
+
+
+'truffle-nfi-libffi', licensed under the Universal Permissive License, Version 1.0, is distributed with the engine.
+The license file can be found at `licenses/Universal_Permissive_License__Version_1.0`.
+Copyright notices related to this dependency can be found in the directory `org.graalvm.truffle.truffle-nfi-libffi-23.1.0`.
+
+
 'jline', licensed under the The BSD License, is distributed with the engine.
 The license file can be found at `licenses/BSD-3-Clause`.
 Copyright notices related to this dependency can be found in the directory `org.jline.jline-3.23.0`.
@@ -406,6 +456,11 @@ The license file can be found at `licenses/MIT`.
 Copyright notices related to this dependency can be found in the directory `org.slf4j.slf4j-api-2.0.9`.
 
 
+'xz', licensed under the Public Domain, is distributed with the engine.
+The license file can be found at `licenses/Public_Domain`.
+Copyright notices related to this dependency can be found in the directory `org.tukaani.xz-1.9`.
+
+
 'cats-core_2.13', licensed under the MIT, is distributed with the engine.
 The license file can be found at `licenses/MIT`.
 Copyright notices related to this dependency can be found in the directory `org.typelevel.cats-core_2.13-2.9.0`.