project-manager can start without prior GraalVM installation (#8410)

This is a follow-up of #7991. #7991 broken `runtime-version-manager`. This is mostly reverts.


### Important Notes

Launcher now correctly recognizes that the newest engine needs some runtime:
```sh
> java -jar launcher.jar list
Enso 2023.2.1-nightly.2023.10.31 -> GraalVM 23.0.0-java17.0.7
Enso 0.0.0-dev -> GraalVM 23.1.0-java21.0.1
```
(this has not worked before)

build.sbt

@@ -2623,7 +2623,7 @@ buildEngineDistribution := {
     log                 = log,
     jarModulesToCopy    = modulesToCopy ++ engineModules,
     graalVersion        = graalMavenPackagesVersion,
-    javaVersion         = javaVersion,
+    javaVersion         = graalVersion,
     ensoVersion         = ensoVersion,
     editionName         = currentEdition,
     sourceStdlibVersion = stdLibVersion,
@@ -2656,7 +2656,7 @@ buildEngineDistributionNoIndex := {
     log                 = log,
     jarModulesToCopy    = modulesToCopy ++ engineModules,
     graalVersion        = graalMavenPackagesVersion,
-    javaVersion         = javaVersion,
+    javaVersion         = graalVersion,
     ensoVersion         = ensoVersion,
     editionName         = currentEdition,
     sourceStdlibVersion = stdLibVersion,

lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/ProjectService.scala

@@ -299,9 +299,7 @@ class ProjectService[
           RuntimeVersionManagerFactory(distributionConfiguration)
             .makeRuntimeVersionManager(progressTracker, missingComponentAction)
         val engine = runtimeVersionManager.findOrInstallEngine(version)
-        if (engine.needsGraalDistribution) {
-          runtimeVersionManager.findOrInstallGraalRuntime(engine)
-        }
+        runtimeVersionManager.findOrInstallGraalRuntime(engine)
         ()
       }
       .mapRuntimeManagerErrors(th =>

lib/scala/project-manager/src/test/scala/org/enso/projectmanager/protocol/MissingComponentBehavior.scala

@@ -4,6 +4,7 @@ import io.circe.Json
 import nl.gn0s1s.bump.SemVer
 import org.enso.projectmanager.BaseServerSpec
 import org.enso.projectmanager.data.MissingComponentAction
+import org.enso.runtimeversionmanager.components.GraalVMVersion
 import org.enso.testkit.RetrySpec
 import org.scalatest.wordspec.AnyWordSpecLike
 
@@ -56,4 +57,26 @@ trait MissingComponentBehavior {
       client.expectTaskStarted()
     }
   }
+
+  /** This behaviour should be tested in a separate test suite, as it affects
+    * the test environment and if run together with other tests it could affect
+    * their results.
+    */
+  def correctlyHandleMissingRuntimeInPresenceOfEngine(): Unit = {
+    "make sure to check if the runtime is installed even if the engine was " +
+    "already installed" in {
+      uninstallRuntime(GraalVMVersion("23.2.0", "21.0.0"))
+
+      val client = new WsTestClient(address)
+      client.send(
+        buildRequest(defaultVersion, MissingComponentAction.Install)
+      )
+
+      /** We do not check for success here as we are concerned onyl that the
+        * installation is attempted. Installation and creating/opening projects
+        * are tested elsewhere.
+        */
+      client.expectTaskStarted()
+    }
+  }
 }

lib/scala/project-manager/src/test/scala/org/enso/projectmanager/protocol/ProjectCreateHandleMissingRuntimeSpec.scala

@@ -0,0 +1,26 @@
+package org.enso.projectmanager.protocol
+
+import org.enso.projectmanager.TestDistributionConfiguration
+import org.enso.runtimeversionmanager.runner.JVMSettings
+import org.enso.runtimeversionmanager.test.FakeReleases
+
+class ProjectCreateHandleMissingRuntimeSpec extends ProjectCreateSpecBase {
+  override val distributionConfiguration =
+    new TestDistributionConfiguration(
+      distributionRoot       = testDistributionRoot.toPath,
+      engineReleaseProvider  = FakeReleases.engineReleaseProvider,
+      runtimeReleaseProvider = FakeReleases.runtimeReleaseProvider,
+      discardChildOutput     = !debugChildLogs
+    ) {
+      override def defaultJVMSettings: JVMSettings = JVMSettings(
+        javaCommandOverride = None,
+        jvmOptions          = Seq()
+      )
+    }
+
+  override val engineToInstall = Some(defaultVersion)
+
+  "project/create" should {
+    behave like correctlyHandleMissingRuntimeInPresenceOfEngine()
+  }
+}

lib/scala/project-manager/src/test/scala/org/enso/projectmanager/protocol/ProjectOpenHandleMissingRuntimeSpec.scala

@@ -0,0 +1,15 @@
+package org.enso.projectmanager.protocol
+
+import nl.gn0s1s.bump.SemVer
+import org.enso.runtimeversionmanager.test.OverrideTestVersionSuite
+
+class ProjectOpenHandleMissingRuntimeSpec
+    extends ProjectOpenSpecBase
+    with OverrideTestVersionSuite {
+
+  override def testVersion: SemVer = SemVer(0, 0, 1)
+
+  "project/open" should {
+    behave like correctlyHandleMissingRuntimeInPresenceOfEngine()
+  }
+}

lib/scala/runtime-version-manager-test/src/main/resources/org/enso/runtimeversionmanager/test/fake-releases/enso/enso-0.0.1/manifest.yaml

@@ -1,6 +1,6 @@
 minimum-launcher-version: 0.0.0-dev
 minimum-project-manager-version: 0.0.0-dev
 graal-vm-version: 23.2.0
-graal-java-version: 21
+graal-java-version: 21.0.0
 jvm-options:
   - value: "-Denso.version.override=0.0.1"

lib/scala/runtime-version-manager-test/src/main/resources/org/enso/runtimeversionmanager/test/fake-releases/enso/enso-0.1.0/manifest.yaml

@@ -1,7 +1,7 @@
 minimum-launcher-version: 0.0.0-dev
 minimum-project-manager-version: 9999.0.0
 graal-vm-version: 23.2.0
-graal-java-version: 21
+graal-java-version: 21.0.0
 jvm-options:
   - value: "-Dpolyglot.compiler.IterativePartialEscape=true"
   - value: "-Denso.version.override=0.1.0"

lib/scala/runtime-version-manager-test/src/main/resources/org/enso/runtimeversionmanager/test/fake-releases/graalvm/jdk-21.0.0/graalvm-community-jdk-21.0.0_linux-x64_bin.tar.gz/graalvm-community-openjdk-21+35.1/bin/java

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "Fake JVM executable has been executed"

lib/scala/runtime-version-manager-test/src/main/resources/org/enso/runtimeversionmanager/test/fake-releases/graalvm/jdk-21.0.0/graalvm-community-jdk-21.0.0_macos-aarch64_bin.tar.gz/graalvm-community-openjdk-21+35.1/Contents/Home/bin/java

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "Fake JVM executable has been executed"

lib/scala/runtime-version-manager-test/src/main/resources/org/enso/runtimeversionmanager/test/fake-releases/graalvm/jdk-21.0.0/graalvm-community-jdk-21.0.0_macos-x64_bin.tar.gz/graalvm-community-openjdk-21+35.1/Contents/Home/bin/java

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "Fake JVM executable has been executed"

lib/scala/runtime-version-manager-test/src/main/resources/org/enso/runtimeversionmanager/test/fake-releases/graalvm/jdk-21.0.0/graalvm-community-jdk-21.0.0_windows-x64_bin.zip/graalvm-community-openjdk-21+35.1/bin/java.exe

@@ -0,0 +1 @@
+placeholder

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/NoopComponentUpdater.scala → lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NoopComponentUpdater.scala

@@ -1,4 +1,9 @@
-package org.enso.runtimeversionmanager.components
+package org.enso.runtimeversionmanager.test
+
+import org.enso.runtimeversionmanager.components.{
+  GraalVMComponent,
+  RuntimeComponentUpdater
+}
 
 import scala.util.Try
 

lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NoopComponentUpdaterFactory.scala

@@ -2,7 +2,6 @@ package org.enso.runtimeversionmanager.test
 
 import org.enso.runtimeversionmanager.components.{
   GraalRuntime,
-  NoopComponentUpdater,
   RuntimeComponentUpdater,
   RuntimeComponentUpdaterFactory
 }

lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala

@@ -13,6 +13,7 @@ import org.enso.runtimeversionmanager.test.{
   RuntimeVersionManagerTest,
   TestRuntimeVersionManagementUserInterface
 }
+import org.enso.runtimeversionmanager.components
 import org.enso.testkit.OsSpec
 
 class RuntimeVersionManagerSpec
@@ -42,19 +43,44 @@ class RuntimeVersionManagerSpec
       )
 
       val runtime = componentsManager.findGraalRuntime(engine)
-      runtime shouldBe empty
+      runtime.value.version shouldEqual GraalVMVersion("23.2.0", "21.0.0")
+      assert(
+        runtime.value.path.startsWith(distributionManager.paths.runtimes),
+        "Engine should be installed in the engines directory."
+      )
     }
 
-    "list installed engines" in {
+    "list installed engines and runtimes" in {
       val componentsManager = makeRuntimeVersionManager()
       val engineVersions =
-        Set(SemVer(0, 0, 0), SemVer(0, 0, 1), SemVer(0, 0, 1, Some("pre")))
+        Set(SemVer(0, 0, 0), SemVer(0, 0, 1), SemVer(0, 1, 0))
+      val runtimeVersions =
+        Set(
+          components.GraalVMVersion("1.0.0", "11"),
+          components.GraalVMVersion("23.2.0", "21.0.0")
+        )
       engineVersions.map(componentsManager.findOrInstallEngine)
 
       componentsManager
         .listInstalledEngines()
         .map(_.version)
         .toSet shouldEqual engineVersions
+      componentsManager
+        .listInstalledGraalRuntimes()
+        .map(_.version)
+        .toSet shouldEqual runtimeVersions
+
+      val runtime1 =
+        componentsManager
+          .findGraalRuntime(components.GraalVMVersion("1.0.0", "11"))
+          .value
+      componentsManager.findEnginesUsingRuntime(runtime1) should have length 1
+
+      val runtime2 =
+        componentsManager
+          .findGraalRuntime(components.GraalVMVersion("23.2.0", "21.0.0"))
+          .value
+      componentsManager.findEnginesUsingRuntime(runtime2) should have length 2
     }
 
     "preserve the broken mark when installing a broken release" in {
@@ -113,6 +139,42 @@ class RuntimeVersionManagerSpec
       exception.getMessage should include("Nightly releases expire")
     }
 
+    "uninstall the runtime iff it is not used by any engines" in {
+      val componentsManager = makeRuntimeVersionManager()
+      val engineVersions =
+        Seq(SemVer(0, 0, 0), SemVer(0, 0, 1), SemVer(0, 1, 0))
+      engineVersions.map(componentsManager.findOrInstallEngine)
+
+      componentsManager.listInstalledEngines() should have length 3
+      componentsManager.listInstalledGraalRuntimes() should have length 2
+
+      // remove the engine that shares the runtime with another one
+      val version1 = SemVer(0, 1, 0)
+      componentsManager.uninstallEngine(version1)
+      val engines1 = componentsManager.listInstalledEngines()
+      engines1 should have length 2
+      engines1.map(_.version) should not contain version1
+      componentsManager.listInstalledGraalRuntimes() should have length 2
+
+      // remove the second engine that shared the runtime
+      val version2 = SemVer(0, 0, 1)
+      componentsManager.uninstallEngine(version2)
+      val engines2 = componentsManager.listInstalledEngines()
+      engines2 should have length 1
+      engines2.map(_.version) should not contain version2
+      val runtimes2 = componentsManager.listInstalledGraalRuntimes()
+      runtimes2 should have length 1
+      runtimes2.map(_.version).head shouldEqual components.GraalVMVersion(
+        "1.0.0",
+        "11"
+      )
+
+      // remove the last engine
+      componentsManager.uninstallEngine(SemVer(0, 0, 0))
+      componentsManager.listInstalledEngines() should have length 0
+      componentsManager.listInstalledGraalRuntimes() should have length 0
+    }
+
     "correctly handle version depending on installer type" in {
       val projectManager =
         makeManagers(installerKind = InstallerKind.ProjectManager)._2
@@ -211,25 +273,25 @@ class RuntimeVersionManagerSpec
 
     "include both bundled and installed components in list" in {
       prepareBundle(
-        engineVersion  = SemVer(0, 0, 0),
-        runtimeVersion = GraalVMVersion("1.0.0", "11")
+        engineVersion  = SemVer(0, 0, 1),
+        runtimeVersion = GraalVMVersion("23.2.0", "21.0.0")
       )
       val manager = makeRuntimeVersionManager()
       manager.findOrInstallEngine(SemVer(0, 0, 1).withPreRelease("pre"))
 
       manager
         .listInstalledEngines()
         .map(_.version) should contain theSameElementsAs Seq(
-        SemVer(0, 0, 0),
+        SemVer(0, 0, 1),
         SemVer(0, 0, 1).withPreRelease("pre")
       )
 
       val runtimeVersions = manager.listInstalledGraalRuntimes().map(_.version)
       runtimeVersions.map(_.graalVersion) should contain theSameElementsAs Seq(
-        "1.0.0",
+        "23.2.0",
         "2.0.0"
       )
-      runtimeVersions.map(_.javaVersion).toSet shouldEqual Set("11")
+      runtimeVersions.map(_.javaVersion).toSet shouldEqual Set("21.0.0", "11")
     }
 
     "cope with semantic versioning of Java" in {
@@ -275,13 +337,8 @@ class RuntimeVersionManagerSpec
     FileSystem.writeTextFile(root / "manifest.yaml", manifest)
     val components = root / "component"
     Files.createDirectories(components)
-    if (runtimeVersion.isUnchained) {
-      Files.createDirectory(components / "runner")
-      makePlaceholder(components / "runner" / "runner.jar")
-    } else {
-      makePlaceholder(components / "runner.jar")
-    }
-    makePlaceholder(components / "runtime.jar")
+    makePlaceholder(components / "runner.jar")
+    FileSystem.writeTextFile(components / "runtime.jar", "placeholder")
   }
 
   private def fakeInstallRuntime(

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala

@@ -29,17 +29,6 @@ case class Engine(version: SemVer, path: Path, manifest: Manifest) {
     */
   def graalRuntimeVersion: GraalVMVersion = manifest.runtimeVersion
 
-  /** The GraalVM distribution policy changed a lot since GraalVM 23.0.0 for JDK 21.
-    * The newest GraalVM is now distributed as artifacts from the Maven central, and therefore,
-    * does not need to be downloaded at runtime.
-    *
-    * @see https://medium.com/graalvm/truffle-unchained-13887b77b62c
-    * @return true if a separate GraalVM distribution download is needed.
-    */
-  def needsGraalDistribution: Boolean = {
-    !graalRuntimeVersion.isUnchained
-  }
-
   /** A set of JVM options that should be added when running this engine.
     */
   def defaultJVMOptions: Seq[JVMOption] = manifest.jvmOptions
@@ -51,11 +40,13 @@ case class Engine(version: SemVer, path: Path, manifest: Manifest) {
 
   /** Path to the runner JAR.
     */
-  def runnerPath: Path = {
-    if (needsGraalDistribution) {
-      componentDirPath / "runner.jar"
+  def runnerPath: Option[Path] = {
+    if (graalRuntimeVersion.isUnchained) {
+      None
     } else {
-      componentDirPath / "runner" / "runner.jar"
+      Some(
+        componentDirPath / "runner.jar"
+      )
     }
   }
 
@@ -66,22 +57,25 @@ case class Engine(version: SemVer, path: Path, manifest: Manifest) {
   /** Checks if the installation is not corrupted and reports any issues as
     * failures.
     */
-  def ensureValid(): Try[Unit] =
-    if (!Files.exists(runnerPath))
-      Failure(
+  def ensureValid(): Try[Unit] = {
+    if (runnerPath.isDefined && !Files.exists(runnerPath.get)) {
+      return Failure(
         CorruptedComponentError(
           s"Engine's runner.jar (expected at " +
-          s"`${MaskedPath(runnerPath).applyMasking()}`) is missing."
+          s"`${MaskedPath(runnerPath.get).applyMasking()}`) is missing."
         )
       )
-    else if (!Files.exists(runtimePath))
-      Failure(
+    }
+    if (!Files.exists(runtimePath)) {
+      return Failure(
         CorruptedComponentError(
           s"`Engine's runtime.jar (expected at " +
           s"${MaskedPath(runtimePath).applyMasking()}`) is missing."
         )
       )
-    else Success(())
+    }
+    Success(())
+  }
 
   /** Returns if the engine release was marked as broken when it was being
     * installed.

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMVersion.scala

@@ -17,10 +17,7 @@ case class GraalVMVersion(graalVersion: String, javaVersion: String) {
 
   /** @inheritdoc
     */
-  override def toString: String = {
-    val unchained = if (isUnchained) "(unchained)" else ""
-    s"GraalVM $graalVersion Java $javaVersion" + unchained
-  }
+  override def toString: String = s"GraalVM $graalVersion Java $javaVersion"
 
   def graalMajorVersion: Int = graalVersion.split("\\.").head.toInt
 
@@ -32,9 +29,10 @@ case class GraalVMVersion(graalVersion: String, javaVersion: String) {
     }
   }
 
-  /** The GraalVM distribution policy changed a lot since GraalVM 23.0.0 for JDK 21.
-    * The newest GraalVM is now distributed as artifacts from the Maven central, and therefore,
-    * does not need to be downloaded at runtime.
+  /** The GraalVM distribution policy changed a lot since GraalVM 23.1.0 for JDK 21.
+    * Most of the components for the newest GraalVM distributions are distributed as
+    * artifacts from the Maven central. This mens there is no longer `gu` tool.
+    *
     * @see https://medium.com/graalvm/truffle-unchained-13887b77b62c
     * @return true if this version is associated with Truffle unchained.
     */