diff --git a/CHANGELOG.md b/CHANGELOG.md index d22291a754ac..4839d44ea863 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -839,8 +839,9 @@ - [Improve and colorize compiler's diagnostic messages][6931] - [Execute some runtime commands synchronously to avoid race conditions][6998] - [Scala 2.13.11 update][7010] -- [Improve parallel execution of commands and jobs in Language Server][7042] - [Add special handling for static method calls on Any][7033] +- [Improve parallel execution of commands and jobs in Language Server][7042] +- [Added retries when executing GraalVM updater][7079] [3227]: https://github.com/enso-org/enso/pull/3227 [3248]: https://github.com/enso-org/enso/pull/3248 @@ -959,8 +960,9 @@ [6931]: https://github.com/enso-org/enso/pull/6931 [6998]: https://github.com/enso-org/enso/pull/6998 [7010]: https://github.com/enso-org/enso/pull/7010 -[7042]: https://github.com/enso-org/enso/pull/7042 [7033]: https://github.com/enso-org/enso/pull/7033 +[7042]: https://github.com/enso-org/enso/pull/7042 +[7079]: https://github.com/enso-org/enso/pull/7079 # Enso 2.0.0-alpha.18 (2021-10-12) diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdater.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdater.scala index c68ad5b1a910..3d1460bb021c 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdater.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdater.scala @@ -1,11 +1,10 @@ package org.enso.runtimeversionmanager.components import java.nio.file.Path - import com.typesafe.scalalogging.Logger import scala.sys.process._ -import scala.util.{Success, Try} +import scala.util.{Failure, Success, Try} /** Module that manages components of the GraalVM distribution. * @@ -19,18 +18,19 @@ class GraalVMComponentUpdater(runtime: GraalRuntime) private val logger = Logger[GraalVMComponentUpdater] private val gu = runtime.findExecutable("gu") + /** Path to the GraalVM's updater. + * + * @return path that will be executed to call the updater + */ + protected def updaterExec: Path = gu + /** List the installed GraalVM components. * * @return the list of installed GraalVM components */ override def list(): Try[Seq[GraalVMComponent]] = { val command = Seq("list", "-v") - val process = Process( - gu.toAbsolutePath.toString +: command, - Some(runtime.javaHome.toFile), - ("JAVA_HOME", runtime.javaHome), - ("GRAALVM_HOME", runtime.javaHome) - ) + logger.trace("{} {}", gu, Properties(gu)) logger.debug( "Executing: JAVA_HOME={} GRRAALVM_HOME={} {} {}", @@ -40,10 +40,23 @@ class GraalVMComponentUpdater(runtime: GraalRuntime) command.mkString(" ") ) - for { - stdout <- Try(process.lazyLines(stderrLogger)) - _ = logger.trace(stdout.mkString(System.lineSeparator())) - } yield ListOut.parse(stdout.toVector) + val executor = new ExponentialBackoffRetry(5, logger) { + override def cmd: String = "list" + override def executeProcess( + logger: ProcessLogger + ): Try[LazyList[String]] = { + val process = Process( + updaterExec.toAbsolutePath.toString +: command, + Some(runtime.javaHome.toFile), + ("JAVA_HOME", runtime.javaHome), + ("GRAALVM_HOME", runtime.javaHome) + ) + Try(process.lazyLines(logger)) + } + } + executor + .execute() + .map(stdout => if (stdout.isEmpty) Seq() else ListOut.parse(stdout)) } /** Install the provided GraalVM components. @@ -53,12 +66,6 @@ class GraalVMComponentUpdater(runtime: GraalRuntime) override def install(components: Seq[GraalVMComponent]): Try[Unit] = { if (components.nonEmpty) { val command = "install" +: components.map(_.id) - val process = Process( - gu.toAbsolutePath.toString +: command, - Some(runtime.path.toFile), - ("JAVA_HOME", runtime.javaHome), - ("GRAALVM_HOME", runtime.javaHome) - ) logger.trace("{} {}", gu, Properties(gu)) logger.debug( "Executing: JAVA_HOME={} GRRAALVM_HOME={} {} {}", @@ -67,20 +74,78 @@ class GraalVMComponentUpdater(runtime: GraalRuntime) gu, command.mkString(" ") ) - for { - stdout <- Try(process.lazyLines(stderrLogger)) - _ = logger.trace(stdout.mkString(System.lineSeparator())) - } yield () + val executor = new ExponentialBackoffRetry(5, logger) { + override def cmd: String = "install" + override def executeProcess( + logger: ProcessLogger + ): Try[LazyList[String]] = { + val process = Process( + updaterExec.toAbsolutePath.toString +: command, + Some(runtime.path.toFile), + ("JAVA_HOME", runtime.javaHome), + ("GRAALVM_HOME", runtime.javaHome) + ) + Try(process.lazyLines(logger)) + } + } + executor.execute().map { stdout => + stdout.foreach(logger.trace(_)) + () + } } else { Success(()) } } - private def stderrLogger = - ProcessLogger(err => logger.trace("[stderr] {}", err)) } object GraalVMComponentUpdater { + abstract class ProcessWithRetries(maxRetries: Int, logger: Logger) { + def executeProcess(logger: ProcessLogger): Try[LazyList[String]] + + def cmd: String + + def execute(): Try[List[String]] = execute(0) + + protected def retryWait(retry: Int): Long + + private def execute(retry: Int): Try[List[String]] = { + val errors = scala.collection.mutable.ListBuffer[String]() + val processLogger = ProcessLogger(err => errors.addOne(err)) + executeProcess(processLogger) match { + case Success(stdout) => + Try(stdout.toList).recoverWith({ + case _ if retry < maxRetries => + try { + Thread.sleep(retryWait(retry)) + } catch { + case _: InterruptedException => + } + execute(retry + 1) + }) + case Failure(exception) if retry < maxRetries => + logger.warn("{} failed: {}. Retrying...", cmd, exception.getMessage) + try { + Thread.sleep(retryWait(retry)) + } catch { + case _: InterruptedException => + } + execute(retry + 1) + case Failure(exception) => + errors.foreach(logger.trace("[stderr] {}", _)) + Failure(exception) + } + } + } + + abstract class ExponentialBackoffRetry(maxRetries: Int, logger: Logger) + extends ProcessWithRetries(maxRetries, logger) { + override def retryWait(retry: Int): Long = { + 200 * 2.toLong ^ retry + } + + } + implicit private def pathToString(path: Path): String = path.toAbsolutePath.toString diff --git a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala b/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala index 085284e3556e..50cd80ebc1fa 100644 --- a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala +++ b/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala @@ -65,10 +65,54 @@ class GraalVMComponentUpdaterSpec extends AnyWordSpec with Matchers { ru.list() match { case Success(components) => - components should not be empty + val componentIds = components.map(_.id) + componentIds should (contain("graalvm") and contain("js")) case Failure(cause) => fail(cause) } + + var maxFailures = 3 + val ruSometimesFailing = new GraalVMComponentUpdater(graal) { + override def updaterExec: Path = if (maxFailures == 0) super.updaterExec + else { + maxFailures = maxFailures - 1 + OS.operatingSystem match { + case OS.Linux => Path.of("/bin/false") + case OS.MacOS => Path.of("/bin/false") + case OS.Windows => Path.of("foobar") + } + } + } + + ruSometimesFailing.list() match { + case Success(components) => + val componentIds = components.map(_.id) + componentIds should (contain("graalvm") and contain("js")) + case Failure(_) => + } + + var attempted = 0 + val ruAlwaysFailing = new GraalVMComponentUpdater(graal) { + override def updaterExec: Path = { + attempted = attempted + 1 + OS.operatingSystem match { + case OS.Linux => Path.of("/bin/false") + case OS.MacOS => Path.of("/bin/false") + case OS.Windows => Path.of("foobar") + } + } + } + + val expectedRetries = 5 + ruAlwaysFailing.list() match { + case Success(_) => + fail("expected `gu list` to always fail") + case Failure(_) => + if (attempted != (expectedRetries + 1)) + fail( + s"should have retried ${expectedRetries + 1} times, got $attempted" + ) + } } }