Add timeout retries for project-related operations (#7173)

It is relatively easy to reach timeouts on weak systems for startup operations on project manager. Once a timeout is reached, startup will not proceed any further. This PR is a bit of an experiment. It adds adds timeout retries to give a bit of a leeway to under-powered machines and to log some progress on the way, so that we know that certain actions are still in progress.
enso-org · Jul 3, 2023 · 414b124 · 414b124
1 parent 4fbe7e3
commit 414b124
Show file tree

Hide file tree

Showing 8 changed files with 132 additions and 104 deletions.
diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala
@@ -472,10 +472,12 @@ class PackageManager[F](implicit val fileSystem: FileSystem[F]) {
   private def copyResource(from: URI, to: F): Unit = {
     val fromStream = getClass.getResourceAsStream(from.toString)
     val toStream   = to.newOutputStream
-    try PackageManager.copyStream(fromStream, toStream)
-    finally {
-      fromStream.close()
-      toStream.close()
+    if (fromStream != null && toStream != null) {
+      try PackageManager.copyStream(fromStream, toStream)
+      finally {
+        fromStream.close()
+        toStream.close()
+      }
     }
   }
 }

diff --git a/lib/scala/project-manager/src/main/resources/application.conf b/lib/scala/project-manager/src/main/resources/application.conf
@@ -66,6 +66,7 @@ project-manager {
     boot-timeout = 40 seconds
     shutdown-timeout = 20 seconds
     socket-close-timeout = 15 seconds
+    retries = 5
   }
 
   tutorials {

diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/boot/configuration.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/boot/configuration.scala
@@ -63,13 +63,18 @@ object configuration {
     *
     * @param ioTimeout a timeout for IO operations
     * @param requestTimeout a timeout for JSON RPC request timeout
+    * @param bootTimeout a timeout for booting process
+    * @param shutdownTimeout a timeout for shutdown request
+    * @param socketCloseTimeout a timeout for closing the socket
+    * @param retries a number of retries attempted when timeout is reached
     */
   case class TimeoutConfig(
     ioTimeout: FiniteDuration,
     requestTimeout: FiniteDuration,
     bootTimeout: FiniteDuration,
     shutdownTimeout: FiniteDuration,
-    socketCloseTimeout: FiniteDuration
+    socketCloseTimeout: FiniteDuration,
+    retries: Int
   )
 
   /** A configuration object for networking.

diff --git a/...la/project-manager/src/main/scala/org/enso/projectmanager/protocol/ClientController.scala b/...la/project-manager/src/main/scala/org/enso/projectmanager/protocol/ClientController.scala
@@ -53,15 +53,17 @@ class ClientController[F[+_, +_]: Exec: CovariantFlatMap: ErrorChannel](
         .props[F](
           globalConfigService,
           projectService,
-          timeoutConfig.requestTimeout
+          timeoutConfig.requestTimeout,
+          timeoutConfig.retries
         ),
       ProjectDelete -> ProjectDeleteHandler
         .props[F](projectService, timeoutConfig.requestTimeout),
       ProjectOpen -> ProjectOpenHandler
         .props[F](
           clientId,
           projectService,
-          timeoutConfig.bootTimeout
+          timeoutConfig.bootTimeout,
+          timeoutConfig.retries
         ),
       ProjectClose -> ProjectCloseHandler
         .props[F](
@@ -70,7 +72,11 @@ class ClientController[F[+_, +_]: Exec: CovariantFlatMap: ErrorChannel](
           timeoutConfig.shutdownTimeout.plus(1.second)
         ),
       ProjectList -> ProjectListHandler
-        .props[F](clientId, projectService, timeoutConfig.requestTimeout),
+        .props[F](
+          projectService,
+          timeoutConfig.requestTimeout,
+          timeoutConfig.retries
+        ),
       ProjectRename -> ProjectRenameHandler
         .props[F](projectService, timeoutConfig.requestTimeout),
       EngineListInstalled -> EngineListInstalledHandler.props(

diff --git a/...-manager/src/main/scala/org/enso/projectmanager/requesthandler/ProjectCreateHandler.scala b/...-manager/src/main/scala/org/enso/projectmanager/requesthandler/ProjectCreateHandler.scala
@@ -22,11 +22,13 @@ import scala.concurrent.duration.FiniteDuration
   * @param configurationService the configuration service
   * @param projectService a project service
   * @param requestTimeout a request timeout
+  * @param timeoutRetries a number of timeouts to wait until a failure is reported
   */
 class ProjectCreateHandler[F[+_, +_]: Exec: CovariantFlatMap: ErrorChannel](
   configurationService: GlobalConfigServiceApi[F],
   projectService: ProjectServiceApi[F],
-  requestTimeout: FiniteDuration
+  requestTimeout: FiniteDuration,
+  timeoutRetries: Int
 ) extends RequestHandler[
       F,
       ProjectServiceFailure,
@@ -35,7 +37,8 @@ class ProjectCreateHandler[F[+_, +_]: Exec: CovariantFlatMap: ErrorChannel](
       ProjectCreate.Result
     ](
       ProjectCreate,
-      Some(requestTimeout)
+      Some(requestTimeout),
+      timeoutRetries
     ) {
 
   override def handleRequest
@@ -76,18 +79,21 @@ object ProjectCreateHandler {
     * @param configurationService
     * @param projectService a project service
     * @param requestTimeout a request timeout
+    * @param timeoutRetries a number of timeouts to wait until a failure is reported
     * @return a configuration object
     */
   def props[F[+_, +_]: Exec: CovariantFlatMap: ErrorChannel](
     configurationService: GlobalConfigServiceApi[F],
     projectService: ProjectServiceApi[F],
-    requestTimeout: FiniteDuration
+    requestTimeout: FiniteDuration,
+    timeoutRetries: Int
   ): Props =
     Props(
       new ProjectCreateHandler(
         configurationService,
         projectService,
-        requestTimeout
+        requestTimeout,
+        timeoutRetries
       )
     )
 

diff --git a/...ct-manager/src/main/scala/org/enso/projectmanager/requesthandler/ProjectListHandler.scala b/...ct-manager/src/main/scala/org/enso/projectmanager/requesthandler/ProjectListHandler.scala
@@ -1,88 +1,48 @@
 package org.enso.projectmanager.requesthandler
 
-import java.util.UUID
-
-import akka.actor.{Actor, ActorRef, Cancellable, Props, Status}
-import akka.pattern.pipe
-import com.typesafe.scalalogging.LazyLogging
-import org.enso.jsonrpc.Errors.ServiceError
-import org.enso.jsonrpc.{Id, Request, ResponseError, ResponseResult}
+import akka.actor.Props
+import org.enso.projectmanager.control.core.CovariantFlatMap
+import org.enso.projectmanager.control.core.syntax._
 import org.enso.projectmanager.control.effect.Exec
 import org.enso.projectmanager.data.ProjectMetadata
 import org.enso.projectmanager.protocol.ProjectManagementApi.ProjectList
-import org.enso.projectmanager.requesthandler.ProjectServiceFailureMapper.mapFailure
+import org.enso.projectmanager.requesthandler.ProjectServiceFailureMapper.failureMapper
 import org.enso.projectmanager.service.{
   ProjectServiceApi,
   ProjectServiceFailure
 }
-import org.enso.projectmanager.util.UnhandledLogging
 
-import scala.annotation.unused
 import scala.concurrent.duration.FiniteDuration
 
 /** A request handler for `project/list` commands.
   *
   * @param clientId the requester id
   * @param service a project service
   * @param requestTimeout a request timeout
+  * @param timeoutRetries a number of timeouts to wait until a failure is reported
   */
-class ProjectListHandler[F[+_, +_]: Exec](
-  @unused clientId: UUID,
-  service: ProjectServiceApi[F],
-  requestTimeout: FiniteDuration
-) extends Actor
-    with LazyLogging
-    with UnhandledLogging {
-
-  override def receive: Receive = requestStage
-
-  import context.dispatcher
-
-  private def requestStage: Receive = {
-    case Request(ProjectList, id, params: ProjectList.Params) =>
-      Exec[F]
-        .exec { service.listProjects(params.numberOfProjects) }
-        .pipeTo(self)
-      val cancellable =
-        context.system.scheduler
-          .scheduleOnce(requestTimeout, self, RequestTimeout)
-      context.become(responseStage(id, sender(), cancellable))
-  }
-
-  private def responseStage(
-    id: Id,
-    replyTo: ActorRef,
-    cancellable: Cancellable
-  ): Receive = {
-    case Status.Failure(ex) =>
-      logger.error("Failure during ProjectList operation.", ex)
-      replyTo ! ResponseError(Some(id), ServiceError)
-      cancellable.cancel()
-      context.stop(self)
-
-    case RequestTimeout =>
-      logger.error("Request {} with {} timed out.", ProjectList, id)
-      replyTo ! ResponseError(Some(id), ServiceError)
-      context.stop(self)
-
-    case Left(failure: ProjectServiceFailure) =>
-      logger.error("Request {} failed due to {}.", id, failure)
-      replyTo ! ResponseError(Some(id), mapFailure(failure))
-      cancellable.cancel()
-      context.stop(self)
-
-    case Right(list: List[_]) =>
-      val metadata = list.collect { case meta: ProjectMetadata =>
-        meta
-      }
-
-      replyTo ! ResponseResult(
-        ProjectList,
-        id,
-        ProjectList.Result(metadata)
-      )
-      cancellable.cancel()
-      context.stop(self)
+class ProjectListHandler[F[+_, +_]: Exec: CovariantFlatMap](
+  projectService: ProjectServiceApi[F],
+  requestTimeout: FiniteDuration,
+  timeoutRetries: Int
+) extends RequestHandler[
+      F,
+      ProjectServiceFailure,
+      ProjectList.type,
+      ProjectList.Params,
+      ProjectList.Result
+    ](
+      ProjectList,
+      Some(requestTimeout),
+      timeoutRetries
+    ) {
+
+  override def handleRequest = { params =>
+    for {
+      projects <- projectService.listProjects(params.numberOfProjects)
+    } yield ProjectList.Result(projects.collect { case meta: ProjectMetadata =>
+      meta
+    })
   }
 
 }
@@ -94,13 +54,16 @@ object ProjectListHandler {
     * @param clientId the requester id
     * @param service a project service
     * @param requestTimeout a request timeout
+    * @param timeoutRetries a number of timeouts to wait until a failure is reported
     * @return a configuration object
     */
-  def props[F[+_, +_]: Exec](
-    clientId: UUID,
+  def props[F[+_, +_]: Exec: CovariantFlatMap](
     service: ProjectServiceApi[F],
-    requestTimeout: FiniteDuration
+    requestTimeout: FiniteDuration,
+    timeoutRetries: Int
   ): Props =
-    Props(new ProjectListHandler(clientId, service, requestTimeout))
+    Props(
+      new ProjectListHandler(service, requestTimeout, timeoutRetries)
+    )
 
 }
diff --git a/...ct-manager/src/main/scala/org/enso/projectmanager/requesthandler/ProjectOpenHandler.scala b/...ct-manager/src/main/scala/org/enso/projectmanager/requesthandler/ProjectOpenHandler.scala
@@ -21,11 +21,13 @@ import scala.concurrent.duration.FiniteDuration
   * @param clientId the requester id
   * @param projectService a project service
   * @param requestTimeout a request timeout
+  * @param timeoutRetries a number of timeouts to wait until a failure is reported
   */
 class ProjectOpenHandler[F[+_, +_]: Exec: CovariantFlatMap](
   clientId: UUID,
   projectService: ProjectServiceApi[F],
-  requestTimeout: FiniteDuration
+  requestTimeout: FiniteDuration,
+  timeoutRetries: Int
 ) extends RequestHandler[
       F,
       ProjectServiceFailure,
@@ -37,7 +39,8 @@ class ProjectOpenHandler[F[+_, +_]: Exec: CovariantFlatMap](
       // TODO [RW] maybe we can get rid of this timeout since boot timeout is
       //  handled by the LanguageServerProcess; still the ? message of
       //  LanguageServerGateway will result in timeouts (#1315)
-      Some(requestTimeout)
+      Some(requestTimeout),
+      timeoutRetries
     ) {
 
   override def handleRequest = { params =>
@@ -69,18 +72,21 @@ object ProjectOpenHandler {
     * @param clientId the requester id
     * @param projectService a project service
     * @param requestTimeout a request timeout
+    * @param timeoutRetries a number of timeouts to wait until a failure is reported
     * @return a configuration object
     */
   def props[F[+_, +_]: Exec: CovariantFlatMap](
     clientId: UUID,
     projectService: ProjectServiceApi[F],
-    requestTimeout: FiniteDuration
+    requestTimeout: FiniteDuration,
+    timeoutRetries: Int
   ): Props =
     Props(
       new ProjectOpenHandler(
         clientId,
         projectService,
-        requestTimeout
+        requestTimeout,
+        timeoutRetries
       )
     )