Pre compute suggestion db during build time (#5698)

Close #5068

Cache suggestions during the `buildEngineDistribution` command, and read them from the disk when the library is loaded. Initial graph coloring takes ~20 seconds vs ~25 seconds on the develop branch.

[peek-develop-branch.webm](https://user-images.githubusercontent.com/357683/223504462-e7d48262-4f5e-4724-b2b0-2cb97fc05140.webm)
[peek-suggestions-branch.webm](https://user-images.githubusercontent.com/357683/223504464-0fe86c04-8c4b-443c-ba96-6c5e2fb1e396.webm)

build.sbt

@@ -643,6 +643,7 @@ lazy val `docs-generator` = (project in file("lib/scala/docs-generator"))
   .dependsOn(syntax.jvm)
   .dependsOn(cli)
   .dependsOn(`version-output`)
+  .dependsOn(`polyglot-api`)
   .configs(Benchmark)
   .settings(
     frgaalJavaCompilerSetting,

engine/language-server/src/main/scala/org/enso/languageserver/boot/resource/RepoInitialization.scala

@@ -89,7 +89,7 @@ class RepoInitialization(
       } yield ()
     initAction.onComplete {
       case Success(()) =>
-        eventStream.publish(InitializedEvent.FileVersionsRepoInitialized)
+        eventStream.publish(InitializedEvent.VersionsRepoInitialized)
       case Failure(ex) =>
         logger.error(
           "Failed to initialize SQL versions repo [{}]. {}",

engine/language-server/src/main/scala/org/enso/languageserver/event/InitializedEvent.scala

@@ -5,9 +5,9 @@ sealed trait InitializedEvent extends Event
 
 object InitializedEvent {
 
-  case object SuggestionsRepoInitialized  extends InitializedEvent
-  case object FileVersionsRepoInitialized extends InitializedEvent
-  case object TruffleContextInitialized   extends InitializedEvent
-  case object InitializationFinished      extends InitializedEvent
-  case object InitializationFailed        extends InitializedEvent
+  case object SuggestionsRepoInitialized extends InitializedEvent
+  case object VersionsRepoInitialized    extends InitializedEvent
+  case object TruffleContextInitialized  extends InitializedEvent
+  case object InitializationFinished     extends InitializedEvent
+  case object InitializationFailed       extends InitializedEvent
 }

engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala

@@ -5,6 +5,7 @@ import java.util.concurrent.Executors
 import akka.actor.{Actor, ActorRef, Props, Stash, Status}
 import akka.pattern.pipe
 import com.typesafe.scalalogging.LazyLogging
+import org.enso.docs.sections.DocSectionsBuilder
 import org.enso.languageserver.capability.CapabilityProtocol.{
   AcquireCapability,
   CapabilityAcquired,
@@ -111,6 +112,12 @@ final class SuggestionsHandler(
       .subscribe(self, classOf[Api.ExpressionUpdates])
     context.system.eventStream
       .subscribe(self, classOf[Api.SuggestionsDatabaseModuleUpdateNotification])
+    context.system.eventStream.subscribe(
+      self,
+      classOf[Api.SuggestionsDatabaseSuggestionsLoadedNotification]
+    )
+    context.system.eventStream
+      .subscribe(self, classOf[Api.LibraryLoaded])
     context.system.eventStream.subscribe(self, classOf[ProjectNameChangedEvent])
     context.system.eventStream.subscribe(self, classOf[FileDeletedEvent])
     context.system.eventStream
@@ -122,7 +129,7 @@ final class SuggestionsHandler(
   override def receive: Receive =
     initializing(SuggestionsHandler.Initialization())
 
-  def initializing(init: SuggestionsHandler.Initialization): Receive = {
+  private def initializing(init: SuggestionsHandler.Initialization): Receive = {
     case ProjectNameChangedEvent(oldName, newName) =>
       logger.info(
         "Initializing: project name changed from [{}] to [{}].",
@@ -178,35 +185,7 @@ final class SuggestionsHandler(
     case _ => stash()
   }
 
-  def verifying(
-    projectName: String,
-    graph: TypeGraph
-  ): Receive = {
-    case Api.Response(_, Api.VerifyModulesIndexResponse(toRemove)) =>
-      logger.info("Verifying: got verification response.")
-      val removeAction = for {
-        _ <- versionsRepo.remove(toRemove)
-        _ <- suggestionsRepo.removeModules(toRemove)
-      } yield SuggestionsHandler.Verified
-      removeAction.pipeTo(self)
-
-    case SuggestionsHandler.Verified =>
-      logger.info("Verified.")
-      context.become(initialized(projectName, graph, Set(), State()))
-      unstashAll()
-
-    case Status.Failure(ex) =>
-      logger.error(
-        "Database verification failure [{}]. {}",
-        ex.getClass,
-        ex.getMessage
-      )
-
-    case _ =>
-      stash()
-  }
-
-  def initialized(
+  private def initialized(
     projectName: String,
     graph: TypeGraph,
     clients: Set[ClientId],
@@ -230,12 +209,37 @@ final class SuggestionsHandler(
         initialized(projectName, graph, clients - client.clientId, state)
       )
 
+    case msg: Api.SuggestionsDatabaseSuggestionsLoadedNotification =>
+      logger.debug(
+        "Starting loading suggestions for library [{}].",
+        msg.libraryName
+      )
+      applyLoadedSuggestions(msg.suggestions)
+        .onComplete {
+          case Success(notification) =>
+            logger.debug(
+              "Complete loading suggestions for library [{}].",
+              msg.libraryName
+            )
+            if (notification.updates.nonEmpty) {
+              clients.foreach { clientId =>
+                sessionRouter ! DeliverToJsonController(clientId, notification)
+              }
+            }
+          case Failure(ex) =>
+            logger.error(
+              "Error applying suggestion updates for loaded library [{}] ({})",
+              msg.libraryName,
+              ex.getMessage
+            )
+        }
+
     case msg: Api.SuggestionsDatabaseModuleUpdateNotification
         if state.isSuggestionUpdatesRunning =>
       state.suggestionUpdatesQueue.enqueue(msg)
 
     case SuggestionUpdatesBatch(updates) if state.isSuggestionUpdatesRunning =>
-      state.suggestionUpdatesQueue.enqueueAll(updates)
+      state.suggestionUpdatesQueue.prependAll(updates)
 
     case SuggestionUpdatesBatch(updates) =>
       val modules = updates.map(_.module)
@@ -493,6 +497,13 @@ final class SuggestionsHandler(
       updates.foreach(sessionRouter ! _)
       context.become(initialized(name, graph, clients, state))
 
+    case libraryLoaded: Api.LibraryLoaded =>
+      logger.debug(
+        "Loaded Library [{}.{}]",
+        libraryLoaded.namespace,
+        libraryLoaded.name
+      )
+
     case SuggestionUpdatesCompleted =>
       if (state.suggestionUpdatesQueue.nonEmpty) {
         self ! SuggestionUpdatesBatch(state.suggestionUpdatesQueue.removeAll())
@@ -515,17 +526,9 @@ final class SuggestionsHandler(
   private def tryInitialize(state: SuggestionsHandler.Initialization): Unit = {
     logger.debug("Trying to initialize with state [{}]", state)
     state.initialized.fold(context.become(initializing(state))) {
-      case (name, graph) =>
+      case (projectName, graph) =>
         logger.debug("Initialized with state [{}].", state)
-        val requestId = UUID.randomUUID()
-        suggestionsRepo.getAllModules
-          .map { modules =>
-            runtimeConnector ! Api.Request(
-              requestId,
-              Api.VerifyModulesIndexRequest(modules)
-            )
-          }
-        context.become(verifying(name, graph))
+        context.become(initialized(projectName, graph, Set(), State()))
         unstashAll()
     }
   }
@@ -543,6 +546,48 @@ final class SuggestionsHandler(
     }
   }
 
+  /** Handle the suggestions of the loaded library.
+    *
+    * Adds the new suggestions to the suggestions database and sends the
+    * appropriate notification to the client.
+    *
+    * @param suggestions the loaded suggestions
+    * @return the API suggestions database update notification
+    */
+  private def applyLoadedSuggestions(
+    suggestions: Vector[Suggestion]
+  ): Future[SuggestionsDatabaseUpdateNotification] = {
+    for {
+      treeResults <- suggestionsRepo.applyTree(
+        suggestions.map(Api.SuggestionUpdate(_, Api.SuggestionAction.Add()))
+      )
+      version <- suggestionsRepo.currentVersion
+    } yield {
+      val treeUpdates = treeResults.flatMap {
+        case QueryResult(ids, Api.SuggestionUpdate(suggestion, action)) =>
+          action match {
+            case Api.SuggestionAction.Add() =>
+              if (ids.isEmpty) {
+                val verb = action.getClass.getSimpleName
+                logger.error("Cannot {} [{}].", verb, suggestion)
+              }
+              ids.map(
+                SuggestionsDatabaseUpdate.Add(
+                  _,
+                  generateDocumentation(suggestion)
+                )
+              )
+            case action =>
+              logger.error(
+                s"Invalid action during suggestions loading [$action]."
+              )
+              Seq()
+          }
+      }
+      SuggestionsDatabaseUpdateNotification(version, treeUpdates)
+    }
+  }
+
   /** Handle the suggestions database update.
     *
     * Function applies notification updates on the suggestions database and
@@ -783,9 +828,6 @@ object SuggestionsHandler {
       new ProjectNameUpdated(projectName, Seq())
   }
 
-  /** The notification that the suggestions database has been verified. */
-  case object Verified
-
   /** The notification that the suggestion updates are processed. */
   case object SuggestionUpdatesCompleted
 
@@ -843,7 +885,7 @@ object SuggestionsHandler {
     * @param config the server configuration
     * @param contentRootManager the content root manager
     * @param suggestionsRepo the suggestions repo
-    * @param fileVersionsRepo the file versions repo
+    * @param versionsRepo the versions repo
     * @param sessionRouter the session router
     * @param runtimeConnector the runtime connector
     * @param docSectionsBuilder the builder of documentation sections
@@ -852,7 +894,7 @@ object SuggestionsHandler {
     config: Config,
     contentRootManager: ContentRootManager,
     suggestionsRepo: SuggestionsRepo[Future],
-    fileVersionsRepo: VersionsRepo[Future],
+    versionsRepo: VersionsRepo[Future],
     sessionRouter: ActorRef,
     runtimeConnector: ActorRef,
     docSectionsBuilder: DocSectionsBuilder = DocSectionsBuilder()
@@ -862,7 +904,7 @@ object SuggestionsHandler {
         config,
         contentRootManager,
         suggestionsRepo,
-        fileVersionsRepo,
+        versionsRepo,
         sessionRouter,
         runtimeConnector,
         docSectionsBuilder

engine/language-server/src/main/scala/org/enso/languageserver/text/BufferRegistry.scala

@@ -102,13 +102,13 @@ class BufferRegistry(
   override def preStart(): Unit = {
     logger.info("Starting initialization.")
     context.system.eventStream
-      .subscribe(self, InitializedEvent.FileVersionsRepoInitialized.getClass)
+      .subscribe(self, InitializedEvent.VersionsRepoInitialized.getClass)
   }
 
   override def receive: Receive = initializing
 
   private def initializing: Receive = {
-    case InitializedEvent.FileVersionsRepoInitialized =>
+    case InitializedEvent.VersionsRepoInitialized =>
       logger.info("Initiaized.")
       context.become(running(Map.empty))
       unstashAll()

engine/language-server/src/test/scala/org/enso/languageserver/boot/resource/RepoInitializationSpec.scala

@@ -66,7 +66,7 @@ class RepoInitializationSpec
 
         expectMsgAllOf(
           InitializedEvent.SuggestionsRepoInitialized,
-          InitializedEvent.FileVersionsRepoInitialized
+          InitializedEvent.VersionsRepoInitialized
         )
     }
 
@@ -96,7 +96,7 @@ class RepoInitializationSpec
 
         expectMsgAllOf(
           InitializedEvent.SuggestionsRepoInitialized,
-          InitializedEvent.FileVersionsRepoInitialized
+          InitializedEvent.VersionsRepoInitialized
         )
     }
 
@@ -123,7 +123,7 @@ class RepoInitializationSpec
         version1 shouldEqual SchemaVersion.CurrentVersion
         expectMsgAllOf(
           InitializedEvent.SuggestionsRepoInitialized,
-          InitializedEvent.FileVersionsRepoInitialized
+          InitializedEvent.VersionsRepoInitialized
         )
 
         // remove schema and re-initialize
@@ -138,7 +138,7 @@ class RepoInitializationSpec
         version2 shouldEqual SchemaVersion.CurrentVersion
         expectMsgAllOf(
           InitializedEvent.SuggestionsRepoInitialized,
-          InitializedEvent.FileVersionsRepoInitialized
+          InitializedEvent.VersionsRepoInitialized
         )
     }
 
@@ -199,7 +199,7 @@ class RepoInitializationSpec
           version2 shouldEqual SchemaVersion.CurrentVersion
           expectMsgAllOf(
             InitializedEvent.SuggestionsRepoInitialized,
-            InitializedEvent.FileVersionsRepoInitialized
+            InitializedEvent.VersionsRepoInitialized
           )
         }
     }

engine/language-server/src/test/scala/org/enso/languageserver/search/Suggestions.scala

@@ -1,6 +1,7 @@
 package org.enso.languageserver.search
 
 import org.enso.docs.generator.DocsGenerator
+import org.enso.docs.sections.DocSectionsBuilder
 import org.enso.polyglot.Suggestion
 
 import java.util.UUID