Pre compute suggestion db during build time

engine/runtime/src/main/java/org/enso/compiler/Cache.java

@@ -6,6 +6,7 @@
 import org.bouncycastle.util.encoders.Hex;
 import org.enso.interpreter.runtime.EnsoContext;
 import org.enso.logger.masking.MaskedPath;
+import org.enso.pkg.SourceFile;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -19,6 +20,8 @@
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.security.MessageDigest;
+import java.util.Comparator;
+import java.util.List;
 import java.util.Optional;
 import java.util.logging.Level;
 
@@ -330,6 +333,30 @@ protected String computeDigestFromBytes(byte[] bytes) {
     return Hex.toHexString(messageDigest().digest(bytes));
   }
 
+  /**
+   * Computes digest from package sources using a default hashing algorithm.
+   *
+   * @param pkgSources the list of package sources
+   * @param logger the truffle logger
+   * @return string representation of bytes' hash
+   */
+  protected String computeDigestOfLibrarySources(
+      List<SourceFile<TruffleFile>> pkgSources, TruffleLogger logger) {
+    pkgSources.sort(Comparator.comparing(o -> o.qualifiedName().toString()));
+
+    var digest = messageDigest();
+    pkgSources.forEach(
+        source -> {
+          try {
+            digest.update(source.file().readAllBytes());
+          } catch (IOException e) {
+            logger.log(
+                logLevel, "failed to compute digest for " + source.qualifiedName().toString(), e);
+          }
+        });
+    return Hex.toHexString(digest.digest());
+  }
+
   /**
    * Returns a default hashing algorithm used for Enso caches.
    *

engine/runtime/src/main/java/org/enso/compiler/ImportExportCache.java

@@ -6,7 +6,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.oracle.truffle.api.TruffleFile;
 import com.oracle.truffle.api.TruffleLogger;
-import org.bouncycastle.util.encoders.Hex;
 import org.enso.compiler.data.BindingsMap;
 import org.enso.editions.LibraryName;
 import org.enso.interpreter.runtime.EnsoContext;
@@ -15,15 +14,13 @@
 import scala.collection.immutable.Map;
 import scala.jdk.CollectionConverters;
 
-import java.io.IOException;
 import java.io.Serializable;
 import java.util.Arrays;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
 import java.util.logging.Level;
 
-public class ImportExportCache extends Cache<ImportExportCache.CachedBindings, ImportExportCache.Metadata> {
+public final class ImportExportCache extends Cache<ImportExportCache.CachedBindings, ImportExportCache.Metadata> {
 
     private final LibraryName libraryName;
 
@@ -79,22 +76,6 @@ protected Optional<String> computeDigestFromSource(EnsoContext context, TruffleL
                 .map(pkg -> computeDigestOfLibrarySources(pkg.listSourcesJava(), logger));
     }
 
-    private String computeDigestOfLibrarySources(List<SourceFile<TruffleFile>> pkgSources, TruffleLogger logger) {
-        pkgSources.sort(Comparator.comparing(o -> o.qualifiedName().toString()));
-
-        var digest = messageDigest();
-        pkgSources.forEach(source ->
-            {
-                try {
-                    digest.update(source.file().readAllBytes());
-                } catch (IOException e) {
-                    logger.log(logLevel, "failed to compute digest for " + source.qualifiedName().toString(), e);
-                }
-            }
-        );
-        return Hex.toHexString(digest.digest());
-    }
-
     @Override
     @SuppressWarnings("unchecked")
     protected Optional<Cache.Roots> getCacheRoots(EnsoContext context) {

engine/runtime/src/main/java/org/enso/compiler/ModuleCache.java

@@ -18,7 +18,7 @@
 import java.util.Optional;
 import java.util.logging.Level;
 
-public class ModuleCache extends Cache<ModuleCache.CachedModule, ModuleCache.Metadata> {
+public final class ModuleCache extends Cache<ModuleCache.CachedModule, ModuleCache.Metadata> {
 
     private final Module module;
 

engine/runtime/src/main/java/org/enso/compiler/SuggestionsCache.java

@@ -0,0 +1,151 @@
+package org.enso.compiler;
+
+import buildinfo.Info;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.oracle.truffle.api.TruffleLogger;
+import org.bouncycastle.util.encoders.Hex;
+import org.enso.editions.LibraryName;
+import org.enso.interpreter.runtime.EnsoContext;
+import org.enso.polyglot.Suggestion;
+import scala.jdk.CollectionConverters;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Level;
+
+public final class SuggestionsCache
+    extends Cache<SuggestionsCache.CachedSuggestions, SuggestionsCache.Metadata> {
+
+  private static final String SUGGESTIONS_CACHE_DATA_EXTENSION = ".suggestions";
+  private static final String SUGGESTIONS_CACHE_METADATA_EXTENSION =".suggestions.meta";
+
+  final LibraryName libraryName;
+
+  public SuggestionsCache(LibraryName libraryName) {
+    this.libraryName = libraryName;
+    this.logLevel = Level.FINEST;
+    this.stringRepr = libraryName.toString();
+    this.entryName = libraryName.name();
+    this.dataSuffix = SUGGESTIONS_CACHE_DATA_EXTENSION;
+    this.metadataSuffix = SUGGESTIONS_CACHE_METADATA_EXTENSION;
+  }
+
+  @Override
+  protected byte[] metadata(String sourceDigest, String blobDigest, CachedSuggestions entry) {
+    var mapper = new ObjectMapper();
+    try {
+      return mapper.writeValueAsString(new Metadata(sourceDigest, blobDigest)).getBytes(metadataCharset);
+    } catch (JsonProcessingException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  protected CachedSuggestions validateReadObject(Object obj, Metadata meta, TruffleLogger logger)
+      throws CacheException {
+    if (obj instanceof Suggestions suggestions) {
+      return new CachedSuggestions(libraryName, suggestions);
+    } else {
+      throw new CacheException("Expected SuggestionsCache.Suggestions, got " + obj.getClass());
+    }
+  }
+
+  @Override
+  protected Optional<Metadata> metadataFromBytes(byte[] bytes) {
+    var maybeJsonString = new String(bytes, Cache.metadataCharset);
+    var mapper = new ObjectMapper();
+    try {
+      return Optional.of(mapper.readValue(maybeJsonString, SuggestionsCache.Metadata.class));
+    } catch (JsonProcessingException e) {
+      return Optional.empty();
+    }
+  }
+
+  @Override
+  protected Optional<String> computeDigest(CachedSuggestions entry, TruffleLogger logger) {
+    var digest = messageDigest();
+    entry.getSuggestions().getSuggestions().forEach(suggestion -> {
+      ByteBuffer bytes = ByteBuffer.allocate(Integer.BYTES).putInt(suggestion.hashCode());
+      digest.update(bytes);
+    });
+    return Optional.of(Hex.toHexString(digest.digest()));
+  }
+
+  @Override
+  protected Optional<String> computeDigestFromSource(EnsoContext context, TruffleLogger logger) {
+    return context
+        .getPackageRepository()
+        .getPackageForLibraryJava(libraryName)
+        .map(pkg -> computeDigestOfLibrarySources(pkg.listSourcesJava(), logger));
+  }
+
+
+  @Override
+  protected Optional<Roots> getCacheRoots(EnsoContext context) {
+    return context.getPackageRepository().getPackageForLibraryJava(libraryName).map(pkg -> {
+      var bindingsCacheRoot = pkg.getSuggestionsCacheRootForPackage(Info.ensoVersion());
+      var localCacheRoot = bindingsCacheRoot.resolve(libraryName.namespace());
+      var distribution = context.getDistributionManager();
+      var pathSegments = CollectionConverters.ListHasAsScala(Arrays.asList(
+              pkg.namespace(),
+              pkg.name(),
+              pkg.config().version(),
+              Info.ensoVersion(),
+              libraryName.namespace())
+      ).asScala();
+      var path = distribution.LocallyInstalledDirectories().irCacheDirectory()
+              .resolve(pathSegments.mkString("/"));
+      var globalCacheRoot = context.getTruffleFile(path.toFile());
+      return new Cache.Roots(localCacheRoot, globalCacheRoot);
+    });
+  }
+
+  @Override
+  protected Object extractObjectToSerialize(CachedSuggestions entry) {
+    return entry.getSuggestions();
+  }
+
+  // Suggestions class is not a record because of a Frgaal bug leading to invalid compilation error.
+  final static class Suggestions implements Serializable {
+
+    private final List<Suggestion> suggestions;
+
+    public Suggestions(List<Suggestion> suggestions)  {
+      this.suggestions = suggestions;
+    }
+
+    public List<Suggestion> getSuggestions() {
+      return suggestions;
+    }
+  }
+
+  // CachedSuggestions class is not a record because of a Frgaal bug leading to invalid compilation error.
+  final static class CachedSuggestions {
+
+    private final LibraryName libraryName;
+    private final Suggestions suggestions;
+
+    public CachedSuggestions(LibraryName libraryName, Suggestions suggestions) {
+      this.libraryName = libraryName;
+      this.suggestions = suggestions;
+    }
+
+    public LibraryName getLibraryName() {
+      return libraryName;
+    }
+
+    public Suggestions getSuggestions() {
+      return suggestions;
+    }
+  }
+
+  record Metadata(
+      @JsonProperty("source_hash") String sourceHash,
+      @JsonProperty("blob_hash") String blobHash
+  ) implements Cache.Metadata { }
+}

engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala

@@ -3,12 +3,7 @@ package org.enso.compiler
 import com.oracle.truffle.api.TruffleLogger
 import com.oracle.truffle.api.source.Source
 import org.enso.compiler.codegen.{AstToIr, IrToTruffle, RuntimeStubsGenerator}
-import org.enso.compiler.context.{
-  FreshNameSupply,
-  InlineContext,
-  ModuleContext,
-  SuggestionBuilder
-}
+import org.enso.compiler.context.{FreshNameSupply, InlineContext, ModuleContext}
 import org.enso.compiler.core.IR
 import org.enso.compiler.core.IR.Expression
 
@@ -68,10 +63,6 @@ class Compiler(
   )
   private val serializationManager: SerializationManager =
     new SerializationManager(this)
-  private val suggestionsSerializationManager: SuggestionsSerializationManager =
-    new SuggestionsSerializationManager(
-      context.getLogger(classOf[SuggestionsSerializationManager])
-    )
   private val logger: TruffleLogger = context.getLogger(getClass)
   private val output: PrintStream =
     if (config.outputRedirect.isDefined)
@@ -200,25 +191,16 @@ class Compiler(
                 mod
             }.toList
 
-            val compilerResult =
-              runInternal(
-                packageModules,
-                generateCode = false,
-                shouldCompileDependencies
-              )
+            runInternal(
+              packageModules,
+              generateCode = false,
+              shouldCompileDependencies
+            )
 
-            serializationManager.serializeLibraryBindings(
+            serializationManager.serializeLibrary(
               pkg.libraryName,
               useGlobalCacheLocations = true
             )
-            val suggestions =
-              compilerResult.compiledModules
-                .flatMap { module =>
-                  SuggestionBuilder(module)
-                    .build(module.getName, module.getIr)
-                    .toVector
-                }
-            suggestionsSerializationManager.serialize(suggestions, pkg)
         }
     }
   }