Store IR caches in suitable format to read them "on demand"

build.sbt

@@ -1428,14 +1428,22 @@ lazy val `runtime-parser` =
     .settings(
       frgaalJavaCompilerSetting,
       instrumentationSettings,
+      commands += WithDebugCommand.withDebug,
+      fork := true,
+      Test / javaOptions ++= Seq(
+        "-Dgraalvm.locatorDisabled=true",
+        s"--upgrade-module-path=${file("engine/runtime/build-cache/truffle-api.jar").absolutePath}"
+      ),
       libraryDependencies ++= Seq(
-        "junit"          % "junit"           % junitVersion     % Test,
-        "com.github.sbt" % "junit-interface" % junitIfVersion   % Test,
-        "org.scalatest" %% "scalatest"       % scalatestVersion % Test
+        "junit"            % "junit"                   % junitVersion       % Test,
+        "com.github.sbt"   % "junit-interface"         % junitIfVersion     % Test,
+        "org.scalatest"   %% "scalatest"               % scalatestVersion   % Test,
+        "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided"
       )
     )
     .dependsOn(syntax)
     .dependsOn(`syntax-rust-definition`)
+    .dependsOn(`interpreter-dsl` % "provided")
 
 lazy val `runtime-compiler` =
   (project in file("engine/runtime-compiler"))
@@ -1453,6 +1461,7 @@ lazy val `runtime-compiler` =
     .dependsOn(pkg)
     .dependsOn(`polyglot-api`)
     .dependsOn(editions)
+    .dependsOn(`interpreter-dsl` % "provided")
 
 lazy val `runtime-instrument-common` =
   (project in file("engine/runtime-instrument-common"))

distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Expand_Objects_Helpers.enso

@@ -13,6 +13,7 @@ import project.Extensions.Prefix_Name.Prefix_Name
 import project.Internal.Fan_Out
 import project.Internal.Java_Exports
 import project.Internal.Java_Problems
+import project.Internal.Widget_Helpers
 from project.Internal.Java_Exports import make_inferred_builder
 
 ## PRIVATE

engine/runner/src/main/scala/org/enso/runner/Main.scala

@@ -572,7 +572,8 @@ object Main {
       topScope.compile(shouldCompileDependencies)
       exitSuccess()
     } catch {
-      case _: Throwable =>
+      case t: Throwable =>
+        logger.error("Unexpected internal error", t)
         exitFail()
     } finally {
       context.context.close()

engine/runtime-compiler/src/main/java/org/enso/compiler/context/CompilerContext.java

@@ -81,8 +81,6 @@ void initializeBuiltinsIr(
 
   boolean wasLoadedFromCache(Module module);
 
-  boolean hasCrossModuleLinks(Module module);
-
   org.enso.compiler.core.ir.Module getIr(Module module);
 
   CompilationStage getCompilationStage(Module module);
@@ -108,8 +106,6 @@ public static interface Updater {
 
     void loadedFromCache(boolean b);
 
-    void hasCrossModuleLinks(boolean b);
-
     void resetScope();
 
     void invalidateCache();
@@ -134,8 +130,6 @@ public abstract static class Module {
 
     public abstract boolean isSynthetic();
 
-    public abstract boolean hasCrossModuleLinks();
-
     public abstract org.enso.compiler.core.ir.Module getIr();
 
     public abstract boolean isPrivate();

engine/runtime-compiler/src/main/java/org/enso/compiler/pass/analyse/PassPersistance.java

@@ -0,0 +1,155 @@
+package org.enso.compiler.pass.analyse;
+
+import java.io.IOException;
+import org.enso.compiler.core.Persistance;
+import org.enso.compiler.pass.analyse.AliasAnalysis.Graph;
+import org.enso.compiler.pass.resolve.DocumentationComments;
+import org.enso.compiler.pass.resolve.FullyQualifiedNames;
+import org.enso.compiler.pass.resolve.IgnoredBindings;
+import org.enso.compiler.pass.resolve.ModuleAnnotations;
+import org.enso.compiler.pass.resolve.TypeSignatures;
+import org.enso.interpreter.dsl.Persistable;
+import org.openide.util.lookup.ServiceProvider;
+import scala.Option;
+
+@Persistable(clazz = CachePreferenceAnalysis.WeightInfo.class, id = 1111)
+@Persistable(clazz = DataflowAnalysis.DependencyInfo.class, id = 1112)
+@Persistable(clazz = DataflowAnalysis.DependencyMapping.class, id = 1113)
+@Persistable(clazz = GatherDiagnostics.DiagnosticsMeta.class, id = 1114)
+@Persistable(clazz = DocumentationComments.Doc.class, id = 1115)
+@Persistable(clazz = AliasAnalysis$Info$Occurrence.class, id = 1116)
+@Persistable(clazz = TypeSignatures.Signature.class, id = 1117)
+@Persistable(clazz = ModuleAnnotations.Annotations.class, id = 1118)
+@Persistable(clazz = AliasAnalysis$Info$Scope$Root.class, id = 1120)
+@Persistable(clazz = DataflowAnalysis$DependencyInfo$Type$Static.class, id = 1121)
+@Persistable(clazz = DataflowAnalysis$DependencyInfo$Type$Dynamic.class, id = 1122)
+@Persistable(clazz = AliasAnalysis$Info$Scope$Child.class, id = 1123)
+@Persistable(clazz = AliasAnalysis$Graph$Occurrence$Use.class, id = 1125)
+@Persistable(clazz = AliasAnalysis$Graph$Occurrence$Def.class, id = 1126)
+@Persistable(clazz = AliasAnalysis$Graph$Link.class, id = 1127)
+@Persistable(clazz = FullyQualifiedNames.FQNResolution.class, id = 1128)
+@Persistable(clazz = FullyQualifiedNames.ResolvedLibrary.class, id = 1129)
+@Persistable(clazz = FullyQualifiedNames.ResolvedModule.class, id = 1130)
+public final class PassPersistance {
+  private PassPersistance() {}
+
+  @ServiceProvider(service = Persistance.class)
+  public static final class PersistState extends Persistance<IgnoredBindings.State> {
+    public PersistState() {
+      super(IgnoredBindings.State.class, true, 1101);
+    }
+
+    @Override
+    protected void writeObject(IgnoredBindings.State obj, Output out) throws IOException {
+      out.writeBoolean(obj.isIgnored());
+    }
+
+    @Override
+    protected IgnoredBindings.State readObject(Input in)
+        throws IOException, ClassNotFoundException {
+      var b = in.readBoolean();
+      return b
+          ? org.enso.compiler.pass.resolve.IgnoredBindings$State$Ignored$.MODULE$
+          : org.enso.compiler.pass.resolve.IgnoredBindings$State$NotIgnored$.MODULE$;
+    }
+  }
+
+  @ServiceProvider(service = Persistance.class)
+  public static final class PersistTail extends Persistance<TailCall.TailPosition> {
+    public PersistTail() {
+      super(TailCall.TailPosition.class, true, 1102);
+    }
+
+    @Override
+    protected void writeObject(TailCall.TailPosition obj, Output out) throws IOException {
+      out.writeBoolean(obj.isTail());
+    }
+
+    @Override
+    protected TailCall.TailPosition readObject(Input in)
+        throws IOException, ClassNotFoundException {
+      var b = in.readBoolean();
+      return b
+          ? org.enso.compiler.pass.analyse.TailCall$TailPosition$Tail$.MODULE$
+          : org.enso.compiler.pass.analyse.TailCall$TailPosition$NotTail$.MODULE$;
+    }
+  }
+
+  @org.openide.util.lookup.ServiceProvider(service = Persistance.class)
+  public static final class PersistAliasAnalysisGraphScope
+      extends Persistance<org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope> {
+    public PersistAliasAnalysisGraphScope() {
+      super(org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope.class, false, 1124);
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope readObject(Input in)
+        throws IOException {
+      var childScopes = in.readInline(scala.collection.immutable.List.class);
+      var occurrences = (scala.collection.immutable.Set) in.readObject();
+      var allDefinitions = in.readInline(scala.collection.immutable.List.class);
+      var parent =
+          new org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope(
+              childScopes, occurrences, allDefinitions);
+      var optionParent = Option.apply(parent);
+      childScopes.forall(
+          (object) -> {
+            var ch = (org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope) object;
+            ch.parent_$eq(optionParent);
+            return null;
+          });
+      return parent;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected void writeObject(
+        org.enso.compiler.pass.analyse.AliasAnalysis$Graph$Scope obj, Output out)
+        throws IOException {
+      out.writeInline(scala.collection.immutable.List.class, obj.childScopes());
+      out.writeObject(obj.occurrences());
+      out.writeInline(scala.collection.immutable.List.class, obj.allDefinitions());
+    }
+  }
+
+  @org.openide.util.lookup.ServiceProvider(service = Persistance.class)
+  public static final class PersistAliasAnalysisGraph extends Persistance<Graph> {
+    public PersistAliasAnalysisGraph() {
+      super(Graph.class, false, 1119);
+    }
+
+    @SuppressWarnings("unchecked")
+    protected Graph readObject(Input in) throws IOException {
+      var g = new Graph();
+
+      var rootScope = (AliasAnalysis$Graph$Scope) in.readObject();
+      assignParents(rootScope);
+      g.rootScope_$eq(rootScope);
+
+      var links =
+          (scala.collection.immutable.Set) in.readInline(scala.collection.immutable.Set.class);
+      g.links_$eq(links);
+      return g;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    protected void writeObject(Graph obj, Output out) throws IOException {
+      out.writeObject(obj.rootScope());
+      out.writeInline(scala.collection.immutable.Set.class, obj.links());
+    }
+
+    private static void assignParents(AliasAnalysis$Graph$Scope scope) {
+      var option = Option.apply(scope);
+      scope
+          .childScopes()
+          .foreach(
+              (ch) -> {
+                assignParents(ch);
+                ch.parent_$eq(option);
+                return null;
+              });
+    }
+  }
+}

engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala

@@ -257,7 +257,7 @@ class Compiler(
       }
     )
 
-    var requiredModules = modules.flatMap { module =>
+    val requiredModules = modules.flatMap { module =>
       val importedModules = runImportsAndExportsResolution(module, generateCode)
       val isLoadedFromSource =
         (m: Module) => !context.wasLoadedFromCache(m) && !context.isSynthetic(m)
@@ -286,47 +286,11 @@ class Compiler(
       }
     }.distinct
 
-    var hasInvalidModuleRelink = false
     if (irCachingEnabled) {
       requiredModules.foreach { module =>
         ensureParsed(module)
-        if (!context.hasCrossModuleLinks(module)) {
-          val flags =
-            context
-              .getIr(module)
-              .preorder
-              .map(_.passData.restoreFromSerialization(this.context))
-
-          if (!flags.contains(false)) {
-            context.log(
-              Compiler.defaultLogLevel,
-              "Restored links (late phase) for module [{0}].",
-              context.getModuleName(module)
-            )
-          } else {
-            hasInvalidModuleRelink = true
-            context.log(
-              Compiler.defaultLogLevel,
-              "Failed to restore links (late phase) for module [{0}].",
-              context.getModuleName(module)
-            )
-            uncachedParseModule(module, isGenDocs = false)
-          }
-        }
       }
     }
-
-    if (hasInvalidModuleRelink) {
-      context.log(
-        Compiler.defaultLogLevel,
-        s"Some modules failed to relink. Re-running import and " +
-        s"export resolution."
-      )
-
-      requiredModules =
-        modules.flatMap(runImportsAndExportsResolution(_, generateCode))
-    }
-
     requiredModules.foreach { module =>
       if (
         !context
@@ -629,7 +593,6 @@ class Compiler(
         u.ir(discoveredModule)
         u.compilationStage(CompilationStage.AFTER_PARSING)
         u.loadedFromCache(false)
-        u.hasCrossModuleLinks(true)
       }
     )
   }
@@ -900,7 +863,9 @@ class Compiler(
   ): Unit = {
     if (config.isStrictErrors) {
       val diagnostics = modules.flatMap { module =>
-        val errors = gatherDiagnostics(module)
+        val errors =
+          if (context.wasLoadedFromCache(module)) List()
+          else gatherDiagnostics(module)
         List((module, errors))
       }
       if (reportDiagnostics(diagnostics)) {

engine/runtime-compiler/src/main/scala/org/enso/compiler/context/LocalScope.scala

@@ -84,7 +84,10 @@ class LocalScope(
     * @return the frame slot index for `id`.
     */
   def getVarSlotIdx(id: Graph.Id): Int = {
-    assert(localFrameSlotIdxs.contains(id))
+    assert(
+      localFrameSlotIdxs.contains(id),
+      "Cannot find " + id + " in " + localFrameSlotIdxs
+    )
     localFrameSlotIdxs(id)
   }
 

engine/runtime-compiler/src/main/scala/org/enso/compiler/context/SuggestionBuilder.scala

@@ -11,6 +11,7 @@ import org.enso.compiler.core.ir.{
   Function,
   IdentifiedLocation,
   Literal,
+  Location,
   Name,
   Type
 }
@@ -28,7 +29,6 @@ import org.enso.compiler.pass.resolve.{
 import org.enso.pkg.QualifiedName
 import org.enso.polyglot.Suggestion
 import org.enso.polyglot.data.{Tree, TypeGraph}
-import org.enso.syntax.text.Location
 import org.enso.text.editing.IndexedSource
 
 import java.util.UUID

engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/analyse/AliasAnalysis.scala

@@ -656,7 +656,7 @@ case object AliasAnalysis extends IRPass {
             )
           )
           .updateMetadata(this -->> Info.Scope.Child(graph, currentScope))
-      case _: Function.Binding =>
+      case _ =>
         throw new CompilerError(
           "Function sugar should not be present during alias analysis."
         )

engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/analyse/DataflowAnalysis.scala

@@ -308,7 +308,7 @@ case object DataflowAnalysis extends IRPass {
             body      = analyseExpression(body, info)
           )
           .updateMetadata(this -->> info)
-      case _: Function.Binding =>
+      case _ =>
         throw new CompilerError(
           "Function sugar should not be present during dataflow analysis."
         )

engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/analyse/DemandAnalysis.scala

@@ -8,6 +8,7 @@ import org.enso.compiler.core.ir.{
   Empty,
   Expression,
   Function,
+  IdentifiedLocation,
   Literal,
   Module,
   Name,
@@ -159,7 +160,7 @@ case object DemandAnalysis extends IRPass {
             isInsideCallArgument = false
           )
         )
-      case _: Function.Binding =>
+      case _ =>
         throw new CompilerError(
           "Function sugar should not be present during demand analysis."
         )
@@ -185,9 +186,10 @@ case object DemandAnalysis extends IRPass {
     } else {
       name match {
         case lit: Name.Literal if isDefined(lit) =>
-          val forceLocation   = name.location
-          val newNameLocation = name.location.map(l => l.copy(id = None))
-          val newName         = lit.copy(location = newNameLocation)
+          val forceLocation = name.location
+          val newNameLocation =
+            name.location.map(l => new IdentifiedLocation(l.location()))
+          val newName = lit.copy(location = newNameLocation)
           Application.Force(newName, forceLocation)
         case _ => name
       }

engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/analyse/TailCall.scala

@@ -450,7 +450,7 @@ case object TailCall extends IRPass {
           arguments = args.map(analyseDefArgument),
           body      = analyseExpression(body, isInTailPosition = markAsTail)
         )
-      case _: Function.Binding =>
+      case _ =>
         throw new CompilerError(
           "Function sugar should not be present during tail call analysis."
         )