Merge branch 'develop' into compiler/wip/mk/dataflow-warnings

engine/runner/src/main/scala/org/enso/runner/Main.scala

@@ -841,7 +841,7 @@ object Main {
 
   /** Default log level to use if the LOG_LEVEL option is not provided.
     */
-  val defaultLogLevel: LogLevel = LogLevel.Warning
+  val defaultLogLevel: LogLevel = LogLevel.Error
 
   /** Main entry point for the CLI program.
     *

engine/runtime/src/main/java/org/enso/interpreter/OptionsHelper.java

@@ -2,11 +2,7 @@
 
 import com.oracle.truffle.api.TruffleFile;
 import com.oracle.truffle.api.TruffleLanguage;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
 import java.util.Optional;
-import java.util.stream.Collectors;
 import org.enso.polyglot.RuntimeOptions;
 
 public class OptionsHelper {

engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java

@@ -5,13 +5,6 @@
 import com.oracle.truffle.api.TruffleLanguage;
 import com.oracle.truffle.api.TruffleLanguage.Env;
 import com.oracle.truffle.api.TruffleLogger;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintStream;
-import java.util.Optional;
-import java.util.UUID;
 import org.enso.compiler.Compiler;
 import org.enso.compiler.PackageRepository;
 import org.enso.compiler.data.CompilerConfig;
@@ -34,6 +27,10 @@
 import org.enso.polyglot.RuntimeOptions;
 import scala.jdk.javaapi.OptionConverters;
 
+import java.io.*;
+import java.util.Optional;
+import java.util.UUID;
+
 /**
  * The language context is the internal state of the language that is associated with each thread in
  * a running Enso program.
@@ -111,17 +108,17 @@ public void initialize() {
 
     Optional<TruffleFile> projectRoot = OptionsHelper.getProjectRoot(environment);
     Optional<Package<TruffleFile>> projectPackage =
-        projectRoot.flatMap(
-            file -> {
-              var result = packageManager.fromDirectory(file);
-              if (result.isEmpty()) {
-                var projectName = file.getName();
-                throw new ProjectLoadingFailure(projectName);
-              }
-              return ScalaConversions.asJava(result);
-            });
-
-    var languageHome =
+        projectRoot.map(
+            file ->
+                packageManager
+                    .loadPackage(file)
+                    .fold(
+                        err -> {
+                          throw new ProjectLoadingFailure(file.getName(), err);
+                        },
+                        res -> res));
+
+    Optional<String> languageHome =
         OptionsHelper.getLanguageHomeOverride(environment).or(() -> Optional.ofNullable(home));
     var resourceManager = new org.enso.distribution.locking.ResourceManager(lockManager);
 

engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala

@@ -1,5 +1,8 @@
 package org.enso.compiler
 
+import java.io.StringReader
+import java.util.logging.Level
+
 import com.oracle.truffle.api.TruffleLogger
 import com.oracle.truffle.api.source.Source
 import org.enso.compiler.codegen.{AstToIr, IrToTruffle, RuntimeStubsGenerator}
@@ -24,8 +27,6 @@ import org.enso.polyglot.{LanguageInfo, RuntimeOptions}
 import org.enso.syntax.text.Parser.IDMap
 import org.enso.syntax.text.{AST, Parser}
 
-import java.io.StringReader
-import java.util.logging.Level
 import scala.jdk.OptionConverters._
 
 /** This class encapsulates the static transformation processes that take place
@@ -54,8 +55,13 @@ class Compiler(
     new SerializationManager(this)
   private val logger: TruffleLogger = context.getLogger(getClass)
 
-  /** Lazy-initializes the IR for the builtins module.
-    */
+  /** Run the initialization sequence. */
+  def initialize(): Unit = {
+    initializeBuiltinsIr()
+    packageRepository.initialize().left.foreach(reportPackageError)
+  }
+
+  /** Lazy-initializes the IR for the builtins module. */
   def initializeBuiltinsIr(): Unit = {
     if (!builtins.isIrInitialized) {
       logger.log(
@@ -97,7 +103,7 @@ class Compiler(
     * @return the list of modules imported by `module`
     */
   def runImportsResolution(module: Module): List[Module] = {
-    initializeBuiltinsIr()
+    initialize()
     importResolver.mapImports(module)
   }
 
@@ -169,7 +175,7 @@ class Compiler(
     * @param module the scope from which docs are generated
     */
   def generateDocs(module: Module): Module = {
-    initializeBuiltinsIr()
+    initialize()
     parseModule(module, isGenDocs = true)
     module
   }
@@ -179,7 +185,7 @@ class Compiler(
     generateCode: Boolean,
     shouldCompileDependencies: Boolean
   ): Unit = {
-    initializeBuiltinsIr()
+    initialize()
     modules.foreach(m => parseModule(m))
 
     var requiredModules = modules.flatMap(runImportsAndExportsResolution)
@@ -667,12 +673,24 @@ class Compiler(
     }
   }
 
+  /** Report the errors encountered when initializing the package repository.
+    *
+    * @param err the package repository error
+    */
+  private def reportPackageError(err: PackageRepository.Error): Unit = {
+    context.getOut.println(
+      s"In package description ${org.enso.pkg.Package.configFileName}:"
+    )
+    context.getOut.println("Compiler encountered warnings:")
+    context.getOut.println(err.toString)
+  }
+
   /** Reports diagnostics from multiple modules.
     *
     * @param diagnostics the mapping between modules and existing diagnostics.
     * @return whether any errors were encountered.
     */
-  def reportDiagnostics(
+  private def reportDiagnostics(
     diagnostics: List[(Module, List[IR.Diagnostic])]
   ): Boolean = {
     val results = diagnostics.map { case (mod, diags) =>
@@ -693,7 +711,7 @@ class Compiler(
     * @param source the original source code.
     * @return whether any errors were encountered.
     */
-  def reportDiagnostics(
+  private def reportDiagnostics(
     diagnostics: List[IR.Diagnostic],
     source: Source
   ): Boolean = {

engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala

@@ -15,14 +15,28 @@ import org.enso.librarymanager.{
   ResolvingLibraryProvider
 }
 import org.enso.logger.masking.MaskedPath
-import org.enso.pkg.{Package, PackageManager, QualifiedName}
-
+import org.enso.pkg.{
+  ComponentGroup,
+  ComponentGroups,
+  ExtendedComponentGroup,
+  Package,
+  PackageManager,
+  QualifiedName
+}
 import java.nio.file.Path
+
 import scala.util.Try
 
 /** Manages loaded packages and modules. */
 trait PackageRepository {
 
+  /** Initialize the package repository.
+    *
+    * @return `Right` if the package repository initialized successfully,
+    * and a `Left` containing an error otherwise.
+    */
+  def initialize(): Either[PackageRepository.Error, Unit]
+
   /** Informs the repository that it should populate the top scope with modules
     * belonging to a given package.
     *
@@ -51,6 +65,9 @@ trait PackageRepository {
     */
   def freezeModuleMap: PackageRepository.FrozenModuleMap
 
+  /** Get the loaded library components. */
+  def getComponents: PackageRepository.ComponentsMap
+
   /** Get a loaded module by its qualified name. */
   def getLoadedModule(qualifiedName: String): Option[Module]
 
@@ -80,6 +97,7 @@ object PackageRepository {
 
   type ModuleMap       = collection.concurrent.Map[String, Module]
   type FrozenModuleMap = Map[String, Module]
+  type ComponentsMap   = Map[LibraryName, ComponentGroups]
 
   /** A trait representing errors reported by this system */
   sealed trait Error
@@ -143,7 +161,7 @@ object PackageRepository {
       * unsynchronized threads read this map, every element it contains is
       * already fully processed.
       */
-    val loadedPackages
+    private val loadedPackages
       : collection.concurrent.Map[LibraryName, Option[Package[TruffleFile]]] = {
       val builtinsName = LibraryName(Builtins.NAMESPACE, Builtins.PACKAGE_NAME)
       collection.concurrent.TrieMap(builtinsName -> None)
@@ -157,15 +175,30 @@ object PackageRepository {
       * Strings, and constantly converting them into [[QualifiedName]]s would
       * add more overhead than is probably necessary.
       */
-    val loadedModules: collection.concurrent.Map[String, Module] =
+    private val loadedModules: collection.concurrent.Map[String, Module] =
       collection.concurrent.TrieMap(Builtins.MODULE_NAME -> builtins.getModule)
 
+    /** The mapping containing loaded component groups.
+      *
+      * The component mapping is added to the collection after ensuring that the
+      * corresponding library was loaded.
+      */
+    private val loadedComponents
+      : collection.concurrent.TrieMap[LibraryName, ComponentGroups] = {
+      val builtinsName = LibraryName(Builtins.NAMESPACE, Builtins.PACKAGE_NAME)
+      collection.concurrent.TrieMap(builtinsName -> ComponentGroups.empty)
+    }
+
     /** @inheritdoc */
     override def getModuleMap: ModuleMap = loadedModules
 
     /** @inheritdoc */
     override def freezeModuleMap: FrozenModuleMap = loadedModules.toMap
 
+    /** @inheritdoc */
+    override def getComponents: ComponentsMap =
+      loadedComponents.readOnlySnapshot().toMap
+
     /** @inheritdoc */
     override def registerMainProjectPackage(
       libraryName: LibraryName,
@@ -215,7 +248,7 @@ object PackageRepository {
       libraryName: LibraryName,
       libraryVersion: LibraryVersion,
       root: Path
-    ): Either[Error, Unit] = Try {
+    ): Either[Error, Package[TruffleFile]] = Try {
       logger.debug(
         s"Loading library $libraryName from " +
         s"[${MaskedPath(root).applyMasking()}]."
@@ -230,8 +263,91 @@ object PackageRepository {
         pkg            = pkg,
         isLibrary      = true
       )
+      pkg
     }.toEither.left.map { error => Error.PackageLoadingError(error.getMessage) }
 
+    /** @inheritdoc */
+    override def initialize(): Either[Error, Unit] = this.synchronized {
+      val unprocessedPackages =
+        loadedPackages.keySet
+          .diff(loadedComponents.keySet)
+          .flatMap(loadedPackages(_))
+      unprocessedPackages.foldLeft[Either[Error, Unit]](Right(())) {
+        (accumulator, pkg) =>
+          for {
+            _ <- accumulator
+            _ <- resolveComponentGroups(pkg)
+          } yield ()
+      }
+    }
+
+    private def resolveComponentGroups(
+      pkg: Package[TruffleFile]
+    ): Either[Error, Unit] = {
+      if (loadedComponents.contains(pkg.libraryName)) Right(())
+      else {
+        pkg.config.componentGroups match {
+          case Left(err) =>
+            Left(Error.PackageLoadingError(err.getMessage()))
+          case Right(componentGroups) =>
+            logger.debug(
+              s"Resolving component groups of package [${pkg.name}]."
+            )
+
+            registerComponentGroups(pkg.libraryName, componentGroups.newGroups)
+            componentGroups.extendedGroups
+              .foldLeft[Either[Error, Unit]](Right(())) {
+                (accumulator, componentGroup) =>
+                  for {
+                    _ <- accumulator
+                    extendedLibraryName = componentGroup.module.libraryName
+                    _ <- ensurePackageIsLoaded(extendedLibraryName)
+                    pkgOpt = loadedPackages(extendedLibraryName)
+                    _ <- pkgOpt.fold[Either[Error, Unit]](Right(()))(
+                      resolveComponentGroups
+                    )
+                    _ = registerExtendedComponentGroup(
+                      pkg.libraryName,
+                      componentGroup
+                    )
+                  } yield ()
+              }
+        }
+      }
+    }
+
+    /** Register the list of component groups defined by a library.
+      *
+      * @param library the library name
+      * @param newGroups the list of component groups that the library defines
+      */
+    private def registerComponentGroups(
+      library: LibraryName,
+      newGroups: List[ComponentGroup]
+    ): Unit =
+      loadedComponents.updateWith(library) {
+        case Some(groups) =>
+          Some(groups.copy(newGroups = groups.newGroups ::: newGroups))
+        case None =>
+          Some(ComponentGroups(newGroups, List()))
+      }
+
+    /** Register a component group extended by a library.
+      *
+      * @param library the library name
+      * @param group the extended component group
+      */
+    private def registerExtendedComponentGroup(
+      library: LibraryName,
+      group: ExtendedComponentGroup
+    ): Unit =
+      loadedComponents.updateWith(library) {
+        case Some(groups) =>
+          Some(groups.copy(extendedGroups = groups.extendedGroups :+ group))
+        case None =>
+          Some(ComponentGroups(List(), List(group)))
+      }
+
     /** @inheritdoc */
     override def ensurePackageIsLoaded(
       libraryName: LibraryName
@@ -242,7 +358,7 @@ object PackageRepository {
         val resolvedLibrary = libraryProvider.findLibrary(libraryName)
         resolvedLibrary match {
           case Left(error) =>
-            logger.error(s"Resolution failed with [$error].", error)
+            logger.warn(s"Resolution failed with [$error].", error)
           case Right(resolved) =>
             logger.info(
               s"Found library ${resolved.name} @ ${resolved.version} " +
@@ -259,6 +375,7 @@ object PackageRepository {
               .flatMap { library =>
                 loadPackage(library.name, library.version, library.location)
               }
+              .flatMap(resolveComponentGroups)
               .left
               .map {
                 case ResolvingLibraryProvider.Error.NotResolved(details) =>