Skip to content

Commit

Permalink
Preinstalling With Dependencies (#1981)
Browse files Browse the repository at this point in the history
  • Loading branch information
radeusgd authored Nov 23, 2021
1 parent d61743d commit 46c31bb
Show file tree
Hide file tree
Showing 41 changed files with 1,161 additions and 123 deletions.
4 changes: 3 additions & 1 deletion .github/workflows/scala.yml
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,8 @@ jobs:
sbt --no-colors "project-manager/assembly"
sbt --no-colors --mem 1536 "project-manager/buildNativeImage"
# The runtime/clean is needed to avoid issues with Truffle Instrumentation.
# It should be removed once #1992 is fixed.
- name: Build the Runner & Runtime Uberjars
run: |
sleep 1
Expand All @@ -167,7 +169,7 @@ jobs:
- name: Check Language Server Benchmark Compilation
run: |
sleep 1
sbt --no-colors language-server/Benchmark/compile
sbt --no-colors "runtime/clean; language-server/Benchmark/compile"
- name: Check Searcher Benchmark Compilation
run: |
sleep 1
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ distribution/lib/Standard/Table/*/polyglot/
distribution/lib/Standard/Database/*/polyglot/
distribution/lib/Standard/Examples/*/data/spreadsheet.xls
distribution/lib/Standard/Examples/*/data/spreadsheet.xlsx
distribution/lib/*/*/*/manifest.yaml

test/Google_Api_Test/data/secret.json
test/Database_Tests/data/redshift_credentials.json
Expand Down
8 changes: 8 additions & 0 deletions RELEASES.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
# Enso Next

## Tooling

- Added the `enso install dependencies` command to the launcher which installs
any project dependencies, ensuring that `enso run` will not need to download
any libraries ([#1981](https://github.com/enso-org/enso/pull/1981)).
Additionally, made the `library/preinstall` endpoint able to install any
transitive dependencies of the library.

## Enso 0.2.31 (2021-10-01)

## Interpreter/Runtime
Expand Down
47 changes: 45 additions & 2 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import LibraryManifestGenerator.BundledLibrary
import org.enso.build.BenchTasks._
import org.enso.build.WithDebugCommand
import sbt.Keys.{libraryDependencies, scalacOptions}
Expand Down Expand Up @@ -1013,6 +1014,27 @@ lazy val `language-server` = (project in file("engine/language-server"))
new TestFramework("org.scalameter.ScalaMeterFramework")
)
)
.settings(
// These settings are needed by language-server tests that create a runtime context.
Test / fork := true,
Test / javaOptions ++= {
// Note [Classpath Separation]
val runtimeClasspath =
(LocalProject("runtime") / Compile / fullClasspath).value
.map(_.data)
.mkString(File.pathSeparator)
Seq(
s"-Dtruffle.class.path.append=$runtimeClasspath",
s"-Duser.dir=${file(".").getCanonicalPath}"
)
},
Test / compile := (Test / compile)
.dependsOn(LocalProject("enso") / updateLibraryManifests)
.value,
Test / envVars ++= Map(
"ENSO_EDITION_PATH" -> file("distribution/editions").getCanonicalPath
)
)
.dependsOn(`json-rpc-server-test` % Test)
.dependsOn(`json-rpc-server`)
.dependsOn(`task-progress-notifications`)
Expand Down Expand Up @@ -1632,7 +1654,7 @@ lazy val `std-database` = project
`database-polyglot-root`,
Some("std-database.jar"),
ignoreScalaLibrary = true,
unpackedDeps = Set("aws-java-sdk-core", "httpclient")
unpackedDeps = Set("aws-java-sdk-core", "httpclient")
)
.value
result
Expand Down Expand Up @@ -1684,7 +1706,8 @@ projectManagerDistributionRoot :=
lazy val buildEngineDistribution =
taskKey[Unit]("Builds the engine distribution")
buildEngineDistribution := {
val _ = (`engine-runner` / assembly).value
val _ = (`engine-runner` / assembly).value
updateLibraryManifests.value
val root = engineDistributionRoot.value
val log = streams.value.log
val cacheFactory = streams.value.cacheStoreFactory
Expand Down Expand Up @@ -1744,3 +1767,23 @@ buildGraalDistribution := {
DistributionPackage.Architecture.X64
)
}

lazy val updateLibraryManifests =
taskKey[Unit](
"Recomputes dependencies to update manifests bundled with libraries."
)
updateLibraryManifests := {
val _ = (`engine-runner` / assembly).value
val log = streams.value.log
val cacheFactory = streams.value.cacheStoreFactory
val libraries = Editions.standardLibraries.map(libName =>
BundledLibrary(libName, stdLibVersion)
)

LibraryManifestGenerator.generateManifests(
libraries,
file("distribution"),
log,
cacheFactory
)
}
15 changes: 15 additions & 0 deletions docs/language-server/protocol-language-server.md
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ transport formats, please look [here](./protocol-architecture).
- [`LocalLibraryNotFound`](#locallibrarynotfound)
- [`LibraryNotResolved`](#librarynotresolved)
- [`InvalidLibraryName`](#invalidlibraryname)
- [`DependencyDiscoveryError`](#dependencydiscoveryerror)

<!-- /MarkdownTOC -->

Expand Down Expand Up @@ -4550,6 +4551,8 @@ null;

- [`LibraryNotResolved`](#librarynotresolved) to signal that the requested
library or one of its dependencies could not be resolved.
- [`DependencyDiscoveryError`](#dependencydiscoveryerror) to signal that
dependencies of the library could not be established.
- [`LibraryDownloadError`](#librarydownloaderror) to signal that the download
operation has failed, for network-related reasons, or because the library was
missing in the repository. The error includes the name and version of the
Expand Down Expand Up @@ -5085,3 +5088,15 @@ For example for `FooBar` it will suggest `Foo_Bar`.
}
}
```

### `DependencyDiscoveryError`

Signals that the library preinstall endpoint could not properly find
dependencies of the requested library.

```typescript
"error" : {
"code" : 8010,
"message" : "Error occurred while discovering dependencies: <reason>."
}
```
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,7 @@ import org.enso.languageserver.effect.ZioExec
import org.enso.languageserver.filemanager._
import org.enso.languageserver.http.server.BinaryWebSocketServer
import org.enso.languageserver.io._
import org.enso.languageserver.libraries.{
EditionReferenceResolver,
LibraryConfig,
LibraryInstallerConfig,
LocalLibraryManager,
ProjectSettingsManager
}
import org.enso.languageserver.libraries._
import org.enso.languageserver.monitoring.{
HealthCheckEndpoint,
IdlenessEndpoint,
Expand Down Expand Up @@ -330,7 +324,8 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: LogLevel) {
installerConfig = LibraryInstallerConfig(
distributionManager,
resourceManager,
Some(languageHome)
Some(languageHome),
new CompilerBasedDependencyExtractor(logLevel)
)
)

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package org.enso.languageserver.libraries

import org.enso.editions.LibraryName
import org.enso.libraryupload.DependencyExtractor
import org.enso.loggingservice.{JavaLoggingLogHandler, LogLevel}
import org.enso.pkg.Package
import org.enso.pkg.SourceFile
import org.enso.polyglot.{PolyglotContext, RuntimeOptions}
import org.graalvm.polyglot.Context

import java.io.File

/** A dependency extractor that runs the compiler in a mode that only parses the
* source code and runs just the basic preprocessing phases to find out what
* libraries are imported by the project.
*
* @param logLevel the log level to use for the runtime context that will do
* the parsing
*/
class CompilerBasedDependencyExtractor(logLevel: LogLevel)
extends DependencyExtractor[File] {

/** @inheritdoc */
override def findDependencies(pkg: Package[File]): Set[LibraryName] = {
val context = createContextWithProject(pkg)

def findImportedLibraries(file: SourceFile[File]): Set[LibraryName] = {
val module = context.getTopScope.getModule(file.qualifiedName.toString)
val imports = module.gatherImportStatements()
val importedLibraries = imports.map { rawName =>
LibraryName.fromString(rawName) match {
case Left(error) =>
throw new IllegalStateException(error)
case Right(value) => value
}
}
importedLibraries.toSet
}

val sourcesImports = pkg.listSources.toSet.flatMap(findImportedLibraries)
val itself = pkg.libraryName

// Builtins need to be removed from the set of the dependencies, because
// even if they are imported, they are not a typical library.
val builtins = LibraryName("Standard", "Builtins")

sourcesImports - itself - builtins
}

/** Creates a simple runtime context with the given package loaded as its
* project root.
*/
private def createContextWithProject(pkg: Package[File]): PolyglotContext = {
val context = Context
.newBuilder()
.allowExperimentalOptions(true)
.allowAllAccess(true)
.option(RuntimeOptions.PROJECT_ROOT, pkg.root.getCanonicalPath)
.option("js.foreign-object-prototype", "true")
.option(
RuntimeOptions.LOG_LEVEL,
JavaLoggingLogHandler.getJavaLogLevelFor(logLevel).getName
)
.logHandler(
JavaLoggingLogHandler.create(JavaLoggingLogHandler.defaultLevelMapping)
)
.build
new PolyglotContext(context)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -250,4 +250,10 @@ object LibraryApi {
} """
)
}

case class DependencyDiscoveryError(reason: String)
extends Error(
8010,
s"Error occurred while discovering dependencies: $reason."
)
}
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
package org.enso.languageserver.libraries

import org.enso.distribution.{DistributionManager, LanguageHome}
import org.enso.distribution.locking.ResourceManager
import org.enso.distribution.{DistributionManager, LanguageHome}
import org.enso.libraryupload.DependencyExtractor

import java.io.File

/** Gathers configuration needed by the library installer used in the
* `library/preinstall` endpoint.
*
* @param distributionManager the distribution manager
* @param resourceManager a resource manager instance
* @param languageHome language home, if detected / applicable
* @param dependencyExtractor a dependency extractor
*/
case class LibraryInstallerConfig(
distributionManager: DistributionManager,
resourceManager: ResourceManager,
languageHome: Option[LanguageHome]
languageHome: Option[LanguageHome],
dependencyExtractor: DependencyExtractor[File]
)
Loading

0 comments on commit 46c31bb

Please sign in to comment.