Library Publishing MVP

.github/workflows/scala.yml

@@ -16,7 +16,7 @@ env:
   # Please ensure that this is in sync with rustVersion in build.sbt
   rustToolchain: nightly-2021-05-12
   # Some moderately recent version of Node.JS is needed to run the library download tests.
-  nodeVersion: 12.18.4
+  nodeVersion: 14.17.2
 
 jobs:
   test_and_publish:

RELEASES.md

@@ -12,6 +12,10 @@
 - Implemented a basic library downloader
   ([#1885](https://github.com/enso-org/enso/pull/1885)), allowing the
   downloading of missing libraries.
+- Implemented a basic library uploader
+  ([#1898](https://github.com/enso-org/enso/pull/1898)). It implements the
+  `library/publish` endpoint of the Language Server and adds a `publish-library`
+  subcommand to the Launcher.
 
 ## Libraries
 

build.sbt

@@ -251,6 +251,7 @@ lazy val enso = (project in file("."))
     `distribution-manager`,
     `edition-updater`,
     `library-manager`,
+    `library-manager-test`,
     syntax.jvm,
     testkit
   )
@@ -1023,6 +1024,7 @@ lazy val `language-server` = (project in file("engine/language-server"))
   .dependsOn(`version-output`)
   .dependsOn(pkg)
   .dependsOn(testkit % Test)
+  .dependsOn(`library-manager-test` % Test)
   .dependsOn(`runtime-version-manager-test` % Test)
 
 lazy val ast = (project in file("lib/scala/ast"))
@@ -1178,22 +1180,6 @@ lazy val runtime = (project in file("engine/runtime"))
       case _ => MergeStrategy.first
     }
   )
-  .settings(
-    (Compile / compile) := (Compile / compile)
-      .dependsOn(
-        Def.task {
-          Editions.writeEditionConfig(
-            ensoVersion = ensoVersion,
-            editionName = currentEdition,
-            libraryVersion =
-              "0.1.0", // TODO [RW] Once we start releasing the standard libraries, this will be synced with engine version.
-            log = streams.value.log
-          )
-        }
-      )
-      .value,
-    cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions"
-  )
   .dependsOn(pkg)
   .dependsOn(`interpreter-dsl`)
   .dependsOn(syntax.jvm)
@@ -1270,6 +1256,7 @@ lazy val `engine-runner` = project
   )
   .dependsOn(`version-output`)
   .dependsOn(pkg)
+  .dependsOn(cli)
   .dependsOn(`library-manager`)
   .dependsOn(`language-server`)
   .dependsOn(`polyglot-api`)
@@ -1359,6 +1346,22 @@ lazy val editions = project
       "org.scalatest"              %% "scalatest"     % scalatestVersion % Test
     )
   )
+  .settings(
+    (Compile / compile) := (Compile / compile)
+      .dependsOn(
+        Def.task {
+          Editions.writeEditionConfig(
+            ensoVersion = ensoVersion,
+            editionName = currentEdition,
+            libraryVersion =
+              "0.1.0", // TODO [RW] Once we start releasing the standard libraries, this will be synced with engine version.
+            log = streams.value.log
+          )
+        }
+      )
+      .value,
+    cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions"
+  )
   .dependsOn(testkit % Test)
 
 lazy val downloader = (project in file("lib/scala/downloader"))
@@ -1406,6 +1409,19 @@ lazy val `library-manager` = project
   .dependsOn(testkit % Test)
   .dependsOn(`logging-service` % Test)
 
+lazy val `library-manager-test` = project
+  .in(file("lib/scala/library-manager-test"))
+  .configs(Test)
+  .settings(
+    libraryDependencies ++= Seq(
+      "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion,
+      "org.scalatest"              %% "scalatest"     % scalatestVersion % Test
+    )
+  )
+  .dependsOn(`library-manager`)
+  .dependsOn(testkit)
+  .dependsOn(`logging-service`)
+
 lazy val `runtime-version-manager` = project
   .in(file("lib/scala/runtime-version-manager"))
   .configs(Test)

docs/language-server/protocol-language-server.md

@@ -200,6 +200,7 @@ transport formats, please look [here](./protocol-architecture).
   - [`LibraryDownloadError`](#librarydownloaderror)
   - [`LocalLibraryNotFound`](#locallibrarynotfound)
   - [`LibraryNotResolved`](#librarynotresolved)
+  - [`InvalidLibraryName`](#invalidlibraryname)
 
 <!-- /MarkdownTOC -->
 
@@ -4328,6 +4329,8 @@ null;
 
 #### Errors
 
+- [`InvalidLibraryName`](#invalidlibraryname) to signal that the selected
+  library name is not valid.
 - [`LibraryAlreadyExists`](#libraryalreadyexists) to signal that a library with
   the given namespace and name already exists.
 - [`FileSystemError`](#filesystemerror) to signal a generic, unrecoverable
@@ -4406,6 +4409,9 @@ versions. This is a temporary solution and in the longer-term it should be
 replaced with separate settings allowing to arbitrarily modify the library
 version from the IDE.
 
+The `uploadUrl` is the URL of the library repository that accepts library
+uploads.
+
 The metadata for publishing the library can be set with
 [`library/setMetadata`](#librarysetmetadata). If it was not set, the publish
 operation will still proceed, but that metadata will be missing.
@@ -4417,6 +4423,7 @@ operation will still proceed, but that metadata will be missing.
   namespace: String;
   name: String;
   authToken: String;
+  uploadUrl: String;
 
   bumpVersionAfterPublish?: Boolean;
 }
@@ -4430,6 +4437,8 @@ null;
 
 #### Errors
 
+- [`LocalLibraryNotFound`](#locallibrarynotfound) to signal that a local library
+  with the given name does not exist on the local libraries path.
 - [`LibraryPublishError`](#librarypublisherror) to signal that the server did
   not accept to publish the library (for example because a library with the same
   version already exists).
@@ -4994,3 +5003,21 @@ there either.
   }
 }
 ```
+
+### `InvalidLibraryName`
+
+Signals that the chosen library name is invalid.
+
+It contains a suggestion of a similar name that is valid.
+
+For example for `FooBar` it will suggest `Foo_Bar`.
+
+```typescript
+"error" : {
+  "code" : 8009,
+  "message" : "[<name>] is not a valid name: <reason>.",
+  "payload" : {
+    "suggestedName" : "<fixed-name>"
+  }
+}
+```

docs/libraries/repositories.md

@@ -234,6 +234,31 @@ them), it will result in the following merged directory structure:
 └── LICENSE.md
 ```
 
+### Publishing
+
+To be able to publish libraries to a repository, the repository must provide an
+upload endpoint which satisfies the following requirements.
+
+The endpoint should get the library name and version from the query parameters:
+`namespace`, `name` and `version`.
+
+It should check any authentication data attached to the query and verify that
+the user has sufficient privileges to upload the library for that `namespace`.
+
+Currently, we use a static check which checks an `Auth-Token` header for a
+pre-determined secret key, but any other authentication schemes can be used, as
+long as they are supported by the GUI or CLI.
+
+Then, the server must check if a library with the given name and version
+combination already exists. If the library already exists, the request should be
+rejected with `409 Conflict` status code indicating that a conflicting library
+is already in the repository.
+
+If the request goes through, the server should create a directory for the
+library and put any files attached to the request there. Each request should
+always contain `package.yaml` and `manifest.yaml` files attached and at least
+one sub-archive, usually called `main.tgz`.
+
 ## Editions Repository
 
 The Editions repository has a very simple structure.

docs/libraries/sharing.md

@@ -62,5 +62,17 @@ import <namespace>.<Project_Name>
 
 ## Publishing
 
-> Soon it will be possible to share the libraries through the Marketplace, but
-> it is still a work in progress.
+To publish a library, first you must obtain the upload URL of the repository, if
+you are hosting the repository locally it will be `http://localhost:8080/upload`
+(or possibly with a different port if that was overridden).
+
+If the repository requires authentication, it is best to set it up by setting
+the `ENSO_AUTH_TOKEN` environment variable to the value of your secret token.
+
+Then you can use the Enso CLI to upload the project:
+
+```bash
+enso publish-library --upload-url <URL> <path to project root>
+```
+
+See `enso publish-library --help` for more information.

engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala

@@ -164,6 +164,7 @@ object LibraryApi {
       namespace: String,
       name: String,
       authToken: String,
+      uploadUrl: String,
       bumpVersionAfterPublish: Option[Boolean]
     )
 
@@ -232,4 +233,16 @@ object LibraryApi {
             } """
     )
   }
+
+  case class InvalidLibraryName(
+    originalName: String,
+    suggestedName: String,
+    reason: String
+  ) extends Error(8009, s"[$originalName] is not a valid name: $reason.") {
+    override def payload: Option[Json] = Some(
+      json""" {
+            "suggestedName" : $suggestedName
+            } """
+    )
+  }
 }

engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManager.scala

@@ -5,8 +5,12 @@ import com.typesafe.scalalogging.LazyLogging
 import org.enso.distribution.{DistributionManager, FileSystem}
 import org.enso.editions.{Editions, LibraryName}
 import org.enso.languageserver.libraries.LocalLibraryManagerProtocol._
-import org.enso.librarymanager.local.LocalLibraryProvider
+import org.enso.librarymanager.local.{
+  DefaultLocalLibraryProvider,
+  LocalLibraryProvider
+}
 import org.enso.pkg.PackageManager
+import org.enso.pkg.validation.NameValidation
 
 import java.io.File
 import java.nio.file.Files
@@ -34,9 +38,18 @@ class LocalLibraryManager(
         sender() ! listLocalLibraries()
       case Create(libraryName, authors, maintainers, license) =>
         sender() ! createLibrary(libraryName, authors, maintainers, license)
-      case Publish(_, _, _) =>
-        logger.error("Publishing libraries is currently not implemented.")
-        sender() ! Failure(new NotImplementedError())
+      case FindLibrary(libraryName) =>
+        sender() ! findLibrary(libraryName)
+    }
+  }
+
+  /** Checks if the library name is a valid Enso module name. */
+  private def validateLibraryName(libraryName: LibraryName): Unit = {
+    // TODO [RW] more specific exceptions
+    NameValidation.validateName(libraryName.name) match {
+      case Left(error) =>
+        throw new RuntimeException(s"Library name is not valid: [$error].")
+      case Right(_) =>
     }
   }
 
@@ -54,6 +67,8 @@ class LocalLibraryManager(
     // TODO [RW] modify protocol to be able to create Contact instances
     val _ = (authors, maintainers)
 
+    validateLibraryName(libraryName)
+
     // TODO [RW] make the exceptions more relevant
     val possibleRoots = LazyList
       .from(distributionManager.paths.localLibrariesSearchPaths)
@@ -104,6 +119,17 @@ class LocalLibraryManager(
     } yield LibraryName(namespace, name)
   }
 
+  /** Finds the path on the filesystem to a local library. */
+  private def findLibrary(
+    libraryName: LibraryName
+  ): Try[FindLibraryResponse] = Try {
+    val localLibraryProvider = new DefaultLocalLibraryProvider(
+      distributionManager.paths.localLibrariesSearchPaths.toList
+    )
+    val pathOpt = localLibraryProvider.findLibrary(libraryName)
+    FindLibraryResponse(pathOpt)
+  }
+
   /** Finds the edition associated with the current project, if specified in its
     * config.
     */

engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManagerProtocol.scala

@@ -2,6 +2,8 @@ package org.enso.languageserver.libraries
 
 import org.enso.editions.LibraryName
 
+import java.nio.file.Path
+
 object LocalLibraryManagerProtocol {
 
   /** A top class representing any request to the [[LocalLibraryManager]]. */
@@ -37,10 +39,9 @@ object LocalLibraryManagerProtocol {
     license: String
   ) extends Request
 
-  /** A request to publish a library. */
-  case class Publish(
-    libraryName: LibraryName,
-    authToken: String,
-    bumpVersionAfterPublish: Boolean
-  ) extends Request
+  /** A request to find the path to a local library. */
+  case class FindLibrary(libraryName: LibraryName) extends Request
+
+  /** A response to [[FindLibrary]]. */
+  case class FindLibraryResponse(libraryRoot: Option[Path])
 }