diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index ae7732164564..4201eda3642e 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -16,7 +16,7 @@ env: # Please ensure that this is in sync with rustVersion in build.sbt rustToolchain: nightly-2021-05-12 # Some moderately recent version of Node.JS is needed to run the library download tests. - nodeVersion: 12.18.4 + nodeVersion: 14.17.2 jobs: test_and_publish: diff --git a/RELEASES.md b/RELEASES.md index a3cd3a0387d3..e310bb6ca456 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,5 +1,12 @@ # Enso Next +## Tooling + +- Implemented a basic library uploader + ([#1933](https://github.com/enso-org/enso/pull/1933)). It implements the + `library/publish` endpoint of the Language Server and adds a `publish-library` + subcommand to the Launcher. + # Enso 0.2.23 (2021-08-09) ## Miscellaneous diff --git a/build.sbt b/build.sbt index 9a736dd00fa5..5e370b0e972e 100644 --- a/build.sbt +++ b/build.sbt @@ -251,6 +251,7 @@ lazy val enso = (project in file(".")) `distribution-manager`, `edition-updater`, `library-manager`, + `library-manager-test`, syntax.jvm, testkit ) @@ -1023,6 +1024,7 @@ lazy val `language-server` = (project in file("engine/language-server")) .dependsOn(`version-output`) .dependsOn(pkg) .dependsOn(testkit % Test) + .dependsOn(`library-manager-test` % Test) .dependsOn(`runtime-version-manager-test` % Test) lazy val ast = (project in file("lib/scala/ast")) @@ -1178,22 +1180,6 @@ lazy val runtime = (project in file("engine/runtime")) case _ => MergeStrategy.first } ) - .settings( - (Compile / compile) := (Compile / compile) - .dependsOn( - Def.task { - Editions.writeEditionConfig( - ensoVersion = ensoVersion, - editionName = currentEdition, - libraryVersion = - "0.1.0", // TODO [RW] Once we start releasing the standard libraries, this will be synced with engine version. - log = streams.value.log - ) - } - ) - .value, - cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions" - ) .dependsOn(pkg) .dependsOn(`interpreter-dsl`) .dependsOn(syntax.jvm) @@ -1270,6 +1256,7 @@ lazy val `engine-runner` = project ) .dependsOn(`version-output`) .dependsOn(pkg) + .dependsOn(cli) .dependsOn(`library-manager`) .dependsOn(`language-server`) .dependsOn(`polyglot-api`) @@ -1359,6 +1346,22 @@ lazy val editions = project "org.scalatest" %% "scalatest" % scalatestVersion % Test ) ) + .settings( + (Compile / compile) := (Compile / compile) + .dependsOn( + Def.task { + Editions.writeEditionConfig( + ensoVersion = ensoVersion, + editionName = currentEdition, + libraryVersion = + "0.1.0", // TODO [RW] Once we start releasing the standard libraries, this will be synced with engine version. + log = streams.value.log + ) + } + ) + .value, + cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions" + ) .dependsOn(testkit % Test) lazy val downloader = (project in file("lib/scala/downloader")) @@ -1406,6 +1409,19 @@ lazy val `library-manager` = project .dependsOn(testkit % Test) .dependsOn(`logging-service` % Test) +lazy val `library-manager-test` = project + .in(file("lib/scala/library-manager-test")) + .configs(Test) + .settings( + libraryDependencies ++= Seq( + "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, + "org.scalatest" %% "scalatest" % scalatestVersion % Test + ) + ) + .dependsOn(`library-manager`) + .dependsOn(testkit) + .dependsOn(`logging-service`) + lazy val `runtime-version-manager` = project .in(file("lib/scala/runtime-version-manager")) .configs(Test) diff --git a/docs/language-server/protocol-language-server.md b/docs/language-server/protocol-language-server.md index cb59887fb0bd..22447bf155bd 100644 --- a/docs/language-server/protocol-language-server.md +++ b/docs/language-server/protocol-language-server.md @@ -200,6 +200,7 @@ transport formats, please look [here](./protocol-architecture). - [`LibraryDownloadError`](#librarydownloaderror) - [`LocalLibraryNotFound`](#locallibrarynotfound) - [`LibraryNotResolved`](#librarynotresolved) + - [`InvalidLibraryName`](#invalidlibraryname) @@ -4344,6 +4345,8 @@ null; #### Errors +- [`InvalidLibraryName`](#invalidlibraryname) to signal that the selected + library name is not valid. - [`LibraryAlreadyExists`](#libraryalreadyexists) to signal that a library with the given namespace and name already exists. - [`FileSystemError`](#filesystemerror) to signal a generic, unrecoverable @@ -4422,6 +4425,9 @@ versions. This is a temporary solution and in the longer-term it should be replaced with separate settings allowing to arbitrarily modify the library version from the IDE. +The `uploadUrl` is the URL of the library repository that accepts library +uploads. + The metadata for publishing the library can be set with [`library/setMetadata`](#librarysetmetadata). If it was not set, the publish operation will still proceed, but that metadata will be missing. @@ -4433,6 +4439,7 @@ operation will still proceed, but that metadata will be missing. namespace: String; name: String; authToken: String; + uploadUrl: String; bumpVersionAfterPublish?: Boolean; } @@ -4446,6 +4453,8 @@ null; #### Errors +- [`LocalLibraryNotFound`](#locallibrarynotfound) to signal that a local library + with the given name does not exist on the local libraries path. - [`LibraryPublishError`](#librarypublisherror) to signal that the server did not accept to publish the library (for example because a library with the same version already exists). @@ -5010,3 +5019,21 @@ there either. } } ``` + +### `InvalidLibraryName` + +Signals that the chosen library name is invalid. + +It contains a suggestion of a similar name that is valid. + +For example for `FooBar` it will suggest `Foo_Bar`. + +```typescript +"error" : { + "code" : 8009, + "message" : "[] is not a valid name: .", + "payload" : { + "suggestedName" : "" + } +} +``` diff --git a/docs/libraries/repositories.md b/docs/libraries/repositories.md index b754a39c7f34..e24b618ef41d 100644 --- a/docs/libraries/repositories.md +++ b/docs/libraries/repositories.md @@ -234,6 +234,31 @@ them), it will result in the following merged directory structure: └── LICENSE.md ``` +### Publishing + +To be able to publish libraries to a repository, the repository must provide an +upload endpoint which satisfies the following requirements. + +The endpoint should get the library name and version from the query parameters: +`namespace`, `name` and `version`. + +It should check any authentication data attached to the query and verify that +the user has sufficient privileges to upload the library for that `namespace`. + +Currently, we use a static check which checks an `Auth-Token` header for a +pre-determined secret key, but any other authentication schemes can be used, as +long as they are supported by the GUI or CLI. + +Then, the server must check if a library with the given name and version +combination already exists. If the library already exists, the request should be +rejected with `409 Conflict` status code indicating that a conflicting library +is already in the repository. + +If the request goes through, the server should create a directory for the +library and put any files attached to the request there. Each request should +always contain `package.yaml` and `manifest.yaml` files attached and at least +one sub-archive, usually called `main.tgz`. + ## Editions Repository The Editions repository has a very simple structure. diff --git a/docs/libraries/sharing.md b/docs/libraries/sharing.md index e0f46b2a0773..fa71214d77f7 100644 --- a/docs/libraries/sharing.md +++ b/docs/libraries/sharing.md @@ -62,5 +62,17 @@ import . ## Publishing -> Soon it will be possible to share the libraries through the Marketplace, but -> it is still a work in progress. +To publish a library, first you must obtain the upload URL of the repository, if +you are hosting the repository locally it will be `http://localhost:8080/upload` +(or possibly with a different port if that was overridden). + +If the repository requires authentication, it is best to set it up by setting +the `ENSO_AUTH_TOKEN` environment variable to the value of your secret token. + +Then you can use the Enso CLI to upload the project: + +```bash +enso publish-library --upload-url +``` + +See `enso publish-library --help` for more information. diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala index e6c04dc883eb..36998b97f1b9 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala @@ -164,6 +164,7 @@ object LibraryApi { namespace: String, name: String, authToken: String, + uploadUrl: String, bumpVersionAfterPublish: Option[Boolean] ) @@ -232,4 +233,16 @@ object LibraryApi { } """ ) } + + case class InvalidLibraryName( + originalName: String, + suggestedName: String, + reason: String + ) extends Error(8009, s"[$originalName] is not a valid name: $reason.") { + override def payload: Option[Json] = Some( + json""" { + "suggestedName" : $suggestedName + } """ + ) + } } diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManager.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManager.scala index cde1650435db..eacb8ba5986e 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManager.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManager.scala @@ -5,8 +5,12 @@ import com.typesafe.scalalogging.LazyLogging import org.enso.distribution.{DistributionManager, FileSystem} import org.enso.editions.{Editions, LibraryName} import org.enso.languageserver.libraries.LocalLibraryManagerProtocol._ -import org.enso.librarymanager.local.LocalLibraryProvider +import org.enso.librarymanager.local.{ + DefaultLocalLibraryProvider, + LocalLibraryProvider +} import org.enso.pkg.PackageManager +import org.enso.pkg.validation.NameValidation import java.io.File import java.nio.file.Files @@ -34,9 +38,18 @@ class LocalLibraryManager( sender() ! listLocalLibraries() case Create(libraryName, authors, maintainers, license) => sender() ! createLibrary(libraryName, authors, maintainers, license) - case Publish(_, _, _) => - logger.error("Publishing libraries is currently not implemented.") - sender() ! Failure(new NotImplementedError()) + case FindLibrary(libraryName) => + sender() ! findLibrary(libraryName) + } + } + + /** Checks if the library name is a valid Enso module name. */ + private def validateLibraryName(libraryName: LibraryName): Unit = { + // TODO [RW] more specific exceptions + NameValidation.validateName(libraryName.name) match { + case Left(error) => + throw new RuntimeException(s"Library name is not valid: [$error].") + case Right(_) => } } @@ -54,6 +67,8 @@ class LocalLibraryManager( // TODO [RW] modify protocol to be able to create Contact instances val _ = (authors, maintainers) + validateLibraryName(libraryName) + // TODO [RW] make the exceptions more relevant val possibleRoots = LazyList .from(distributionManager.paths.localLibrariesSearchPaths) @@ -104,6 +119,17 @@ class LocalLibraryManager( } yield LibraryName(namespace, name) } + /** Finds the path on the filesystem to a local library. */ + private def findLibrary( + libraryName: LibraryName + ): Try[FindLibraryResponse] = Try { + val localLibraryProvider = new DefaultLocalLibraryProvider( + distributionManager.paths.localLibrariesSearchPaths.toList + ) + val pathOpt = localLibraryProvider.findLibrary(libraryName) + FindLibraryResponse(pathOpt) + } + /** Finds the edition associated with the current project, if specified in its * config. */ diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManagerProtocol.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManagerProtocol.scala index 9f3996509c49..5414816064af 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManagerProtocol.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManagerProtocol.scala @@ -2,6 +2,8 @@ package org.enso.languageserver.libraries import org.enso.editions.LibraryName +import java.nio.file.Path + object LocalLibraryManagerProtocol { /** A top class representing any request to the [[LocalLibraryManager]]. */ @@ -37,10 +39,9 @@ object LocalLibraryManagerProtocol { license: String ) extends Request - /** A request to publish a library. */ - case class Publish( - libraryName: LibraryName, - authToken: String, - bumpVersionAfterPublish: Boolean - ) extends Request + /** A request to find the path to a local library. */ + case class FindLibrary(libraryName: LibraryName) extends Request + + /** A response to [[FindLibrary]]. */ + case class FindLibraryResponse(libraryRoot: Option[Path]) } diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala index 0b0095b454e8..18900011677c 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala @@ -1,32 +1,161 @@ package org.enso.languageserver.libraries.handler -import akka.actor.{Actor, Props} +import akka.actor.{Actor, ActorRef, Cancellable, Props} import com.typesafe.scalalogging.LazyLogging -import org.enso.jsonrpc.{Request, ResponseError} +import org.enso.cli.task.notifications.ActorProgressNotificationForwarder +import org.enso.editions.LibraryName +import org.enso.jsonrpc.{Id, Request, ResponseError, ResponseResult, Unused} import org.enso.languageserver.filemanager.FileManagerApi.FileSystemError import org.enso.languageserver.libraries.LibraryApi._ +import org.enso.languageserver.libraries.LocalLibraryManagerProtocol.{ + FindLibrary, + FindLibraryResponse +} +import org.enso.languageserver.requesthandler.RequestTimeout import org.enso.languageserver.util.UnhandledLogging +import org.enso.libraryupload.{auth, LibraryUploader} + +import scala.concurrent.duration.FiniteDuration +import scala.util.{Failure, Success} /** A request handler for the `library/publish` endpoint. * - * It is currently a stub implementation which will be refined later on. + * @param timeout request timeout + * @param localLibraryManager a reference to the LocalLibraryManager */ -class LibraryPublishHandler - extends Actor +class LibraryPublishHandler( + timeout: FiniteDuration, + localLibraryManager: ActorRef +) extends Actor with LazyLogging with UnhandledLogging { - override def receive: Receive = { - case Request(LibraryPublish, id, _: LibraryPublish.Params) => - // TODO [RW] actual implementation - sender() ! ResponseError( + override def receive: Receive = requestStage + + import context.dispatcher + + private def requestStage: Receive = { + case Request( + LibraryPublish, + id, + LibraryPublish.Params( + namespace, + name, + authToken, + uploadUrl, + bumpVersionAfterPublish + ) + ) => + val shouldBump = bumpVersionAfterPublish.getOrElse(false) + val replyTo = sender() + val token = auth.SimpleHeaderToken(authToken) + val libraryName = LibraryName(namespace, name) + localLibraryManager ! FindLibrary(libraryName) + + val timeoutCancellable = + context.system.scheduler.scheduleOnce(timeout, self, RequestTimeout) + context.become( + waitForLibraryResolutionStage( + replyTo, + libraryName, + id, + uploadUrl, + token, + timeoutCancellable, + shouldBump + ) + ) + } + + private def stop(timeoutCancellable: Cancellable): Unit = { + timeoutCancellable.cancel() + context.stop(self) + } + + /** Waits for the response of LocalLibraryManager and continues the publishing + * process. + */ + private def waitForLibraryResolutionStage( + replyTo: ActorRef, + libraryName: LibraryName, + id: Id, + uploadUrl: String, + token: auth.Token, + timeoutCancellable: Cancellable, + shouldBumpAfterPublishing: Boolean + ): Receive = { + case RequestTimeout => + replyTo ! RequestTimeout + context.stop(self) + + case Success(FindLibraryResponse(Some(libraryRoot))) => + val progressReporter = + ActorProgressNotificationForwarder.translateAndForward( + LibraryPublish.name, + replyTo + ) + + val result = LibraryUploader.uploadLibrary( + libraryRoot, + uploadUrl, + token, + progressReporter + ) + + result match { + case Failure(exception) => + replyTo ! ResponseError( + Some(id), + FileSystemError(s"Upload failed: $exception") + ) + case Success(_) => + if (shouldBumpAfterPublishing) { + logger.warn( + "`bumpVersionAfterPublish` was set to true, but this feature " + + "is not currently implemented. Ignoring." + ) + } + replyTo ! ResponseResult(LibraryPublish, id, Unused) + } + + stop(timeoutCancellable) + + case Success(FindLibraryResponse(None)) => + replyTo ! ResponseError( Some(id), - FileSystemError("Feature not implemented") + FileSystemError( + s"The library [$libraryName] was not found in local libraries " + + s"search paths." + ) ) + + stop(timeoutCancellable) + + case Failure(exception) => + replyTo ! ResponseError( + Some(id), + FileSystemError( + s"Failed to find the requested local library: $exception" + ) + ) + + stop(timeoutCancellable) } } object LibraryPublishHandler { - /** Creates a configuration object to create [[LibraryPublishHandler]]. */ - def props(): Props = Props(new LibraryPublishHandler) + /** Creates a configuration object to create [[LibraryPublishHandler]]. + * + * @param timeout request timeout + * @param localLibraryManager a reference to the LocalLibraryManager + */ + def props( + timeout: FiniteDuration, + localLibraryManager: ActorRef + ): Props = Props( + new LibraryPublishHandler( + timeout, + localLibraryManager + ) + ) } diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala b/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala index 53b157d36eb1..1435dc707a10 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala @@ -505,7 +505,8 @@ class JsonConnectionController( .props(requestTimeout, localLibraryManager), LibraryGetMetadata -> LibraryGetMetadataHandler.props(), LibraryPreinstall -> LibraryPreinstallHandler.props(), - LibraryPublish -> LibraryPublishHandler.props(), + LibraryPublish -> LibraryPublishHandler + .props(requestTimeout, localLibraryManager), LibrarySetMetadata -> LibrarySetMetadataHandler.props() ) } diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala index 1ad058252bda..340e27019af6 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala @@ -4,6 +4,9 @@ import io.circe.literal._ import io.circe.{Json, JsonObject} import org.enso.languageserver.libraries.LibraryEntry import org.enso.languageserver.libraries.LibraryEntry.PublishedLibraryVersion +import org.enso.librarymanager.published.repository.EmptyRepository + +import java.nio.file.Files class LibrariesTest extends BaseServerTest { "LocalLibraryManager" should { @@ -29,8 +32,8 @@ class LibrariesTest extends BaseServerTest { "method": "library/create", "id": 1, "params": { - "namespace": "User", - "name": "MyLocalLib", + "namespace": "user", + "name": "My_Local_Lib", "authors": [], "maintainers": [], "license": "" @@ -56,8 +59,8 @@ class LibrariesTest extends BaseServerTest { "result": { "localLibraries": [ { - "namespace": "User", - "name": "MyLocalLib", + "namespace": "user", + "name": "My_Local_Lib", "version": { "type": "LocalLibraryVersion" } @@ -72,6 +75,83 @@ class LibrariesTest extends BaseServerTest { "existed" ignore { // TODO [RW] error handling (#1877) } + + "validate the library name" ignore { + // TODO [RW] error handling (#1877) + } + + def port: Int = 47308 + + "create and publish a library" in { + val client = getInitialisedWsClient() + client.send(json""" + { "jsonrpc": "2.0", + "method": "library/create", + "id": 0, + "params": { + "namespace": "user", + "name": "Publishable_Lib", + "authors": [], + "maintainers": [], + "license": "" + } + } + """) + client.expectJson(json""" + { "jsonrpc": "2.0", + "id": 0, + "result": null + } + """) + + val repoRoot = getTestDirectory.resolve("libraries_repo_root") + val server = EmptyRepository.startServer(port, repoRoot, uploads = true) + try { + val uploadUrl = s"http://localhost:$port/upload" + client.send(json""" + { "jsonrpc": "2.0", + "method": "library/publish", + "id": 1, + "params": { + "namespace": "user", + "name": "Publishable_Lib", + "authToken": "SOME TOKEN", + "uploadUrl": $uploadUrl, + "bumpVersionAfterPublish": null + } + } + """) + + var found = false + while (!found) { + val rawResponse = client.expectSomeJson() + val response = rawResponse.asObject.value + val idMatches = + response("id").flatMap(_.asNumber).flatMap(_.toInt).contains(1) + if (idMatches) { + rawResponse shouldEqual json""" + { "jsonrpc": "2.0", + "id": 1, + "result": null + } + """ + + found = true + } + } + + val libraryRoot = repoRoot + .resolve("libraries") + .resolve("user") + .resolve("Publishable_Lib") + .resolve("0.0.1") + val mainPackage = libraryRoot.resolve("main.tgz") + assert(Files.exists(mainPackage)) + } finally { + server.kill(killDescendants = true) + server.join(waitForDescendants = true) + } + } } "mocked library/preinstall" should { diff --git a/engine/launcher/src/main/scala/org/enso/launcher/Constants.scala b/engine/launcher/src/main/scala/org/enso/launcher/Constants.scala new file mode 100644 index 000000000000..0a624b03a4e5 --- /dev/null +++ b/engine/launcher/src/main/scala/org/enso/launcher/Constants.scala @@ -0,0 +1,14 @@ +package org.enso.launcher + +import nl.gn0s1s.bump.SemVer + +object Constants { + + /** The engine version in which the uploads command has been introduced. + * + * It is used to check by the launcher if the engine can handle this command + * and provide better error messages if it cannot. + */ + val uploadIntroducedVersion: SemVer = + SemVer(0, 2, 17, Some("SNAPSHOT")) +} diff --git a/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala b/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala index 70b3576a2fe7..9a2fc6f4e5c4 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala @@ -343,6 +343,52 @@ case class Launcher(cliOptions: GlobalCLIOptions) { 0 } + /** Uploads a library to a repository. + * + * @param path path to the library, if not specified, the current working + * directory and its ancestors are searched for an Enso project + * to upload + * @param uploadUrl a URL of an upload endpoint of a repository; if not + * specified, falls back to the default Enso repository + * @param authToken a token to use for authentication + * @param logLevel log level for the language server + * @param useSystemJVM if set, forces to use the default configured JVM, + * instead of the JVM associated with the engine version + * @param jvmOpts additional options to pass to the launched JVM + * @param additionalArguments additional arguments to pass to the runner + * @return exit code of the launched program + */ + def uploadLibrary( + path: Option[Path], + uploadUrl: Option[String], + authToken: Option[String], + logLevel: LogLevel, + useSystemJVM: Boolean, + jvmOpts: Seq[(String, String)], + additionalArguments: Seq[String] + ): Int = { + val settings = runner + .uploadLibrary( + path, + uploadUrl.getOrElse { + throw new IllegalArgumentException( + "The default repository is currently not defined. " + + "You need to explicitly specify the `--upload-url`." + ) + }, + authToken.orElse(LauncherEnvironment.getEnvVar("ENSO_AUTH_TOKEN")), + cliOptions.hideProgress, + logLevel, + cliOptions.internalOptions.logMasking, + additionalArguments + ) + .get + + runner.withCommand(settings, JVMSettings(useSystemJVM, jvmOpts)) { + command => command.run().get + } + } + /** Prints the value of `key` from the global configuration. * * If the `key` is not set in the config, sets exit code to 1 and prints a diff --git a/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIProgressReporter.scala b/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIProgressReporter.scala index e0a302e93f1c..cf2636886783 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIProgressReporter.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIProgressReporter.scala @@ -7,14 +7,14 @@ import org.enso.launcher.InfoLogger /** A [[ProgressReporter]] that displays a progress bar in the console or waits * for the task silently, depending on CLI options. */ -class CLIProgressReporter(cliOptions: GlobalCLIOptions) - extends ProgressReporter { +class CLIProgressReporter(hideProgress: Boolean) extends ProgressReporter { /** @inheritdoc */ override def trackProgress(message: String, task: TaskProgress[_]): Unit = { InfoLogger.info(message) - if (cliOptions.hideProgress) () - else ProgressBar.waitWithProgress(task) + if (!hideProgress) { + ProgressBar.waitWithProgress(task) + } } } @@ -22,5 +22,5 @@ object CLIProgressReporter { /** A helper method to create [[CLIProgressReporter]] instances. */ def apply(globalCLIOptions: GlobalCLIOptions): CLIProgressReporter = - new CLIProgressReporter(globalCLIOptions) + new CLIProgressReporter(globalCLIOptions.hideProgress) } diff --git a/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIRuntimeVersionManagementUserInterface.scala b/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIRuntimeVersionManagementUserInterface.scala index 9ce501e1bcdc..dcff6c0a0afc 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIRuntimeVersionManagementUserInterface.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIRuntimeVersionManagementUserInterface.scala @@ -18,7 +18,7 @@ import org.enso.runtimeversionmanager.components.{ class CLIRuntimeVersionManagementUserInterface( cliOptions: GlobalCLIOptions, alwaysInstallMissing: Boolean -) extends CLIProgressReporter(cliOptions) +) extends CLIProgressReporter(hideProgress = cliOptions.hideProgress) with RuntimeVersionManagementUserInterface { private val logger = Logger[CLIRuntimeVersionManagementUserInterface] diff --git a/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala b/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala index b8c42db1fe46..aa4f8094da38 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala @@ -322,6 +322,64 @@ object LauncherApplication { } } + private def uploadLibraryCommand: Command[Config => Int] = + Command( + "publish-library", + "Publish an Enso library to a repository. " + + "If `auto-confirm` is set, this will install missing engines or " + + "runtimes without asking." + ) { + val pathOpt = Opts.optionalArgument[Path]( + "PATH", + "If PATH is provided, the project at this path or the closest one of " + + "its ancestors that contains an Enso project, is uploaded. If not, " + + "the project to upload is searched for in the current directory and " + + "its ancestors." + ) + val uploadUrlOpt = Opts.optionalParameter[String]( + "upload-url", + "URL", + "Upload URL of the repository to upload the library to.", + showInUsage = true + ) + val authTokenOpt = Opts.optionalParameter[String]( + "auth-token", + "TOKEN", + "An optional token to add to request headers for use in " + + "authorization. If this parameter is not set, the ENSO_AUTH_TOKEN " + + "environment variable is checked." + ) + val additionalArgs = Opts.additionalArguments() + ( + pathOpt, + uploadUrlOpt, + authTokenOpt, + engineLogLevel, + systemJVMOverride, + jvmOpts, + additionalArgs + ) mapN { + ( + path, + uploadUrl, + authToken, + engineLogLevel, + systemJVMOverride, + jvmOpts, + additionalArgs + ) => (config: Config) => + Launcher(config).uploadLibrary( + path = path, + uploadUrl = uploadUrl, + authToken = authToken, + logLevel = engineLogLevel, + useSystemJVM = systemJVMOverride, + jvmOpts = jvmOpts, + additionalArguments = additionalArgs + ) + } + } + private def installEngineCommand: Command[Config => Int] = Command( "engine", @@ -481,7 +539,7 @@ object LauncherApplication { private def topLevelOpts: Opts[() => TopLevelBehavior[Config]] = { val version = - Opts.flag("version", 'V', "Display version.", showInUsage = true) + Opts.flag("version", 'V', "Display version.", showInUsage = false) val json = Opts.flag( GlobalCLIOptions.USE_JSON, "Use JSON instead of plain text for version output.", @@ -614,6 +672,7 @@ object LauncherApplication { replCommand, runCommand, languageServerCommand, + uploadLibraryCommand, defaultCommand, installCommand, uninstallCommand, diff --git a/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala b/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala index b78b72a0b275..c30ea3693fc7 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala @@ -3,7 +3,9 @@ package org.enso.launcher.components import akka.http.scaladsl.model.Uri import nl.gn0s1s.bump.SemVer import org.enso.distribution.{DistributionManager, EditionManager, Environment} +import org.enso.launcher.Constants import org.enso.launcher.project.ProjectManager +import org.enso.logger.masking.MaskedPath import org.enso.loggingservice.LogLevel import org.enso.runtimeversionmanager.components.RuntimeVersionManager import org.enso.runtimeversionmanager.config.GlobalConfigurationManager @@ -191,4 +193,51 @@ class LauncherRunner( ) } } + + /** Creates [[RunSettings]] for uploading a library. + * + * See [[org.enso.launcher.Launcher.uploadLibrary]] for more details. + */ + def uploadLibrary( + path: Option[Path], + uploadUrl: String, + token: Option[String], + hideProgress: Boolean, + logLevel: LogLevel, + logMasking: Boolean, + additionalArguments: Seq[String] + ): Try[RunSettings] = + Try { + val actualPath = path.getOrElse(currentWorkingDirectory) + val project = projectManager.findProject(actualPath).get.getOrElse { + throw RunnerError( + s"Could not find a project at " + + s"${MaskedPath(actualPath).applyMasking()} or any of its parent " + + s"directories." + ) + } + + val version = resolveVersion(None, Some(project)) + if (version < Constants.uploadIntroducedVersion) { + throw RunnerError( + s"Library Upload feature is not available in Enso $version. " + + s"Please upgrade your project to a newer version." + ) + } + + val tokenOpts = token.map(Seq("--auth-token", _)).toSeq.flatten + val hideProgressOpts = + if (hideProgress) Seq("--hide-progress") else Seq.empty + + val arguments = + Seq("--upload", uploadUrl) ++ + Seq("--in-project", project.path.toAbsolutePath.normalize.toString) ++ + tokenOpts ++ hideProgressOpts + RunSettings( + version, + arguments ++ setLogLevelArgs(logLevel, logMasking) + ++ additionalArguments, + connectLoggerIfAvailable = true + ) + } } diff --git a/engine/launcher/src/test/scala/org/enso/launcher/components/UploadVersionCheck.scala b/engine/launcher/src/test/scala/org/enso/launcher/components/UploadVersionCheck.scala new file mode 100644 index 000000000000..4d54be924b9c --- /dev/null +++ b/engine/launcher/src/test/scala/org/enso/launcher/components/UploadVersionCheck.scala @@ -0,0 +1,27 @@ +package org.enso.launcher.components + +import nl.gn0s1s.bump.SemVer +import org.enso.launcher.Constants +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +class UploadVersionCheck extends AnyWordSpec with Matchers { + "Constants.uploadIntroducedVersion" should { + "correctly compare with nearby versions" in { + assert( + SemVer("0.2.17-SNAPSHOT.2021-07-23").get >= + Constants.uploadIntroducedVersion + ) + + assert( + SemVer("0.2.17").get > + Constants.uploadIntroducedVersion + ) + + assert( + SemVer("0.2.16").get < + Constants.uploadIntroducedVersion + ) + } + } +} diff --git a/engine/runner/src/main/scala/org/enso/runner/Main.scala b/engine/runner/src/main/scala/org/enso/runner/Main.scala index 51777674eb74..6875c4124b60 100644 --- a/engine/runner/src/main/scala/org/enso/runner/Main.scala +++ b/engine/runner/src/main/scala/org/enso/runner/Main.scala @@ -13,6 +13,7 @@ import org.enso.polyglot.{LanguageInfo, Module, PolyglotContext} import org.enso.version.VersionDescription import org.graalvm.polyglot.PolyglotException import java.io.File +import java.nio.file.Path import java.util.UUID import scala.Console.err @@ -44,6 +45,9 @@ object Main { private val LOG_LEVEL = "log-level" private val LOGGER_CONNECT = "logger-connect" private val NO_LOG_MASKING = "no-log-masking" + private val UPLOAD_OPTION = "upload" + private val HIDE_PROGRESS = "hide-progress" + private val AUTH_TOKEN = "auth-token" private lazy val logger = Logger[Main.type] @@ -203,6 +207,27 @@ object Main { "variable." ) .build() + val uploadOption = CliOption.builder + .hasArg(true) + .numberOfArgs(1) + .argName("url") + .longOpt(UPLOAD_OPTION) + .desc( + "Uploads the library to a repository. " + + "The url defines the repository to upload to." + ) + .build() + val hideProgressOption = CliOption.builder + .longOpt(HIDE_PROGRESS) + .desc("If specified, progress bars will not be displayed.") + .build() + val authTokenOption = CliOption.builder + .hasArg(true) + .numberOfArgs(1) + .argName("token") + .longOpt(AUTH_TOKEN) + .desc("Authentication token for the upload.") + .build() val options = new Options options @@ -228,6 +253,9 @@ object Main { .addOption(logLevelOption) .addOption(loggerConnectOption) .addOption(noLogMaskingOption) + .addOption(uploadOption) + .addOption(hideProgressOption) + .addOption(authTokenOption) options } @@ -660,6 +688,27 @@ object Main { ) } + if (line.hasOption(UPLOAD_OPTION)) { + val projectRoot = + Option(line.getOptionValue(IN_PROJECT_OPTION)) + .map(Path.of(_)) + .getOrElse { + logger.error( + s"When uploading, the $IN_PROJECT_OPTION is mandatory " + + s"to specify which project to upload." + ) + exitFail() + } + + ProjectUploader.uploadProject( + projectRoot = projectRoot, + uploadUrl = line.getOptionValue(UPLOAD_OPTION), + authToken = Option(line.getOptionValue(AUTH_TOKEN)), + showProgress = !line.hasOption(HIDE_PROGRESS) + ) + exitSuccess() + } + if (line.hasOption(RUN_OPTION)) { run( line.getOptionValue(RUN_OPTION), diff --git a/engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala b/engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala new file mode 100644 index 000000000000..6012df69a919 --- /dev/null +++ b/engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala @@ -0,0 +1,56 @@ +package org.enso.runner + +import com.typesafe.scalalogging.Logger +import org.enso.cli.ProgressBar +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.libraryupload.{auth, LibraryUploader} + +import java.nio.file.Path + +/** Gathers helper functions for uploading a library project. */ +object ProjectUploader { + + private lazy val logger = Logger[ProjectUploader.type] + + /** Uploads a project to a library repository. + * + * @param projectRoot path to the root of the project + * @param uploadUrl URL of upload endpoint of the repository to upload to + * @param authToken an optional token used for authentication in the + * repository + * @param showProgress specifies if CLI progress bars should be displayed + * showing progress of compression and upload + */ + def uploadProject( + projectRoot: Path, + uploadUrl: String, + authToken: Option[String], + showProgress: Boolean + ): Unit = { + import scala.concurrent.ExecutionContext.Implicits.global + val progressReporter = new ProgressReporter { + override def trackProgress( + message: String, + task: TaskProgress[_] + ): Unit = { + logger.info(message) + if (showProgress) { + ProgressBar.waitWithProgress(task) + } + } + } + + val token = authToken match { + case Some(value) => auth.SimpleHeaderToken(value) + case None => auth.NoAuthorization + } + LibraryUploader + .uploadLibrary( + projectRoot, + uploadUrl, + token, + progressReporter + ) + .get + } +} diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala index 5217ce9430f5..5a831c490a9c 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala @@ -51,6 +51,7 @@ object NotificationHandler { /** @inheritdoc */ override def trackProgress(message: String, task: TaskProgress[_]): Unit = { logger.info(message) + // TODO [RW] check the hideProgress flag provided by the launcher if (System.console() != null) { ProgressBar.waitWithProgress(task) } diff --git a/lib/scala/cli/src/main/scala/org/enso/cli/task/TaskProgress.scala b/lib/scala/cli/src/main/scala/org/enso/cli/task/TaskProgress.scala index a78d47d99f24..2df5e223ce63 100644 --- a/lib/scala/cli/src/main/scala/org/enso/cli/task/TaskProgress.scala +++ b/lib/scala/cli/src/main/scala/org/enso/cli/task/TaskProgress.scala @@ -1,7 +1,7 @@ package org.enso.cli.task import java.util.concurrent.LinkedTransferQueue - +import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Try} /** Represents a long-running background task. @@ -98,6 +98,21 @@ object TaskProgress { } } + /** Creates a [[TaskProgress]] from a [[Future]]. */ + def fromFuture[A]( + future: Future[A] + )(implicit ec: ExecutionContext): TaskProgress[A] = { + new TaskProgress[A] { + override def addProgressListener( + listener: ProgressListener[A] + ): Unit = { + future.onComplete { result => + listener.done(result) + } + } + } + } + /** Blocks and waits for the task to complete. */ def waitForTask[A](task: TaskProgress[A]): Try[A] = { diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/TarGzWriter.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/TarGzWriter.scala new file mode 100644 index 000000000000..059b5579246d --- /dev/null +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/TarGzWriter.scala @@ -0,0 +1,126 @@ +package org.enso.downloader.archive + +import org.apache.commons.compress.archivers.tar.{ + TarArchiveEntry, + TarArchiveOutputStream +} +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream +import org.enso.cli.task.{TaskProgress, TaskProgressImplementation} + +import java.io.{BufferedOutputStream, FileOutputStream} +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} +import scala.util.{Try, Using} + +/** A helper class for writing TAR archives compressed with gzip. */ +class TarGzWriter private (archive: TarArchiveOutputStream) { + + /** Adds a text file to the archive. + * + * @param relativePath path of the file in the archive + * @param content the text content to put in the file + */ + def writeTextFile(relativePath: String, content: String): Unit = { + val bytes = content.getBytes(StandardCharsets.UTF_8) + val entry = new TarArchiveEntry(relativePath) + entry.setSize(bytes.size.toLong) + archive.putArchiveEntry(entry) + archive.write(bytes) + archive.closeArchiveEntry() + } + + /** Adds a file from the filesystem to the archive. + * + * @param relativePath path of the file in the archive + * @param filePath a path to a file on the filesystem that will be read and + * put into the archive + * @return returns the number of bytes that were transferrred from the input + * file + */ + def writeFile(relativePath: String, filePath: Path): Long = { + val entry = new TarArchiveEntry(filePath.toFile, relativePath) + archive.putArchiveEntry(entry) + val bytesTransferred = Files.copy(filePath, archive) + archive.closeArchiveEntry() + bytesTransferred + } +} + +object TarGzWriter { + + /** Creates a .tar.gz archive at the specified destination. + * + * It calls the `actions` callback with a [[TarGzWriter]] instance which can + * be used to add files to the archive. The [[TarGzWriter]] instance is only + * valid during the call of this callback, it should not be leaked anywhere + * else as it will then be invalid. + */ + def createArchive( + destination: Path + )(actions: TarGzWriter => Unit): Try[Unit] = + Using(new FileOutputStream(destination.toFile)) { outputStream => + Using(new BufferedOutputStream(outputStream)) { bufferedStream => + Using(new GzipCompressorOutputStream(bufferedStream)) { gzipStream => + Using(new TarArchiveOutputStream(gzipStream)) { archive => + val writer = new TarGzWriter(archive) + actions(writer) + }.get + }.get + }.get + } + + /** Creates a .tar.gz archive from a list of files. + * + * @param archiveDestination path specifying where to put the archive + * @param files list of paths to files that should be compressed; these + * should be regular files, the behaviour is undefined if one of + * these paths is a directory + * @param basePath the base path to compute the relative paths of the + * compressed files; all `files` should be inside of the + * directory denoted by `basePath` + */ + def compress( + archiveDestination: Path, + files: Seq[Path], + basePath: Path + ): TaskProgress[Unit] = { + val normalizedBase = basePath.toAbsolutePath.normalize + def relativePath(file: Path): String = { + val normalized = file.toAbsolutePath.normalize + if (!normalized.startsWith(normalizedBase)) { + throw new IllegalArgumentException( + "TarGzWriter precondition failure: " + + "Files should all be inside of the provided basePath." + ) + } + normalizedBase.relativize(normalized).toString + } + + val taskProgress = new TaskProgressImplementation[Unit]() + + def runCompresion(): Unit = { + val sumSize = files.map(Files.size).sum + + val result = TarGzWriter.createArchive(archiveDestination) { writer => + var totalBytesWritten: Long = 0 + def update(): Unit = + taskProgress.reportProgress(totalBytesWritten, Some(sumSize)) + update() + for (file <- files) { + // TODO [RW] Ideally we could report progress for each chunk, offering + // more granular feedback for big data files. + val bytesWritten = writer.writeFile(relativePath(file), file) + totalBytesWritten += bytesWritten + update() + } + } + + taskProgress.setComplete(result) + } + + val thread = new Thread(() => runCompresion(), "Writing-Archive") + thread.start() + + taskProgress + } +} diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala index 497aa0ade71a..e89171b791b9 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala @@ -157,6 +157,7 @@ object HTTPDownload { "akka.library-extensions", ConfigValueFactory.fromAnyRef(Seq.empty.asJava) ) + .withValue("akka.daemonic", ConfigValueFactory.fromAnyRef("on")) .withValue("akka.loggers", ConfigValueFactory.fromAnyRef(loggers)) .withValue( "akka.logging-filter", diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala index f525d20da772..7757d0f6fa05 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala @@ -11,13 +11,16 @@ import org.enso.downloader.http */ case class HTTPRequestBuilder private ( uri: Uri, - headers: Vector[(String, String)] + headers: Vector[(String, String)], + httpEntity: RequestEntity ) { - /** Builds a GET request with the specified settings. - */ + /** Builds a GET request with the specified settings. */ def GET: HTTPRequest = build(HttpMethods.GET) + /** Builds a POST request with the specified settings. */ + def POST: HTTPRequest = build(HttpMethods.POST) + /** Adds an additional header that will be included in the request. * * @param name name of the header @@ -26,6 +29,14 @@ case class HTTPRequestBuilder private ( def addHeader(name: String, value: String): HTTPRequestBuilder = copy(headers = headers.appended((name, value))) + /** Sets the [[RequestEntity]] for the request. + * + * It can be used for example to specify form data to send for a POST + * request. + */ + def setEntity(entity: RequestEntity): HTTPRequestBuilder = + copy(httpEntity = entity) + private def build( method: HttpMethod ): HTTPRequest = { @@ -41,7 +52,12 @@ case class HTTPRequestBuilder private ( } } http.HTTPRequest( - HttpRequest(method = method, uri = uri, headers = httpHeaders) + HttpRequest( + method = method, + uri = uri, + headers = httpHeaders, + entity = httpEntity + ) ) } } @@ -51,7 +67,7 @@ object HTTPRequestBuilder { /** Creates a request builder that will send the request for the given URI. */ def fromURI(uri: Uri): HTTPRequestBuilder = - new HTTPRequestBuilder(uri, Vector.empty) + new HTTPRequestBuilder(uri, Vector.empty, HttpEntity.Empty) /** Tries to parse the URI provided as a [[String]] and returns a request * builder that will send the request to the given `uri`. diff --git a/lib/scala/editions/src/main/scala/org/enso/yaml/YamlHelper.scala b/lib/scala/editions/src/main/scala/org/enso/yaml/YamlHelper.scala index b7e82fa3ed96..56b1890ad2fd 100644 --- a/lib/scala/editions/src/main/scala/org/enso/yaml/YamlHelper.scala +++ b/lib/scala/editions/src/main/scala/org/enso/yaml/YamlHelper.scala @@ -1,6 +1,11 @@ package org.enso.yaml -import io.circe.{yaml, Decoder} +import io.circe.yaml.Printer +import io.circe.{yaml, Decoder, Encoder} + +import java.io.FileReader +import java.nio.file.Path +import scala.util.{Try, Using} /** A helper for parsing YAML configs. */ object YamlHelper { @@ -14,4 +19,17 @@ object YamlHelper { .flatMap(_.as[R]) .left .map(ParseError(_)) + + /** Tries to load and parse a YAML file at the provided path. */ + def load[R](path: Path)(implicit decoder: Decoder[R]): Try[R] = + Using(new FileReader(path.toFile)) { reader => + yaml.parser + .parse(reader) + .flatMap(_.as[R]) + .toTry + }.flatten + + /** Saves a YAML representation of an object into a string. */ + def toYaml[A](obj: A)(implicit encoder: Encoder[A]): String = + Printer.spaces2.copy(preserveOrder = true).pretty(encoder(obj)) } diff --git a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DownloaderTest.scala b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DownloaderTest.scala new file mode 100644 index 000000000000..252dd46b1662 --- /dev/null +++ b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DownloaderTest.scala @@ -0,0 +1,49 @@ +package org.enso.librarymanager.published.repository + +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.distribution.TemporaryDirectoryManager +import org.enso.distribution.locking.{ + LockUserInterface, + Resource, + ResourceManager, + ThreadSafeFileLockManager +} +import org.enso.librarymanager.published.cache.DownloadingLibraryCache +import org.enso.testkit.HasTestDirectory + +trait DownloaderTest { self: HasTestDirectory => + def withDownloader[R](action: DownloadingLibraryCache => R): R = { + val lockManager = + new ThreadSafeFileLockManager(getTestDirectory.resolve("locks")) + val resourceManager = new ResourceManager(lockManager) + try { + val cache = new DownloadingLibraryCache( + cacheRoot = getTestDirectory.resolve("cache"), + temporaryDirectoryManager = new TemporaryDirectoryManager( + getTestDirectory.resolve("tmp"), + resourceManager + ), + resourceManager = resourceManager, + lockUserInterface = new LockUserInterface { + override def startWaitingForResource(resource: Resource): Unit = + println(s"Waiting for ${resource.name}") + + override def finishWaitingForResource(resource: Resource): Unit = + println(s"${resource.name} is ready") + }, + progressReporter = new ProgressReporter { + override def trackProgress( + message: String, + task: TaskProgress[_] + ): Unit = {} + } + ) + + action(cache) + } finally { + resourceManager.releaseMainLock() + resourceManager.unlockTemporaryDirectory() + } + } + +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala similarity index 88% rename from lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala rename to lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index 3c163855ec8a..6c99ad5a33b9 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -3,6 +3,7 @@ package org.enso.librarymanager.published.repository import nl.gn0s1s.bump.SemVer import org.enso.cli.OS import org.enso.distribution.FileSystem +import org.enso.downloader.archive.TarGzWriter import org.enso.editions.Editions.RawEdition import org.enso.editions.{Editions, LibraryName} import org.enso.pkg.{Package, PackageManager} @@ -48,10 +49,13 @@ abstract class DummyRepository { .resolve(lib.version.toString) Files.createDirectories(libraryRoot) createLibraryProject(libraryRoot, lib) - val files = Seq( - ArchiveWriter.TextFile("src/Main.enso", lib.mainContent) - ) - ArchiveWriter.writeTarArchive(libraryRoot.resolve("main.tgz"), files) + + TarGzWriter + .createArchive(libraryRoot.resolve("main.tgz")) { writer => + writer.writeTextFile("src/Main.enso", lib.mainContent) + } + .get + createManifest(libraryRoot) } } @@ -106,12 +110,18 @@ abstract class DummyRepository { * @param port port to listen on * @param root root of the library repository, the same as the argument to * [[createRepository]] + * @param uploads specifies whether to enable uploads in the server */ - def startServer(port: Int, root: Path): WrappedProcess = { + def startServer( + port: Int, + root: Path, + uploads: Boolean = false + ): WrappedProcess = { val serverDirectory = Path.of("tools/simple-library-server").toAbsolutePath.normalize - val preinstallCommand = commandPrefix ++ Seq(npmCommand, "install") + val preinstallCommand = + commandPrefix ++ Seq(npmCommand, "install") val preinstallExitCode = new ProcessBuilder() .command(preinstallCommand: _*) .directory(serverDirectory.toFile) @@ -125,6 +135,7 @@ abstract class DummyRepository { s"npm exited with code $preinstallCommand." ) + val uploadsArgs = if (uploads) Seq("--upload", "no-auth") else Seq() val command = commandPrefix ++ Seq( nodeCommand, "main.js", @@ -132,7 +143,7 @@ abstract class DummyRepository { port.toString, "--root", root.toAbsolutePath.normalize.toString - ) + ) ++ uploadsArgs val rawProcess = (new ProcessBuilder) .command(command: _*) .directory(serverDirectory.toFile) diff --git a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/EmptyRepository.scala b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/EmptyRepository.scala new file mode 100644 index 000000000000..1314df4eedd6 --- /dev/null +++ b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/EmptyRepository.scala @@ -0,0 +1,5 @@ +package org.enso.librarymanager.published.repository + +object EmptyRepository extends DummyRepository { + override def libraries: Seq[EmptyRepository.DummyLibrary] = Seq.empty +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala similarity index 100% rename from lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala rename to lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager-test/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala similarity index 58% rename from lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala rename to lib/scala/library-manager-test/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala index 882516116dab..4c47ad016e11 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala +++ b/lib/scala/library-manager-test/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -1,17 +1,8 @@ package org.enso.librarymanager.published.repository -import org.enso.cli.task.{ProgressReporter, TaskProgress} -import org.enso.distribution.TemporaryDirectoryManager -import org.enso.distribution.locking.{ - LockUserInterface, - Resource, - ResourceManager, - ThreadSafeFileLockManager -} import org.enso.editions.Editions -import org.enso.librarymanager.published.cache.DownloadingLibraryCache -import org.enso.loggingservice.TestLogger.TestLogMessage import org.enso.loggingservice.{LogLevel, TestLogger} +import org.enso.loggingservice.TestLogger.TestLogMessage import org.enso.pkg.PackageManager import org.enso.testkit.WithTemporaryDirectory import org.scalatest.matchers.should.Matchers @@ -22,7 +13,8 @@ import java.nio.file.Files class LibraryDownloadTest extends AnyWordSpec with Matchers - with WithTemporaryDirectory { + with WithTemporaryDirectory + with DownloaderTest { val port: Int = 47306 @@ -32,32 +24,7 @@ class LibraryDownloadTest val repoRoot = getTestDirectory.resolve("repo") repo.createRepository(repoRoot) - val lockManager = - new ThreadSafeFileLockManager(getTestDirectory.resolve("locks")) - val resourceManager = new ResourceManager(lockManager) - try { - val cache = new DownloadingLibraryCache( - cacheRoot = getTestDirectory.resolve("cache"), - temporaryDirectoryManager = new TemporaryDirectoryManager( - getTestDirectory.resolve("tmp"), - resourceManager - ), - resourceManager = resourceManager, - lockUserInterface = new LockUserInterface { - override def startWaitingForResource(resource: Resource): Unit = - println(s"Waiting for ${resource.name}") - - override def finishWaitingForResource(resource: Resource): Unit = - println(s"${resource.name} is ready") - }, - progressReporter = new ProgressReporter { - override def trackProgress( - message: String, - task: TaskProgress[_] - ): Unit = {} - } - ) - + withDownloader { cache => val server = repo.startServer(port, repoRoot) try { cache.findCachedLibrary( @@ -95,9 +62,6 @@ class LibraryDownloadTest server.kill(killDescendants = true) server.join(waitForDescendants = true) } - } finally { - resourceManager.releaseMainLock() - resourceManager.unlockTemporaryDirectory() } } } diff --git a/lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala b/lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala new file mode 100644 index 000000000000..abf698dc0fdd --- /dev/null +++ b/lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala @@ -0,0 +1,92 @@ +package org.enso.libraryupload + +import nl.gn0s1s.bump.SemVer +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.editions.{Editions, LibraryName} +import org.enso.librarymanager.published.repository.{ + DownloaderTest, + EmptyRepository +} +import org.enso.libraryupload.auth.SimpleHeaderToken +import org.enso.pkg.PackageManager +import org.enso.testkit.WithTemporaryDirectory +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import java.nio.file.Files + +class LibraryUploadTest + extends AnyWordSpec + with Matchers + with WithTemporaryDirectory + with DownloaderTest { + + def port: Int = 47305 + + "LibraryUploader" should { + "upload the files to the server" in { + val projectRoot = getTestDirectory.resolve("lib_root") + val repoRoot = getTestDirectory.resolve("repo") + + val libraryName = LibraryName("tester", "Upload_Test") + val libraryVersion = SemVer(1, 2, 3) + PackageManager.Default.create( + projectRoot.toFile, + name = libraryName.name, + namespace = libraryName.namespace, + version = libraryVersion.toString + ) + + val server = EmptyRepository.startServer(port, repoRoot, uploads = true) + try { + val uploadUrl = s"http://localhost:$port/upload" + val token = SimpleHeaderToken("TODO") + import scala.concurrent.ExecutionContext.Implicits.global + LibraryUploader + .uploadLibrary( + projectRoot, + uploadUrl, + token, + new ProgressReporter { + override def trackProgress(message: String, task: TaskProgress[_]) + : Unit = () + } + ) + .get + + val libRoot = repoRoot + .resolve("libraries") + .resolve("tester") + .resolve("Upload_Test") + .resolve("1.2.3") + + PackageManager.Default + .loadPackage(libRoot.toFile) + .get + .name shouldEqual libraryName.name + assert(Files.exists(libRoot.resolve("manifest.yaml"))) + assert(Files.exists(libRoot.resolve("main.tgz"))) + + withDownloader { cache => + cache.findCachedLibrary(libraryName, libraryVersion) shouldBe empty + + val repo = Editions.Repository( + "test_repo", + s"http://localhost:$port/libraries" + ) + val installedRoot = + cache.findOrInstallLibrary(libraryName, libraryVersion, repo).get + val pkg = PackageManager.Default.loadPackage(installedRoot.toFile).get + pkg.name shouldEqual libraryName.name + val sources = pkg.listSources + sources should have size 1 + sources.head.file.getName shouldEqual "Main.enso" + } + + } finally { + server.kill(killDescendants = true) + server.join(waitForDescendants = true) + } + } + } +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala index af2cf5229db2..60add2710a24 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala @@ -1,6 +1,7 @@ package org.enso.librarymanager.published.repository -import io.circe.Decoder +import io.circe.syntax.EncoderOps +import io.circe.{Decoder, Encoder, Json} import org.enso.editions.LibraryName /** The manifest file containing metadata related to a published library. @@ -19,6 +20,17 @@ case class LibraryManifest( ) object LibraryManifest { + + /** Creates an empty manifest. + * + * Such a manifest is invalid as at least one archive should be specified in + * a valid manifest. + * + * It can however be useful as a temporary value for logic that updates or + * creates a new manifest. + */ + def empty: LibraryManifest = LibraryManifest(Seq.empty, Seq.empty, None, None) + object Fields { val archives = "archives" val dependencies = "dependencies" @@ -43,6 +55,20 @@ object LibraryManifest { ) } + /** An [[Encoder]] instance for parsing [[LibraryManifest]]. */ + implicit val encoder: Encoder[LibraryManifest] = { manifest => + val baseFields = Seq( + Fields.archives -> manifest.archives.asJson, + Fields.dependencies -> manifest.dependencies.asJson + ) + + val allFields = baseFields ++ + manifest.tagLine.map(Fields.tagLine -> _.asJson).toSeq ++ + manifest.description.map(Fields.description -> _.asJson).toSeq + + Json.obj(allFields: _*) + } + /** The name of the manifest file as included in the directory associated with * a given library in the library repository. */ diff --git a/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala b/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala new file mode 100644 index 000000000000..ea3157d32338 --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala @@ -0,0 +1,233 @@ +package org.enso.libraryupload + +import akka.http.scaladsl.marshalling.Marshal +import akka.http.scaladsl.model._ +import akka.stream.scaladsl.Source +import com.typesafe.scalalogging.Logger +import nl.gn0s1s.bump.SemVer +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.distribution.FileSystem +import org.enso.distribution.FileSystem.PathSyntax +import org.enso.downloader.archive.TarGzWriter +import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} +import org.enso.editions.LibraryName +import org.enso.librarymanager.published.repository.LibraryManifest +import org.enso.pkg.{Package, PackageManager} +import org.enso.yaml.YamlHelper + +import java.nio.file.{Files, Path} +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ +import scala.util.{Failure, Success, Try, Using} + +/** Gathers functions used for uploading libraries. */ +object LibraryUploader { + private lazy val logger = Logger[LibraryUploader.type] + + /** Uploads a library to a repository. + * + * @param projectRoot path to the library project root + * @param uploadUrl an URL to the upload endpoint of a library repository + * @param authToken a token describing the authentication method to use with + * the repository + * @param progressReporter a [[ProgressReporter]] to track long running tasks + * like compression and upload + * @param ec an execution context used for handling Futures + */ + def uploadLibrary( + projectRoot: Path, + uploadUrl: String, + authToken: auth.Token, + progressReporter: ProgressReporter + )(implicit ec: ExecutionContext): Try[Unit] = Try { + FileSystem.withTemporaryDirectory("enso-upload") { tmpDir => + val pkg = PackageManager.Default.loadPackage(projectRoot.toFile).get + val version = SemVer(pkg.config.version).getOrElse { + throw new IllegalStateException( + s"Project version [${pkg.config.version}] is not a valid semver " + + s"string." + ) + } + val uri = buildUploadUri(uploadUrl, pkg.libraryName, version) + + val mainArchiveName = "main.tgz" + val filesToIgnoreInArchive = Seq( + Package.configFileName, + LibraryManifest.filename + ) + val archivePath = tmpDir / mainArchiveName + val compressing = + createMainArchive(projectRoot, filesToIgnoreInArchive, archivePath) + progressReporter.trackProgress( + s"Creating the [$mainArchiveName] archive.", + compressing + ) + compressing.force() + + val manifestPath = projectRoot / LibraryManifest.filename + val loadedManifest = + loadSavedManifest(manifestPath).getOrElse(LibraryManifest.empty) + val updatedManifest = + // TODO [RW] update dependencies in the manifest (#1773) + loadedManifest.copy(archives = Seq(mainArchiveName)) + FileSystem.writeTextFile(manifestPath, YamlHelper.toYaml(updatedManifest)) + + logger.info(s"Uploading library package to the server at [$uploadUrl].") + val upload = uploadFiles( + uri, + authToken, + files = Seq( + projectRoot / Package.configFileName, + projectRoot / LibraryManifest.filename, + archivePath + ) + ) + progressReporter.trackProgress( + s"Uploading packages to [$uploadUrl].", + upload + ) + upload.force() + + logger.info(s"Upload complete.") + } + } + + /** Creates an URL for the upload, including information identifying the + * library version. + */ + private def buildUploadUri( + baseUploadUrl: String, + libraryName: LibraryName, + version: SemVer + ): Uri = { + URIBuilder + .fromUri(baseUploadUrl) + .addQuery("namespace", libraryName.namespace) + .addQuery("name", libraryName.name) + .addQuery("version", version.toString) + .build() + } + + /** Gathers project files to create the main archive. + * + * For now it just filters out the files like manifest which are uploaded + * separately. In the future this may be extended to create separate + * sub-archives for platform specific binaries or tests. + * + * @param projectRoot path to the project root + * @param rootFilesToIgnore names of files at the root that should *not* be + * included in the archive + * @param destination path at which the archive is created + * @return + */ + private def createMainArchive( + projectRoot: Path, + rootFilesToIgnore: Seq[String], + destination: Path + ): TaskProgress[Unit] = { + def relativePath(file: Path): String = projectRoot.relativize(file).toString + def shouldBeUploaded(file: Path): Boolean = { + def isIgnored = rootFilesToIgnore.contains(relativePath(file)) + Files.isRegularFile(file) && !isIgnored + } + + logger.trace("Gathering files to compress.") + val filesToCompress = Using(Files.walk(projectRoot)) { filesStream => + filesStream.iterator().asScala.filter(shouldBeUploaded).toSeq + }.get + + logger.info( + s"Compressing ${filesToCompress.size} project files " + + s"into [${destination.getFileName}]." + ) + + val compression = TarGzWriter.compress( + archiveDestination = destination, + files = filesToCompress, + basePath = projectRoot + ) + + compression.map { _ => + logger.info(s"Archive [${destination.getFileName}] created.") + } + } + + /** Creates a [[RequestEntity]] that will upload the provided files. */ + private def createRequestEntity( + files: Seq[Path] + )(implicit ec: ExecutionContext): Future[RequestEntity] = { + + val fileBodies = files.map { path => + val filename = path.getFileName.toString + Multipart.FormData.BodyPart( + filename, + HttpEntity.fromPath(detectContentType(path), path), + Map("filename" -> filename) + ) + } + + val formData = Multipart.FormData(Source(fileBodies)) + Marshal(formData).to[RequestEntity] + } + + /** Loads a manifest, if it exists. */ + private def loadSavedManifest(manifestPath: Path): Option[LibraryManifest] = { + if (Files.exists(manifestPath)) { + val loaded = YamlHelper.load[LibraryManifest](manifestPath).get + Some(loaded) + } else None + } + + /** Tries to detect the content type of the file to upload. + * + * If it is not a known type, it falls back to `application/octet-stream`. + */ + private def detectContentType(path: Path): ContentType = { + val filename = path.getFileName.toString + if (filename.endsWith(".tgz") || filename.endsWith(".tar.gz")) + ContentType(MediaTypes.`application/x-gtar`) + else if (filename.endsWith(".yaml") || filename.endsWith(".enso")) + ContentTypes.`text/plain(UTF-8)` + else ContentTypes.`application/octet-stream` + } + + /** Uploads the provided files to the provided url, using the provided token + * for authentication. + */ + private def uploadFiles( + uri: Uri, + authToken: auth.Token, + files: Seq[Path] + )(implicit ec: ExecutionContext): TaskProgress[Unit] = { + val future = createRequestEntity(files).map { entity => + val request = authToken + .alterRequest(HTTPRequestBuilder.fromURI(uri)) + .setEntity(entity) + .POST + // TODO [RW] upload progress + HTTPDownload.fetchString(request).force() + } + TaskProgress.fromFuture(future).flatMap { response => + if (response.statusCode == 200) { + logger.debug("Server responded with 200 OK.") + Success(()) + } else { + // TODO [RW] we may want to have more precise error messages to handle auth errors etc. (#1773) + val includedMessage = for { + json <- io.circe.parser.parse(response.content).toOption + obj <- json.asObject + message <- obj("error").flatMap(_.asString) + } yield message + val message = includedMessage.getOrElse("Unknown error") + val errorMessage = + s"Upload failed: $message (Status code: ${response.statusCode})." + logger.error(errorMessage) + Failure( + new RuntimeException( + errorMessage + ) + ) + } + } + } +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/auth/Token.scala b/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/auth/Token.scala new file mode 100644 index 000000000000..f82b47fdfae9 --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/auth/Token.scala @@ -0,0 +1,35 @@ +package org.enso.libraryupload.auth + +import org.enso.downloader.http.HTTPRequestBuilder + +/** Represents an authentication method that can be used to authenticate + * requests to the library repository. + */ +trait Token { + + /** Alters the request adding any properties (like headers) necessary to + * successfully authenticate. + */ + def alterRequest(request: HTTPRequestBuilder): HTTPRequestBuilder +} + +/** A simple authentication method that adds an `Auth-Token` header to the + * request. + */ +case class SimpleHeaderToken(value: String) extends Token { + + /** @inheritdoc */ + override def alterRequest(request: HTTPRequestBuilder): HTTPRequestBuilder = + request.addHeader("Auth-Token", value) +} + +/** A dummy authentication method that does not do anything. + * + * It can be used for servers that do not require any authentication. + */ +case object NoAuthorization extends Token { + + /** @inheritdoc */ + override def alterRequest(request: HTTPRequestBuilder): HTTPRequestBuilder = + request +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala deleted file mode 100644 index 023f1b89a8e2..000000000000 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala +++ /dev/null @@ -1,50 +0,0 @@ -package org.enso.librarymanager.published.repository - -import org.apache.commons.compress.archivers.tar.{ - TarArchiveEntry, - TarArchiveOutputStream -} -import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream - -import java.io.{BufferedOutputStream, FileOutputStream} -import java.nio.file.Path -import scala.util.Using - -/** A helper class used for creating TAR-GZ archives in tests. */ -object ArchiveWriter { - - /** A file to add to the archive. */ - sealed trait FileToWrite { - - /** The path that this file should have within the archive. */ - def relativePath: String - } - - /** Represents a text file to be added to a test archive. - * - * @param relativePath the path in the archive - * @param content the text contents for the file - */ - case class TextFile(relativePath: String, content: String) extends FileToWrite - - /** Creates a tar archive at the given path, containing the provided files. */ - def writeTarArchive(path: Path, files: Seq[FileToWrite]): Unit = { - Using(new FileOutputStream(path.toFile)) { outputStream => - Using(new BufferedOutputStream(outputStream)) { bufferedStream => - Using(new GzipCompressorOutputStream(bufferedStream)) { gzipStream => - Using(new TarArchiveOutputStream(gzipStream)) { archive => - for (file <- files) { - file match { - case TextFile(relativePath, content) => - val entry = new TarArchiveEntry(relativePath) - archive.putArchiveEntry(entry) - archive.write(content.getBytes) - archive.closeArchiveEntry() - } - } - } - } - } - } - } -} diff --git a/tools/simple-library-server/main.js b/tools/simple-library-server/main.js index d33ce4079d0d..39d056609de6 100755 --- a/tools/simple-library-server/main.js +++ b/tools/simple-library-server/main.js @@ -1,7 +1,13 @@ #!/usr/bin/env node const express = require("express"); +const path = require("path"); +const os = require("os"); +const fs = require("fs"); +const fsPromises = require("fs/promises"); +const multer = require("multer"); const compression = require("compression"); const yargs = require("yargs"); +const semverValid = require("semver/functions/valid"); const argv = yargs .usage( @@ -19,11 +25,44 @@ const argv = yargs type: "string", default: ".", }) + .option("upload", { + description: + "Specifies whether to allow uploading libraries and which authentication model to choose.", + choices: ["disabled", "no-auth", "constant-token"], + default: "disabled", + }) .help() .alias("help", "h").argv; +const libraryRoot = path.join(argv.root, "libraries"); + const app = express(); +const tmpDir = path.join(os.tmpdir(), "enso-library-repo-uploads"); +const upload = multer({ dest: tmpDir }); app.use(compression({ filter: shouldCompress })); + +/** The token to compare against for simple authentication. + * + * If it is not set, no authentication checks are made. + */ +let token = null; +if (argv.upload == "disabled") { + console.log("Uploads are disabled."); +} else { + app.post("/upload", upload.any(), handleUpload); + + if (argv.upload == "constant-token") { + const envVar = "ENSO_AUTH_TOKEN"; + token = process.env[envVar]; + if (!token) { + throw `${envVar} is not defined.`; + } else { + console.log(`Checking the ${envVar} to authorize requests.`); + } + } else { + console.log("WARNING: Uploads are enabled without any authentication."); + } +} app.use(express.static(argv.root)); console.log( @@ -32,6 +71,7 @@ console.log( app.listen(argv.port); +/// Specifies if a particular file can be compressed in transfer, if supported. function shouldCompress(req, res) { if (req.path.endsWith(".yaml")) { return true; @@ -39,3 +79,121 @@ function shouldCompress(req, res) { return compression.filter(req, res); } + +/** Handles upload of a library. */ +async function handleUpload(req, res) { + function fail(code, message) { + res.status(code).json({ error: message }); + cleanFiles(req.files); + } + + if (token !== null) { + const userToken = req.get("Auth-Token"); + if (userToken != token) { + return fail(403, "Authorization failed."); + } + } + + const version = req.query.version; + const namespace = req.query.namespace; + const name = req.query.name; + + if (version === undefined || namespace == undefined || name === undefined) { + return fail(400, "One or more required fields were missing."); + } + + if (!isVersionValid(version)) { + return fail(400, `Invalid semver version string [${version}].`); + } + + if (!isNamespaceValid(namespace)) { + return fail(400, `Invalid username [${namespace}].`); + } + + if (!isNameValid(name)) { + return fail(400, `Invalid library name [${name}].`); + } + + for (var i = 0; i < req.files.length; ++i) { + const filename = req.files[i].originalname; + if (!isFilenameValid(filename)) { + return fail(400, `Invalid filename: ${filename}.`); + } + } + + const libraryPath = path.join(libraryRoot, namespace, name, version); + + if (fs.existsSync(libraryPath)) { + return fail( + 409, + "A library with the given name and version " + + "combination already exists. Versions are immutable, so you must " + + "bump the library version when uploading a newer version." + ); + } + + await fsPromises.mkdir(libraryPath, { recursive: true }); + + console.log(`Uploading library [${namespace}.${name}:${version}].`); + try { + await putFiles(libraryPath, req.files); + } catch (error) { + console.log(`Upload failed: [${error}].`); + console.error(error.stack); + return fail(500, "Upload failed due to an internal error."); + } + + console.log("Upload complete."); + res.status(200).json({ message: "Successfully uploaded the library." }); +} + +/// Checks if a version complies with the semver specification. +function isVersionValid(version) { + return semverValid(version) !== null; +} + +/// Checks if the namespace/username is valid. +function isNamespaceValid(namespace) { + return /^[a-z][a-z0-9]*$/.test(namespace) && namespace.length >= 3; +} + +/** Checks if the library name is valid. + * + * It may actually accept more identifiers as valid than Enso would, the actual + * check should be done when creating the library. This is just a sanity check + * for safety. + */ +function isNameValid(name) { + return /^[A-Za-z0-9_]+$/.test(name); +} + +// TODO [RW] for now slashes are not permitted to avoid attacks; later on at least the `meta` directory should be allowed, but not much besides that +/// Checks if the uploaded filename is valid. +function isFilenameValid(name) { + return /^[A-Za-z0-9][A-Za-z0-9\._\-]*$/.test(name); +} + +/// Schedules to remove the files, if they still exist. +function cleanFiles(files) { + files.forEach((file) => { + if (fs.existsSync(file.path)) { + fs.unlink(file.path, (err) => { + if (err) { + console.error( + `Failed to remove ${file.path} ($file.originalname) from a failed upload: ${err}.` + ); + } + }); + } + }); +} + +/// Moves the files to the provided destination directory. +async function putFiles(directory, files) { + for (var i = 0; i < files.length; ++i) { + const file = files[i]; + const filename = file.originalname; + const destination = path.join(directory, filename); + await fsPromises.rename(file.path, destination); + } +} diff --git a/tools/simple-library-server/package.json b/tools/simple-library-server/package.json index 14a14e70cdc0..77d9c71af66b 100644 --- a/tools/simple-library-server/package.json +++ b/tools/simple-library-server/package.json @@ -16,6 +16,8 @@ "dependencies": { "compression": "^1.7.4", "express": "^4.17.1", + "multer": "^1.4.2", + "semver": "^7.3.5", "yargs": "^17.0.1" } }