From 810cebea7b6dc48f8934b230fe2614c1baa87f70 Mon Sep 17 00:00:00 2001 From: Ara Adkins Date: Mon, 26 Jul 2021 10:32:19 +0100 Subject: [PATCH] Revert "Library Publishing MVP (#1898)" This reverts commit 1bd4e5824ec6f2777718fd6ad2287ad7a054c15f. --- .github/workflows/scala.yml | 2 +- RELEASES.md | 4 - build.sbt | 48 ++-- .../protocol-language-server.md | 27 -- docs/libraries/repositories.md | 25 -- docs/libraries/sharing.md | 16 +- .../languageserver/libraries/LibraryApi.scala | 13 - .../libraries/LocalLibraryManager.scala | 34 +-- .../LocalLibraryManagerProtocol.scala | 13 +- .../handler/LibraryPublishHandler.scala | 153 +----------- .../json/JsonConnectionController.scala | 3 +- .../websocket/json/LibrariesTest.scala | 88 +------ .../scala/org/enso/launcher/Constants.scala | 14 -- .../scala/org/enso/launcher/Launcher.scala | 46 ---- .../launcher/cli/CLIProgressReporter.scala | 10 +- ...untimeVersionManagementUserInterface.scala | 2 +- .../launcher/cli/LauncherApplication.scala | 61 +---- .../launcher/components/LauncherRunner.scala | 49 ---- .../components/UploadVersionCheck.scala | 27 -- .../src/main/scala/org/enso/runner/Main.scala | 49 ---- .../org/enso/runner/ProjectUploader.scala | 56 ----- .../instrument/NotificationHandler.scala | 1 - .../org/enso/cli/task/TaskProgress.scala | 17 +- .../enso/downloader/archive/TarGzWriter.scala | 126 ---------- .../enso/downloader/http/HTTPDownload.scala | 1 - .../downloader/http/HTTPRequestBuilder.scala | 26 +- .../main/scala/org/enso/yaml/YamlHelper.scala | 20 +- .../published/repository/DownloaderTest.scala | 49 ---- .../repository/EmptyRepository.scala | 5 - .../libraryupload/LibraryUploadTest.scala | 92 ------- .../repository/LibraryManifest.scala | 28 +-- .../enso/libraryupload/LibraryUploader.scala | 233 ------------------ .../org/enso/libraryupload/auth/Token.scala | 35 --- .../published/repository/ArchiveWriter.scala | 50 ++++ .../repository/DummyRepository.scala | 25 +- .../repository/ExampleRepository.scala | 0 .../repository/LibraryDownloadTest.scala | 44 +++- tools/simple-library-server/main.js | 158 ------------ tools/simple-library-server/package.json | 2 - 39 files changed, 158 insertions(+), 1494 deletions(-) delete mode 100644 engine/launcher/src/main/scala/org/enso/launcher/Constants.scala delete mode 100644 engine/launcher/src/test/scala/org/enso/launcher/components/UploadVersionCheck.scala delete mode 100644 engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala delete mode 100644 lib/scala/downloader/src/main/scala/org/enso/downloader/archive/TarGzWriter.scala delete mode 100644 lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DownloaderTest.scala delete mode 100644 lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/EmptyRepository.scala delete mode 100644 lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala delete mode 100644 lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala delete mode 100644 lib/scala/library-manager/src/main/scala/org/enso/libraryupload/auth/Token.scala create mode 100644 lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala rename lib/scala/{library-manager-test/src/main => library-manager/src/test}/scala/org/enso/librarymanager/published/repository/DummyRepository.scala (88%) rename lib/scala/{library-manager-test/src/main => library-manager/src/test}/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala (100%) rename lib/scala/{library-manager-test => library-manager}/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala (58%) diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index 6302bb82710c..01d7ad50a4c4 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -16,7 +16,7 @@ env: # Please ensure that this is in sync with rustVersion in build.sbt rustToolchain: nightly-2021-05-12 # Some moderately recent version of Node.JS is needed to run the library download tests. - nodeVersion: 14.17.2 + nodeVersion: 12.18.4 jobs: test_and_publish: diff --git a/RELEASES.md b/RELEASES.md index 39592dffcaa1..7aa7a01003b8 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -12,10 +12,6 @@ - Implemented a basic library downloader ([#1885](https://github.com/enso-org/enso/pull/1885)), allowing the downloading of missing libraries. -- Implemented a basic library uploader - ([#1898](https://github.com/enso-org/enso/pull/1898)). It implements the - `library/publish` endpoint of the Language Server and adds a `publish-library` - subcommand to the Launcher. ## Libraries diff --git a/build.sbt b/build.sbt index 76295af94475..7a32aff26be7 100644 --- a/build.sbt +++ b/build.sbt @@ -251,7 +251,6 @@ lazy val enso = (project in file(".")) `distribution-manager`, `edition-updater`, `library-manager`, - `library-manager-test`, syntax.jvm, testkit ) @@ -1024,7 +1023,6 @@ lazy val `language-server` = (project in file("engine/language-server")) .dependsOn(`version-output`) .dependsOn(pkg) .dependsOn(testkit % Test) - .dependsOn(`library-manager-test` % Test) .dependsOn(`runtime-version-manager-test` % Test) lazy val ast = (project in file("lib/scala/ast")) @@ -1180,6 +1178,22 @@ lazy val runtime = (project in file("engine/runtime")) case _ => MergeStrategy.first } ) + .settings( + (Compile / compile) := (Compile / compile) + .dependsOn( + Def.task { + Editions.writeEditionConfig( + ensoVersion = ensoVersion, + editionName = currentEdition, + libraryVersion = + "0.1.0", // TODO [RW] Once we start releasing the standard libraries, this will be synced with engine version. + log = streams.value.log + ) + } + ) + .value, + cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions" + ) .dependsOn(pkg) .dependsOn(`interpreter-dsl`) .dependsOn(syntax.jvm) @@ -1256,7 +1270,6 @@ lazy val `engine-runner` = project ) .dependsOn(`version-output`) .dependsOn(pkg) - .dependsOn(cli) .dependsOn(`library-manager`) .dependsOn(`language-server`) .dependsOn(`polyglot-api`) @@ -1346,22 +1359,6 @@ lazy val editions = project "org.scalatest" %% "scalatest" % scalatestVersion % Test ) ) - .settings( - (Compile / compile) := (Compile / compile) - .dependsOn( - Def.task { - Editions.writeEditionConfig( - ensoVersion = ensoVersion, - editionName = currentEdition, - libraryVersion = - "0.1.0", // TODO [RW] Once we start releasing the standard libraries, this will be synced with engine version. - log = streams.value.log - ) - } - ) - .value, - cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions" - ) .dependsOn(testkit % Test) lazy val downloader = (project in file("lib/scala/downloader")) @@ -1409,19 +1406,6 @@ lazy val `library-manager` = project .dependsOn(testkit % Test) .dependsOn(`logging-service` % Test) -lazy val `library-manager-test` = project - .in(file("lib/scala/library-manager-test")) - .configs(Test) - .settings( - libraryDependencies ++= Seq( - "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, - "org.scalatest" %% "scalatest" % scalatestVersion % Test - ) - ) - .dependsOn(`library-manager`) - .dependsOn(testkit) - .dependsOn(`logging-service`) - lazy val `runtime-version-manager` = project .in(file("lib/scala/runtime-version-manager")) .configs(Test) diff --git a/docs/language-server/protocol-language-server.md b/docs/language-server/protocol-language-server.md index 12ffa1685cae..42091a364bf8 100644 --- a/docs/language-server/protocol-language-server.md +++ b/docs/language-server/protocol-language-server.md @@ -200,7 +200,6 @@ transport formats, please look [here](./protocol-architecture). - [`LibraryDownloadError`](#librarydownloaderror) - [`LocalLibraryNotFound`](#locallibrarynotfound) - [`LibraryNotResolved`](#librarynotresolved) - - [`InvalidLibraryName`](#invalidlibraryname) @@ -4329,8 +4328,6 @@ null; #### Errors -- [`InvalidLibraryName`](#invalidlibraryname) to signal that the selected - library name is not valid. - [`LibraryAlreadyExists`](#libraryalreadyexists) to signal that a library with the given namespace and name already exists. - [`FileSystemError`](#filesystemerror) to signal a generic, unrecoverable @@ -4409,9 +4406,6 @@ versions. This is a temporary solution and in the longer-term it should be replaced with separate settings allowing to arbitrarily modify the library version from the IDE. -The `uploadUrl` is the URL of the library repository that accepts library -uploads. - The metadata for publishing the library can be set with [`library/setMetadata`](#librarysetmetadata). If it was not set, the publish operation will still proceed, but that metadata will be missing. @@ -4423,7 +4417,6 @@ operation will still proceed, but that metadata will be missing. namespace: String; name: String; authToken: String; - uploadUrl: String; bumpVersionAfterPublish?: Boolean; } @@ -4437,8 +4430,6 @@ null; #### Errors -- [`LocalLibraryNotFound`](#locallibrarynotfound) to signal that a local library - with the given name does not exist on the local libraries path. - [`LibraryPublishError`](#librarypublisherror) to signal that the server did not accept to publish the library (for example because a library with the same version already exists). @@ -5003,21 +4994,3 @@ there either. } } ``` - -### `InvalidLibraryName` - -Signals that the chosen library name is invalid. - -It contains a suggestion of a similar name that is valid. - -For example for `FooBar` it will suggest `Foo_Bar`. - -```typescript -"error" : { - "code" : 8009, - "message" : "[] is not a valid name: .", - "payload" : { - "suggestedName" : "" - } -} -``` diff --git a/docs/libraries/repositories.md b/docs/libraries/repositories.md index e24b618ef41d..b754a39c7f34 100644 --- a/docs/libraries/repositories.md +++ b/docs/libraries/repositories.md @@ -234,31 +234,6 @@ them), it will result in the following merged directory structure: └── LICENSE.md ``` -### Publishing - -To be able to publish libraries to a repository, the repository must provide an -upload endpoint which satisfies the following requirements. - -The endpoint should get the library name and version from the query parameters: -`namespace`, `name` and `version`. - -It should check any authentication data attached to the query and verify that -the user has sufficient privileges to upload the library for that `namespace`. - -Currently, we use a static check which checks an `Auth-Token` header for a -pre-determined secret key, but any other authentication schemes can be used, as -long as they are supported by the GUI or CLI. - -Then, the server must check if a library with the given name and version -combination already exists. If the library already exists, the request should be -rejected with `409 Conflict` status code indicating that a conflicting library -is already in the repository. - -If the request goes through, the server should create a directory for the -library and put any files attached to the request there. Each request should -always contain `package.yaml` and `manifest.yaml` files attached and at least -one sub-archive, usually called `main.tgz`. - ## Editions Repository The Editions repository has a very simple structure. diff --git a/docs/libraries/sharing.md b/docs/libraries/sharing.md index fa71214d77f7..e0f46b2a0773 100644 --- a/docs/libraries/sharing.md +++ b/docs/libraries/sharing.md @@ -62,17 +62,5 @@ import . ## Publishing -To publish a library, first you must obtain the upload URL of the repository, if -you are hosting the repository locally it will be `http://localhost:8080/upload` -(or possibly with a different port if that was overridden). - -If the repository requires authentication, it is best to set it up by setting -the `ENSO_AUTH_TOKEN` environment variable to the value of your secret token. - -Then you can use the Enso CLI to upload the project: - -```bash -enso publish-library --upload-url -``` - -See `enso publish-library --help` for more information. +> Soon it will be possible to share the libraries through the Marketplace, but +> it is still a work in progress. diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala index 36998b97f1b9..e6c04dc883eb 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala @@ -164,7 +164,6 @@ object LibraryApi { namespace: String, name: String, authToken: String, - uploadUrl: String, bumpVersionAfterPublish: Option[Boolean] ) @@ -233,16 +232,4 @@ object LibraryApi { } """ ) } - - case class InvalidLibraryName( - originalName: String, - suggestedName: String, - reason: String - ) extends Error(8009, s"[$originalName] is not a valid name: $reason.") { - override def payload: Option[Json] = Some( - json""" { - "suggestedName" : $suggestedName - } """ - ) - } } diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManager.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManager.scala index eacb8ba5986e..cde1650435db 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManager.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManager.scala @@ -5,12 +5,8 @@ import com.typesafe.scalalogging.LazyLogging import org.enso.distribution.{DistributionManager, FileSystem} import org.enso.editions.{Editions, LibraryName} import org.enso.languageserver.libraries.LocalLibraryManagerProtocol._ -import org.enso.librarymanager.local.{ - DefaultLocalLibraryProvider, - LocalLibraryProvider -} +import org.enso.librarymanager.local.LocalLibraryProvider import org.enso.pkg.PackageManager -import org.enso.pkg.validation.NameValidation import java.io.File import java.nio.file.Files @@ -38,18 +34,9 @@ class LocalLibraryManager( sender() ! listLocalLibraries() case Create(libraryName, authors, maintainers, license) => sender() ! createLibrary(libraryName, authors, maintainers, license) - case FindLibrary(libraryName) => - sender() ! findLibrary(libraryName) - } - } - - /** Checks if the library name is a valid Enso module name. */ - private def validateLibraryName(libraryName: LibraryName): Unit = { - // TODO [RW] more specific exceptions - NameValidation.validateName(libraryName.name) match { - case Left(error) => - throw new RuntimeException(s"Library name is not valid: [$error].") - case Right(_) => + case Publish(_, _, _) => + logger.error("Publishing libraries is currently not implemented.") + sender() ! Failure(new NotImplementedError()) } } @@ -67,8 +54,6 @@ class LocalLibraryManager( // TODO [RW] modify protocol to be able to create Contact instances val _ = (authors, maintainers) - validateLibraryName(libraryName) - // TODO [RW] make the exceptions more relevant val possibleRoots = LazyList .from(distributionManager.paths.localLibrariesSearchPaths) @@ -119,17 +104,6 @@ class LocalLibraryManager( } yield LibraryName(namespace, name) } - /** Finds the path on the filesystem to a local library. */ - private def findLibrary( - libraryName: LibraryName - ): Try[FindLibraryResponse] = Try { - val localLibraryProvider = new DefaultLocalLibraryProvider( - distributionManager.paths.localLibrariesSearchPaths.toList - ) - val pathOpt = localLibraryProvider.findLibrary(libraryName) - FindLibraryResponse(pathOpt) - } - /** Finds the edition associated with the current project, if specified in its * config. */ diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManagerProtocol.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManagerProtocol.scala index 5414816064af..9f3996509c49 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManagerProtocol.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LocalLibraryManagerProtocol.scala @@ -2,8 +2,6 @@ package org.enso.languageserver.libraries import org.enso.editions.LibraryName -import java.nio.file.Path - object LocalLibraryManagerProtocol { /** A top class representing any request to the [[LocalLibraryManager]]. */ @@ -39,9 +37,10 @@ object LocalLibraryManagerProtocol { license: String ) extends Request - /** A request to find the path to a local library. */ - case class FindLibrary(libraryName: LibraryName) extends Request - - /** A response to [[FindLibrary]]. */ - case class FindLibraryResponse(libraryRoot: Option[Path]) + /** A request to publish a library. */ + case class Publish( + libraryName: LibraryName, + authToken: String, + bumpVersionAfterPublish: Boolean + ) extends Request } diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala index 18900011677c..0b0095b454e8 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala @@ -1,161 +1,32 @@ package org.enso.languageserver.libraries.handler -import akka.actor.{Actor, ActorRef, Cancellable, Props} +import akka.actor.{Actor, Props} import com.typesafe.scalalogging.LazyLogging -import org.enso.cli.task.notifications.ActorProgressNotificationForwarder -import org.enso.editions.LibraryName -import org.enso.jsonrpc.{Id, Request, ResponseError, ResponseResult, Unused} +import org.enso.jsonrpc.{Request, ResponseError} import org.enso.languageserver.filemanager.FileManagerApi.FileSystemError import org.enso.languageserver.libraries.LibraryApi._ -import org.enso.languageserver.libraries.LocalLibraryManagerProtocol.{ - FindLibrary, - FindLibraryResponse -} -import org.enso.languageserver.requesthandler.RequestTimeout import org.enso.languageserver.util.UnhandledLogging -import org.enso.libraryupload.{auth, LibraryUploader} - -import scala.concurrent.duration.FiniteDuration -import scala.util.{Failure, Success} /** A request handler for the `library/publish` endpoint. * - * @param timeout request timeout - * @param localLibraryManager a reference to the LocalLibraryManager + * It is currently a stub implementation which will be refined later on. */ -class LibraryPublishHandler( - timeout: FiniteDuration, - localLibraryManager: ActorRef -) extends Actor +class LibraryPublishHandler + extends Actor with LazyLogging with UnhandledLogging { - override def receive: Receive = requestStage - - import context.dispatcher - - private def requestStage: Receive = { - case Request( - LibraryPublish, - id, - LibraryPublish.Params( - namespace, - name, - authToken, - uploadUrl, - bumpVersionAfterPublish - ) - ) => - val shouldBump = bumpVersionAfterPublish.getOrElse(false) - val replyTo = sender() - val token = auth.SimpleHeaderToken(authToken) - val libraryName = LibraryName(namespace, name) - localLibraryManager ! FindLibrary(libraryName) - - val timeoutCancellable = - context.system.scheduler.scheduleOnce(timeout, self, RequestTimeout) - context.become( - waitForLibraryResolutionStage( - replyTo, - libraryName, - id, - uploadUrl, - token, - timeoutCancellable, - shouldBump - ) - ) - } - - private def stop(timeoutCancellable: Cancellable): Unit = { - timeoutCancellable.cancel() - context.stop(self) - } - - /** Waits for the response of LocalLibraryManager and continues the publishing - * process. - */ - private def waitForLibraryResolutionStage( - replyTo: ActorRef, - libraryName: LibraryName, - id: Id, - uploadUrl: String, - token: auth.Token, - timeoutCancellable: Cancellable, - shouldBumpAfterPublishing: Boolean - ): Receive = { - case RequestTimeout => - replyTo ! RequestTimeout - context.stop(self) - - case Success(FindLibraryResponse(Some(libraryRoot))) => - val progressReporter = - ActorProgressNotificationForwarder.translateAndForward( - LibraryPublish.name, - replyTo - ) - - val result = LibraryUploader.uploadLibrary( - libraryRoot, - uploadUrl, - token, - progressReporter - ) - - result match { - case Failure(exception) => - replyTo ! ResponseError( - Some(id), - FileSystemError(s"Upload failed: $exception") - ) - case Success(_) => - if (shouldBumpAfterPublishing) { - logger.warn( - "`bumpVersionAfterPublish` was set to true, but this feature " + - "is not currently implemented. Ignoring." - ) - } - replyTo ! ResponseResult(LibraryPublish, id, Unused) - } - - stop(timeoutCancellable) - - case Success(FindLibraryResponse(None)) => - replyTo ! ResponseError( + override def receive: Receive = { + case Request(LibraryPublish, id, _: LibraryPublish.Params) => + // TODO [RW] actual implementation + sender() ! ResponseError( Some(id), - FileSystemError( - s"The library [$libraryName] was not found in local libraries " + - s"search paths." - ) + FileSystemError("Feature not implemented") ) - - stop(timeoutCancellable) - - case Failure(exception) => - replyTo ! ResponseError( - Some(id), - FileSystemError( - s"Failed to find the requested local library: $exception" - ) - ) - - stop(timeoutCancellable) } } object LibraryPublishHandler { - /** Creates a configuration object to create [[LibraryPublishHandler]]. - * - * @param timeout request timeout - * @param localLibraryManager a reference to the LocalLibraryManager - */ - def props( - timeout: FiniteDuration, - localLibraryManager: ActorRef - ): Props = Props( - new LibraryPublishHandler( - timeout, - localLibraryManager - ) - ) + /** Creates a configuration object to create [[LibraryPublishHandler]]. */ + def props(): Props = Props(new LibraryPublishHandler) } diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala b/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala index 1435dc707a10..53b157d36eb1 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala @@ -505,8 +505,7 @@ class JsonConnectionController( .props(requestTimeout, localLibraryManager), LibraryGetMetadata -> LibraryGetMetadataHandler.props(), LibraryPreinstall -> LibraryPreinstallHandler.props(), - LibraryPublish -> LibraryPublishHandler - .props(requestTimeout, localLibraryManager), + LibraryPublish -> LibraryPublishHandler.props(), LibrarySetMetadata -> LibrarySetMetadataHandler.props() ) } diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala index 340e27019af6..1ad058252bda 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala @@ -4,9 +4,6 @@ import io.circe.literal._ import io.circe.{Json, JsonObject} import org.enso.languageserver.libraries.LibraryEntry import org.enso.languageserver.libraries.LibraryEntry.PublishedLibraryVersion -import org.enso.librarymanager.published.repository.EmptyRepository - -import java.nio.file.Files class LibrariesTest extends BaseServerTest { "LocalLibraryManager" should { @@ -32,8 +29,8 @@ class LibrariesTest extends BaseServerTest { "method": "library/create", "id": 1, "params": { - "namespace": "user", - "name": "My_Local_Lib", + "namespace": "User", + "name": "MyLocalLib", "authors": [], "maintainers": [], "license": "" @@ -59,8 +56,8 @@ class LibrariesTest extends BaseServerTest { "result": { "localLibraries": [ { - "namespace": "user", - "name": "My_Local_Lib", + "namespace": "User", + "name": "MyLocalLib", "version": { "type": "LocalLibraryVersion" } @@ -75,83 +72,6 @@ class LibrariesTest extends BaseServerTest { "existed" ignore { // TODO [RW] error handling (#1877) } - - "validate the library name" ignore { - // TODO [RW] error handling (#1877) - } - - def port: Int = 47308 - - "create and publish a library" in { - val client = getInitialisedWsClient() - client.send(json""" - { "jsonrpc": "2.0", - "method": "library/create", - "id": 0, - "params": { - "namespace": "user", - "name": "Publishable_Lib", - "authors": [], - "maintainers": [], - "license": "" - } - } - """) - client.expectJson(json""" - { "jsonrpc": "2.0", - "id": 0, - "result": null - } - """) - - val repoRoot = getTestDirectory.resolve("libraries_repo_root") - val server = EmptyRepository.startServer(port, repoRoot, uploads = true) - try { - val uploadUrl = s"http://localhost:$port/upload" - client.send(json""" - { "jsonrpc": "2.0", - "method": "library/publish", - "id": 1, - "params": { - "namespace": "user", - "name": "Publishable_Lib", - "authToken": "SOME TOKEN", - "uploadUrl": $uploadUrl, - "bumpVersionAfterPublish": null - } - } - """) - - var found = false - while (!found) { - val rawResponse = client.expectSomeJson() - val response = rawResponse.asObject.value - val idMatches = - response("id").flatMap(_.asNumber).flatMap(_.toInt).contains(1) - if (idMatches) { - rawResponse shouldEqual json""" - { "jsonrpc": "2.0", - "id": 1, - "result": null - } - """ - - found = true - } - } - - val libraryRoot = repoRoot - .resolve("libraries") - .resolve("user") - .resolve("Publishable_Lib") - .resolve("0.0.1") - val mainPackage = libraryRoot.resolve("main.tgz") - assert(Files.exists(mainPackage)) - } finally { - server.kill(killDescendants = true) - server.join(waitForDescendants = true) - } - } } "mocked library/preinstall" should { diff --git a/engine/launcher/src/main/scala/org/enso/launcher/Constants.scala b/engine/launcher/src/main/scala/org/enso/launcher/Constants.scala deleted file mode 100644 index 0a624b03a4e5..000000000000 --- a/engine/launcher/src/main/scala/org/enso/launcher/Constants.scala +++ /dev/null @@ -1,14 +0,0 @@ -package org.enso.launcher - -import nl.gn0s1s.bump.SemVer - -object Constants { - - /** The engine version in which the uploads command has been introduced. - * - * It is used to check by the launcher if the engine can handle this command - * and provide better error messages if it cannot. - */ - val uploadIntroducedVersion: SemVer = - SemVer(0, 2, 17, Some("SNAPSHOT")) -} diff --git a/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala b/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala index d1e05b0e9253..36d1a4f3c068 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala @@ -341,52 +341,6 @@ case class Launcher(cliOptions: GlobalCLIOptions) { 0 } - /** Uploads a library to a repository. - * - * @param path path to the library, if not specified, the current working - * directory and its ancestors are searched for an Enso project - * to upload - * @param uploadUrl a URL of an upload endpoint of a repository; if not - * specified, falls back to the default Enso repository - * @param authToken a token to use for authentication - * @param logLevel log level for the language server - * @param useSystemJVM if set, forces to use the default configured JVM, - * instead of the JVM associated with the engine version - * @param jvmOpts additional options to pass to the launched JVM - * @param additionalArguments additional arguments to pass to the runner - * @return exit code of the launched program - */ - def uploadLibrary( - path: Option[Path], - uploadUrl: Option[String], - authToken: Option[String], - logLevel: LogLevel, - useSystemJVM: Boolean, - jvmOpts: Seq[(String, String)], - additionalArguments: Seq[String] - ): Int = { - val settings = runner - .uploadLibrary( - path, - uploadUrl.getOrElse { - throw new IllegalArgumentException( - "The default repository is currently not defined. " + - "You need to explicitly specify the `--upload-url`." - ) - }, - authToken.orElse(LauncherEnvironment.getEnvVar("ENSO_AUTH_TOKEN")), - cliOptions.hideProgress, - logLevel, - cliOptions.internalOptions.logMasking, - additionalArguments - ) - .get - - runner.withCommand(settings, JVMSettings(useSystemJVM, jvmOpts)) { - command => command.run().get - } - } - /** Prints the value of `key` from the global configuration. * * If the `key` is not set in the config, sets exit code to 1 and prints a diff --git a/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIProgressReporter.scala b/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIProgressReporter.scala index cf2636886783..e0a302e93f1c 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIProgressReporter.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIProgressReporter.scala @@ -7,14 +7,14 @@ import org.enso.launcher.InfoLogger /** A [[ProgressReporter]] that displays a progress bar in the console or waits * for the task silently, depending on CLI options. */ -class CLIProgressReporter(hideProgress: Boolean) extends ProgressReporter { +class CLIProgressReporter(cliOptions: GlobalCLIOptions) + extends ProgressReporter { /** @inheritdoc */ override def trackProgress(message: String, task: TaskProgress[_]): Unit = { InfoLogger.info(message) - if (!hideProgress) { - ProgressBar.waitWithProgress(task) - } + if (cliOptions.hideProgress) () + else ProgressBar.waitWithProgress(task) } } @@ -22,5 +22,5 @@ object CLIProgressReporter { /** A helper method to create [[CLIProgressReporter]] instances. */ def apply(globalCLIOptions: GlobalCLIOptions): CLIProgressReporter = - new CLIProgressReporter(globalCLIOptions.hideProgress) + new CLIProgressReporter(globalCLIOptions) } diff --git a/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIRuntimeVersionManagementUserInterface.scala b/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIRuntimeVersionManagementUserInterface.scala index dcff6c0a0afc..9ce501e1bcdc 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIRuntimeVersionManagementUserInterface.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/cli/CLIRuntimeVersionManagementUserInterface.scala @@ -18,7 +18,7 @@ import org.enso.runtimeversionmanager.components.{ class CLIRuntimeVersionManagementUserInterface( cliOptions: GlobalCLIOptions, alwaysInstallMissing: Boolean -) extends CLIProgressReporter(hideProgress = cliOptions.hideProgress) +) extends CLIProgressReporter(cliOptions) with RuntimeVersionManagementUserInterface { private val logger = Logger[CLIRuntimeVersionManagementUserInterface] diff --git a/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala b/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala index c9f05188b52a..85b5b3cd0464 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala @@ -312,64 +312,6 @@ object LauncherApplication { } } - private def uploadLibraryCommand: Command[Config => Int] = - Command( - "publish-library", - "Publish an Enso library to a repository. " + - "If `auto-confirm` is set, this will install missing engines or " + - "runtimes without asking." - ) { - val pathOpt = Opts.optionalArgument[Path]( - "PATH", - "If PATH is provided, the project at this path or the closest one of " + - "its ancestors that contains an Enso project, is uploaded. If not, " + - "the project to upload is searched for in the current directory and " + - "its ancestors." - ) - val uploadUrlOpt = Opts.optionalParameter[String]( - "upload-url", - "URL", - "Upload URL of the repository to upload the library to.", - showInUsage = true - ) - val authTokenOpt = Opts.optionalParameter[String]( - "auth-token", - "TOKEN", - "An optional token to add to request headers for use in " + - "authorization. If this parameter is not set, the ENSO_AUTH_TOKEN " + - "environment variable is checked." - ) - val additionalArgs = Opts.additionalArguments() - ( - pathOpt, - uploadUrlOpt, - authTokenOpt, - engineLogLevel, - systemJVMOverride, - jvmOpts, - additionalArgs - ) mapN { - ( - path, - uploadUrl, - authToken, - engineLogLevel, - systemJVMOverride, - jvmOpts, - additionalArgs - ) => (config: Config) => - Launcher(config).uploadLibrary( - path = path, - uploadUrl = uploadUrl, - authToken = authToken, - logLevel = engineLogLevel, - useSystemJVM = systemJVMOverride, - jvmOpts = jvmOpts, - additionalArguments = additionalArgs - ) - } - } - private def installEngineCommand: Command[Config => Int] = Command( "engine", @@ -529,7 +471,7 @@ object LauncherApplication { private def topLevelOpts: Opts[() => TopLevelBehavior[Config]] = { val version = - Opts.flag("version", 'V', "Display version.", showInUsage = false) + Opts.flag("version", 'V', "Display version.", showInUsage = true) val json = Opts.flag( GlobalCLIOptions.USE_JSON, "Use JSON instead of plain text for version output.", @@ -662,7 +604,6 @@ object LauncherApplication { replCommand, runCommand, languageServerCommand, - uploadLibraryCommand, defaultCommand, installCommand, uninstallCommand, diff --git a/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala b/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala index c30ea3693fc7..b78b72a0b275 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala @@ -3,9 +3,7 @@ package org.enso.launcher.components import akka.http.scaladsl.model.Uri import nl.gn0s1s.bump.SemVer import org.enso.distribution.{DistributionManager, EditionManager, Environment} -import org.enso.launcher.Constants import org.enso.launcher.project.ProjectManager -import org.enso.logger.masking.MaskedPath import org.enso.loggingservice.LogLevel import org.enso.runtimeversionmanager.components.RuntimeVersionManager import org.enso.runtimeversionmanager.config.GlobalConfigurationManager @@ -193,51 +191,4 @@ class LauncherRunner( ) } } - - /** Creates [[RunSettings]] for uploading a library. - * - * See [[org.enso.launcher.Launcher.uploadLibrary]] for more details. - */ - def uploadLibrary( - path: Option[Path], - uploadUrl: String, - token: Option[String], - hideProgress: Boolean, - logLevel: LogLevel, - logMasking: Boolean, - additionalArguments: Seq[String] - ): Try[RunSettings] = - Try { - val actualPath = path.getOrElse(currentWorkingDirectory) - val project = projectManager.findProject(actualPath).get.getOrElse { - throw RunnerError( - s"Could not find a project at " + - s"${MaskedPath(actualPath).applyMasking()} or any of its parent " + - s"directories." - ) - } - - val version = resolveVersion(None, Some(project)) - if (version < Constants.uploadIntroducedVersion) { - throw RunnerError( - s"Library Upload feature is not available in Enso $version. " + - s"Please upgrade your project to a newer version." - ) - } - - val tokenOpts = token.map(Seq("--auth-token", _)).toSeq.flatten - val hideProgressOpts = - if (hideProgress) Seq("--hide-progress") else Seq.empty - - val arguments = - Seq("--upload", uploadUrl) ++ - Seq("--in-project", project.path.toAbsolutePath.normalize.toString) ++ - tokenOpts ++ hideProgressOpts - RunSettings( - version, - arguments ++ setLogLevelArgs(logLevel, logMasking) - ++ additionalArguments, - connectLoggerIfAvailable = true - ) - } } diff --git a/engine/launcher/src/test/scala/org/enso/launcher/components/UploadVersionCheck.scala b/engine/launcher/src/test/scala/org/enso/launcher/components/UploadVersionCheck.scala deleted file mode 100644 index 4d54be924b9c..000000000000 --- a/engine/launcher/src/test/scala/org/enso/launcher/components/UploadVersionCheck.scala +++ /dev/null @@ -1,27 +0,0 @@ -package org.enso.launcher.components - -import nl.gn0s1s.bump.SemVer -import org.enso.launcher.Constants -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -class UploadVersionCheck extends AnyWordSpec with Matchers { - "Constants.uploadIntroducedVersion" should { - "correctly compare with nearby versions" in { - assert( - SemVer("0.2.17-SNAPSHOT.2021-07-23").get >= - Constants.uploadIntroducedVersion - ) - - assert( - SemVer("0.2.17").get > - Constants.uploadIntroducedVersion - ) - - assert( - SemVer("0.2.16").get < - Constants.uploadIntroducedVersion - ) - } - } -} diff --git a/engine/runner/src/main/scala/org/enso/runner/Main.scala b/engine/runner/src/main/scala/org/enso/runner/Main.scala index c51e790df65f..b3dbc57d8752 100644 --- a/engine/runner/src/main/scala/org/enso/runner/Main.scala +++ b/engine/runner/src/main/scala/org/enso/runner/Main.scala @@ -14,7 +14,6 @@ import org.enso.version.VersionDescription import org.graalvm.polyglot.PolyglotException import java.io.File -import java.nio.file.Path import java.util.UUID import scala.Console.err import scala.jdk.CollectionConverters._ @@ -44,9 +43,6 @@ object Main { private val LOG_LEVEL = "log-level" private val LOGGER_CONNECT = "logger-connect" private val NO_LOG_MASKING = "no-log-masking" - private val UPLOAD_OPTION = "upload" - private val HIDE_PROGRESS = "hide-progress" - private val AUTH_TOKEN = "auth-token" private lazy val logger = Logger[Main.type] @@ -197,27 +193,6 @@ object Main { "variable." ) .build() - val uploadOption = CliOption.builder - .hasArg(true) - .numberOfArgs(1) - .argName("url") - .longOpt(UPLOAD_OPTION) - .desc( - "Uploads the library to a repository. " + - "The url defines the repository to upload to." - ) - .build() - val hideProgressOption = CliOption.builder - .longOpt(HIDE_PROGRESS) - .desc("If specified, progress bars will not be displayed.") - .build() - val authTokenOption = CliOption.builder - .hasArg(true) - .numberOfArgs(1) - .argName("token") - .longOpt(AUTH_TOKEN) - .desc("Authentication token for the upload.") - .build() val options = new Options options @@ -242,9 +217,6 @@ object Main { .addOption(logLevelOption) .addOption(loggerConnectOption) .addOption(noLogMaskingOption) - .addOption(uploadOption) - .addOption(hideProgressOption) - .addOption(authTokenOption) options } @@ -664,27 +636,6 @@ object Main { ) } - if (line.hasOption(UPLOAD_OPTION)) { - val projectRoot = - Option(line.getOptionValue(IN_PROJECT_OPTION)) - .map(Path.of(_)) - .getOrElse { - logger.error( - s"When uploading, the $IN_PROJECT_OPTION is mandatory " + - s"to specify which project to upload." - ) - exitFail() - } - - ProjectUploader.uploadProject( - projectRoot = projectRoot, - uploadUrl = line.getOptionValue(UPLOAD_OPTION), - authToken = Option(line.getOptionValue(AUTH_TOKEN)), - showProgress = !line.hasOption(HIDE_PROGRESS) - ) - exitSuccess() - } - if (line.hasOption(RUN_OPTION)) { run( line.getOptionValue(RUN_OPTION), diff --git a/engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala b/engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala deleted file mode 100644 index 6012df69a919..000000000000 --- a/engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala +++ /dev/null @@ -1,56 +0,0 @@ -package org.enso.runner - -import com.typesafe.scalalogging.Logger -import org.enso.cli.ProgressBar -import org.enso.cli.task.{ProgressReporter, TaskProgress} -import org.enso.libraryupload.{auth, LibraryUploader} - -import java.nio.file.Path - -/** Gathers helper functions for uploading a library project. */ -object ProjectUploader { - - private lazy val logger = Logger[ProjectUploader.type] - - /** Uploads a project to a library repository. - * - * @param projectRoot path to the root of the project - * @param uploadUrl URL of upload endpoint of the repository to upload to - * @param authToken an optional token used for authentication in the - * repository - * @param showProgress specifies if CLI progress bars should be displayed - * showing progress of compression and upload - */ - def uploadProject( - projectRoot: Path, - uploadUrl: String, - authToken: Option[String], - showProgress: Boolean - ): Unit = { - import scala.concurrent.ExecutionContext.Implicits.global - val progressReporter = new ProgressReporter { - override def trackProgress( - message: String, - task: TaskProgress[_] - ): Unit = { - logger.info(message) - if (showProgress) { - ProgressBar.waitWithProgress(task) - } - } - } - - val token = authToken match { - case Some(value) => auth.SimpleHeaderToken(value) - case None => auth.NoAuthorization - } - LibraryUploader - .uploadLibrary( - projectRoot, - uploadUrl, - token, - progressReporter - ) - .get - } -} diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala index 5a831c490a9c..5217ce9430f5 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala @@ -51,7 +51,6 @@ object NotificationHandler { /** @inheritdoc */ override def trackProgress(message: String, task: TaskProgress[_]): Unit = { logger.info(message) - // TODO [RW] check the hideProgress flag provided by the launcher if (System.console() != null) { ProgressBar.waitWithProgress(task) } diff --git a/lib/scala/cli/src/main/scala/org/enso/cli/task/TaskProgress.scala b/lib/scala/cli/src/main/scala/org/enso/cli/task/TaskProgress.scala index 2df5e223ce63..a78d47d99f24 100644 --- a/lib/scala/cli/src/main/scala/org/enso/cli/task/TaskProgress.scala +++ b/lib/scala/cli/src/main/scala/org/enso/cli/task/TaskProgress.scala @@ -1,7 +1,7 @@ package org.enso.cli.task import java.util.concurrent.LinkedTransferQueue -import scala.concurrent.{ExecutionContext, Future} + import scala.util.{Failure, Try} /** Represents a long-running background task. @@ -98,21 +98,6 @@ object TaskProgress { } } - /** Creates a [[TaskProgress]] from a [[Future]]. */ - def fromFuture[A]( - future: Future[A] - )(implicit ec: ExecutionContext): TaskProgress[A] = { - new TaskProgress[A] { - override def addProgressListener( - listener: ProgressListener[A] - ): Unit = { - future.onComplete { result => - listener.done(result) - } - } - } - } - /** Blocks and waits for the task to complete. */ def waitForTask[A](task: TaskProgress[A]): Try[A] = { diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/TarGzWriter.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/TarGzWriter.scala deleted file mode 100644 index 059b5579246d..000000000000 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/TarGzWriter.scala +++ /dev/null @@ -1,126 +0,0 @@ -package org.enso.downloader.archive - -import org.apache.commons.compress.archivers.tar.{ - TarArchiveEntry, - TarArchiveOutputStream -} -import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream -import org.enso.cli.task.{TaskProgress, TaskProgressImplementation} - -import java.io.{BufferedOutputStream, FileOutputStream} -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path} -import scala.util.{Try, Using} - -/** A helper class for writing TAR archives compressed with gzip. */ -class TarGzWriter private (archive: TarArchiveOutputStream) { - - /** Adds a text file to the archive. - * - * @param relativePath path of the file in the archive - * @param content the text content to put in the file - */ - def writeTextFile(relativePath: String, content: String): Unit = { - val bytes = content.getBytes(StandardCharsets.UTF_8) - val entry = new TarArchiveEntry(relativePath) - entry.setSize(bytes.size.toLong) - archive.putArchiveEntry(entry) - archive.write(bytes) - archive.closeArchiveEntry() - } - - /** Adds a file from the filesystem to the archive. - * - * @param relativePath path of the file in the archive - * @param filePath a path to a file on the filesystem that will be read and - * put into the archive - * @return returns the number of bytes that were transferrred from the input - * file - */ - def writeFile(relativePath: String, filePath: Path): Long = { - val entry = new TarArchiveEntry(filePath.toFile, relativePath) - archive.putArchiveEntry(entry) - val bytesTransferred = Files.copy(filePath, archive) - archive.closeArchiveEntry() - bytesTransferred - } -} - -object TarGzWriter { - - /** Creates a .tar.gz archive at the specified destination. - * - * It calls the `actions` callback with a [[TarGzWriter]] instance which can - * be used to add files to the archive. The [[TarGzWriter]] instance is only - * valid during the call of this callback, it should not be leaked anywhere - * else as it will then be invalid. - */ - def createArchive( - destination: Path - )(actions: TarGzWriter => Unit): Try[Unit] = - Using(new FileOutputStream(destination.toFile)) { outputStream => - Using(new BufferedOutputStream(outputStream)) { bufferedStream => - Using(new GzipCompressorOutputStream(bufferedStream)) { gzipStream => - Using(new TarArchiveOutputStream(gzipStream)) { archive => - val writer = new TarGzWriter(archive) - actions(writer) - }.get - }.get - }.get - } - - /** Creates a .tar.gz archive from a list of files. - * - * @param archiveDestination path specifying where to put the archive - * @param files list of paths to files that should be compressed; these - * should be regular files, the behaviour is undefined if one of - * these paths is a directory - * @param basePath the base path to compute the relative paths of the - * compressed files; all `files` should be inside of the - * directory denoted by `basePath` - */ - def compress( - archiveDestination: Path, - files: Seq[Path], - basePath: Path - ): TaskProgress[Unit] = { - val normalizedBase = basePath.toAbsolutePath.normalize - def relativePath(file: Path): String = { - val normalized = file.toAbsolutePath.normalize - if (!normalized.startsWith(normalizedBase)) { - throw new IllegalArgumentException( - "TarGzWriter precondition failure: " + - "Files should all be inside of the provided basePath." - ) - } - normalizedBase.relativize(normalized).toString - } - - val taskProgress = new TaskProgressImplementation[Unit]() - - def runCompresion(): Unit = { - val sumSize = files.map(Files.size).sum - - val result = TarGzWriter.createArchive(archiveDestination) { writer => - var totalBytesWritten: Long = 0 - def update(): Unit = - taskProgress.reportProgress(totalBytesWritten, Some(sumSize)) - update() - for (file <- files) { - // TODO [RW] Ideally we could report progress for each chunk, offering - // more granular feedback for big data files. - val bytesWritten = writer.writeFile(relativePath(file), file) - totalBytesWritten += bytesWritten - update() - } - } - - taskProgress.setComplete(result) - } - - val thread = new Thread(() => runCompresion(), "Writing-Archive") - thread.start() - - taskProgress - } -} diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala index e89171b791b9..497aa0ade71a 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala @@ -157,7 +157,6 @@ object HTTPDownload { "akka.library-extensions", ConfigValueFactory.fromAnyRef(Seq.empty.asJava) ) - .withValue("akka.daemonic", ConfigValueFactory.fromAnyRef("on")) .withValue("akka.loggers", ConfigValueFactory.fromAnyRef(loggers)) .withValue( "akka.logging-filter", diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala index 7757d0f6fa05..f525d20da772 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala @@ -11,16 +11,13 @@ import org.enso.downloader.http */ case class HTTPRequestBuilder private ( uri: Uri, - headers: Vector[(String, String)], - httpEntity: RequestEntity + headers: Vector[(String, String)] ) { - /** Builds a GET request with the specified settings. */ + /** Builds a GET request with the specified settings. + */ def GET: HTTPRequest = build(HttpMethods.GET) - /** Builds a POST request with the specified settings. */ - def POST: HTTPRequest = build(HttpMethods.POST) - /** Adds an additional header that will be included in the request. * * @param name name of the header @@ -29,14 +26,6 @@ case class HTTPRequestBuilder private ( def addHeader(name: String, value: String): HTTPRequestBuilder = copy(headers = headers.appended((name, value))) - /** Sets the [[RequestEntity]] for the request. - * - * It can be used for example to specify form data to send for a POST - * request. - */ - def setEntity(entity: RequestEntity): HTTPRequestBuilder = - copy(httpEntity = entity) - private def build( method: HttpMethod ): HTTPRequest = { @@ -52,12 +41,7 @@ case class HTTPRequestBuilder private ( } } http.HTTPRequest( - HttpRequest( - method = method, - uri = uri, - headers = httpHeaders, - entity = httpEntity - ) + HttpRequest(method = method, uri = uri, headers = httpHeaders) ) } } @@ -67,7 +51,7 @@ object HTTPRequestBuilder { /** Creates a request builder that will send the request for the given URI. */ def fromURI(uri: Uri): HTTPRequestBuilder = - new HTTPRequestBuilder(uri, Vector.empty, HttpEntity.Empty) + new HTTPRequestBuilder(uri, Vector.empty) /** Tries to parse the URI provided as a [[String]] and returns a request * builder that will send the request to the given `uri`. diff --git a/lib/scala/editions/src/main/scala/org/enso/yaml/YamlHelper.scala b/lib/scala/editions/src/main/scala/org/enso/yaml/YamlHelper.scala index 56b1890ad2fd..b7e82fa3ed96 100644 --- a/lib/scala/editions/src/main/scala/org/enso/yaml/YamlHelper.scala +++ b/lib/scala/editions/src/main/scala/org/enso/yaml/YamlHelper.scala @@ -1,11 +1,6 @@ package org.enso.yaml -import io.circe.yaml.Printer -import io.circe.{yaml, Decoder, Encoder} - -import java.io.FileReader -import java.nio.file.Path -import scala.util.{Try, Using} +import io.circe.{yaml, Decoder} /** A helper for parsing YAML configs. */ object YamlHelper { @@ -19,17 +14,4 @@ object YamlHelper { .flatMap(_.as[R]) .left .map(ParseError(_)) - - /** Tries to load and parse a YAML file at the provided path. */ - def load[R](path: Path)(implicit decoder: Decoder[R]): Try[R] = - Using(new FileReader(path.toFile)) { reader => - yaml.parser - .parse(reader) - .flatMap(_.as[R]) - .toTry - }.flatten - - /** Saves a YAML representation of an object into a string. */ - def toYaml[A](obj: A)(implicit encoder: Encoder[A]): String = - Printer.spaces2.copy(preserveOrder = true).pretty(encoder(obj)) } diff --git a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DownloaderTest.scala b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DownloaderTest.scala deleted file mode 100644 index 252dd46b1662..000000000000 --- a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DownloaderTest.scala +++ /dev/null @@ -1,49 +0,0 @@ -package org.enso.librarymanager.published.repository - -import org.enso.cli.task.{ProgressReporter, TaskProgress} -import org.enso.distribution.TemporaryDirectoryManager -import org.enso.distribution.locking.{ - LockUserInterface, - Resource, - ResourceManager, - ThreadSafeFileLockManager -} -import org.enso.librarymanager.published.cache.DownloadingLibraryCache -import org.enso.testkit.HasTestDirectory - -trait DownloaderTest { self: HasTestDirectory => - def withDownloader[R](action: DownloadingLibraryCache => R): R = { - val lockManager = - new ThreadSafeFileLockManager(getTestDirectory.resolve("locks")) - val resourceManager = new ResourceManager(lockManager) - try { - val cache = new DownloadingLibraryCache( - cacheRoot = getTestDirectory.resolve("cache"), - temporaryDirectoryManager = new TemporaryDirectoryManager( - getTestDirectory.resolve("tmp"), - resourceManager - ), - resourceManager = resourceManager, - lockUserInterface = new LockUserInterface { - override def startWaitingForResource(resource: Resource): Unit = - println(s"Waiting for ${resource.name}") - - override def finishWaitingForResource(resource: Resource): Unit = - println(s"${resource.name} is ready") - }, - progressReporter = new ProgressReporter { - override def trackProgress( - message: String, - task: TaskProgress[_] - ): Unit = {} - } - ) - - action(cache) - } finally { - resourceManager.releaseMainLock() - resourceManager.unlockTemporaryDirectory() - } - } - -} diff --git a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/EmptyRepository.scala b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/EmptyRepository.scala deleted file mode 100644 index 1314df4eedd6..000000000000 --- a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/EmptyRepository.scala +++ /dev/null @@ -1,5 +0,0 @@ -package org.enso.librarymanager.published.repository - -object EmptyRepository extends DummyRepository { - override def libraries: Seq[EmptyRepository.DummyLibrary] = Seq.empty -} diff --git a/lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala b/lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala deleted file mode 100644 index abf698dc0fdd..000000000000 --- a/lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala +++ /dev/null @@ -1,92 +0,0 @@ -package org.enso.libraryupload - -import nl.gn0s1s.bump.SemVer -import org.enso.cli.task.{ProgressReporter, TaskProgress} -import org.enso.editions.{Editions, LibraryName} -import org.enso.librarymanager.published.repository.{ - DownloaderTest, - EmptyRepository -} -import org.enso.libraryupload.auth.SimpleHeaderToken -import org.enso.pkg.PackageManager -import org.enso.testkit.WithTemporaryDirectory -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -import java.nio.file.Files - -class LibraryUploadTest - extends AnyWordSpec - with Matchers - with WithTemporaryDirectory - with DownloaderTest { - - def port: Int = 47305 - - "LibraryUploader" should { - "upload the files to the server" in { - val projectRoot = getTestDirectory.resolve("lib_root") - val repoRoot = getTestDirectory.resolve("repo") - - val libraryName = LibraryName("tester", "Upload_Test") - val libraryVersion = SemVer(1, 2, 3) - PackageManager.Default.create( - projectRoot.toFile, - name = libraryName.name, - namespace = libraryName.namespace, - version = libraryVersion.toString - ) - - val server = EmptyRepository.startServer(port, repoRoot, uploads = true) - try { - val uploadUrl = s"http://localhost:$port/upload" - val token = SimpleHeaderToken("TODO") - import scala.concurrent.ExecutionContext.Implicits.global - LibraryUploader - .uploadLibrary( - projectRoot, - uploadUrl, - token, - new ProgressReporter { - override def trackProgress(message: String, task: TaskProgress[_]) - : Unit = () - } - ) - .get - - val libRoot = repoRoot - .resolve("libraries") - .resolve("tester") - .resolve("Upload_Test") - .resolve("1.2.3") - - PackageManager.Default - .loadPackage(libRoot.toFile) - .get - .name shouldEqual libraryName.name - assert(Files.exists(libRoot.resolve("manifest.yaml"))) - assert(Files.exists(libRoot.resolve("main.tgz"))) - - withDownloader { cache => - cache.findCachedLibrary(libraryName, libraryVersion) shouldBe empty - - val repo = Editions.Repository( - "test_repo", - s"http://localhost:$port/libraries" - ) - val installedRoot = - cache.findOrInstallLibrary(libraryName, libraryVersion, repo).get - val pkg = PackageManager.Default.loadPackage(installedRoot.toFile).get - pkg.name shouldEqual libraryName.name - val sources = pkg.listSources - sources should have size 1 - sources.head.file.getName shouldEqual "Main.enso" - } - - } finally { - server.kill(killDescendants = true) - server.join(waitForDescendants = true) - } - } - } -} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala index 60add2710a24..af2cf5229db2 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala @@ -1,7 +1,6 @@ package org.enso.librarymanager.published.repository -import io.circe.syntax.EncoderOps -import io.circe.{Decoder, Encoder, Json} +import io.circe.Decoder import org.enso.editions.LibraryName /** The manifest file containing metadata related to a published library. @@ -20,17 +19,6 @@ case class LibraryManifest( ) object LibraryManifest { - - /** Creates an empty manifest. - * - * Such a manifest is invalid as at least one archive should be specified in - * a valid manifest. - * - * It can however be useful as a temporary value for logic that updates or - * creates a new manifest. - */ - def empty: LibraryManifest = LibraryManifest(Seq.empty, Seq.empty, None, None) - object Fields { val archives = "archives" val dependencies = "dependencies" @@ -55,20 +43,6 @@ object LibraryManifest { ) } - /** An [[Encoder]] instance for parsing [[LibraryManifest]]. */ - implicit val encoder: Encoder[LibraryManifest] = { manifest => - val baseFields = Seq( - Fields.archives -> manifest.archives.asJson, - Fields.dependencies -> manifest.dependencies.asJson - ) - - val allFields = baseFields ++ - manifest.tagLine.map(Fields.tagLine -> _.asJson).toSeq ++ - manifest.description.map(Fields.description -> _.asJson).toSeq - - Json.obj(allFields: _*) - } - /** The name of the manifest file as included in the directory associated with * a given library in the library repository. */ diff --git a/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala b/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala deleted file mode 100644 index ea3157d32338..000000000000 --- a/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala +++ /dev/null @@ -1,233 +0,0 @@ -package org.enso.libraryupload - -import akka.http.scaladsl.marshalling.Marshal -import akka.http.scaladsl.model._ -import akka.stream.scaladsl.Source -import com.typesafe.scalalogging.Logger -import nl.gn0s1s.bump.SemVer -import org.enso.cli.task.{ProgressReporter, TaskProgress} -import org.enso.distribution.FileSystem -import org.enso.distribution.FileSystem.PathSyntax -import org.enso.downloader.archive.TarGzWriter -import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} -import org.enso.editions.LibraryName -import org.enso.librarymanager.published.repository.LibraryManifest -import org.enso.pkg.{Package, PackageManager} -import org.enso.yaml.YamlHelper - -import java.nio.file.{Files, Path} -import scala.concurrent.{ExecutionContext, Future} -import scala.jdk.CollectionConverters._ -import scala.util.{Failure, Success, Try, Using} - -/** Gathers functions used for uploading libraries. */ -object LibraryUploader { - private lazy val logger = Logger[LibraryUploader.type] - - /** Uploads a library to a repository. - * - * @param projectRoot path to the library project root - * @param uploadUrl an URL to the upload endpoint of a library repository - * @param authToken a token describing the authentication method to use with - * the repository - * @param progressReporter a [[ProgressReporter]] to track long running tasks - * like compression and upload - * @param ec an execution context used for handling Futures - */ - def uploadLibrary( - projectRoot: Path, - uploadUrl: String, - authToken: auth.Token, - progressReporter: ProgressReporter - )(implicit ec: ExecutionContext): Try[Unit] = Try { - FileSystem.withTemporaryDirectory("enso-upload") { tmpDir => - val pkg = PackageManager.Default.loadPackage(projectRoot.toFile).get - val version = SemVer(pkg.config.version).getOrElse { - throw new IllegalStateException( - s"Project version [${pkg.config.version}] is not a valid semver " + - s"string." - ) - } - val uri = buildUploadUri(uploadUrl, pkg.libraryName, version) - - val mainArchiveName = "main.tgz" - val filesToIgnoreInArchive = Seq( - Package.configFileName, - LibraryManifest.filename - ) - val archivePath = tmpDir / mainArchiveName - val compressing = - createMainArchive(projectRoot, filesToIgnoreInArchive, archivePath) - progressReporter.trackProgress( - s"Creating the [$mainArchiveName] archive.", - compressing - ) - compressing.force() - - val manifestPath = projectRoot / LibraryManifest.filename - val loadedManifest = - loadSavedManifest(manifestPath).getOrElse(LibraryManifest.empty) - val updatedManifest = - // TODO [RW] update dependencies in the manifest (#1773) - loadedManifest.copy(archives = Seq(mainArchiveName)) - FileSystem.writeTextFile(manifestPath, YamlHelper.toYaml(updatedManifest)) - - logger.info(s"Uploading library package to the server at [$uploadUrl].") - val upload = uploadFiles( - uri, - authToken, - files = Seq( - projectRoot / Package.configFileName, - projectRoot / LibraryManifest.filename, - archivePath - ) - ) - progressReporter.trackProgress( - s"Uploading packages to [$uploadUrl].", - upload - ) - upload.force() - - logger.info(s"Upload complete.") - } - } - - /** Creates an URL for the upload, including information identifying the - * library version. - */ - private def buildUploadUri( - baseUploadUrl: String, - libraryName: LibraryName, - version: SemVer - ): Uri = { - URIBuilder - .fromUri(baseUploadUrl) - .addQuery("namespace", libraryName.namespace) - .addQuery("name", libraryName.name) - .addQuery("version", version.toString) - .build() - } - - /** Gathers project files to create the main archive. - * - * For now it just filters out the files like manifest which are uploaded - * separately. In the future this may be extended to create separate - * sub-archives for platform specific binaries or tests. - * - * @param projectRoot path to the project root - * @param rootFilesToIgnore names of files at the root that should *not* be - * included in the archive - * @param destination path at which the archive is created - * @return - */ - private def createMainArchive( - projectRoot: Path, - rootFilesToIgnore: Seq[String], - destination: Path - ): TaskProgress[Unit] = { - def relativePath(file: Path): String = projectRoot.relativize(file).toString - def shouldBeUploaded(file: Path): Boolean = { - def isIgnored = rootFilesToIgnore.contains(relativePath(file)) - Files.isRegularFile(file) && !isIgnored - } - - logger.trace("Gathering files to compress.") - val filesToCompress = Using(Files.walk(projectRoot)) { filesStream => - filesStream.iterator().asScala.filter(shouldBeUploaded).toSeq - }.get - - logger.info( - s"Compressing ${filesToCompress.size} project files " + - s"into [${destination.getFileName}]." - ) - - val compression = TarGzWriter.compress( - archiveDestination = destination, - files = filesToCompress, - basePath = projectRoot - ) - - compression.map { _ => - logger.info(s"Archive [${destination.getFileName}] created.") - } - } - - /** Creates a [[RequestEntity]] that will upload the provided files. */ - private def createRequestEntity( - files: Seq[Path] - )(implicit ec: ExecutionContext): Future[RequestEntity] = { - - val fileBodies = files.map { path => - val filename = path.getFileName.toString - Multipart.FormData.BodyPart( - filename, - HttpEntity.fromPath(detectContentType(path), path), - Map("filename" -> filename) - ) - } - - val formData = Multipart.FormData(Source(fileBodies)) - Marshal(formData).to[RequestEntity] - } - - /** Loads a manifest, if it exists. */ - private def loadSavedManifest(manifestPath: Path): Option[LibraryManifest] = { - if (Files.exists(manifestPath)) { - val loaded = YamlHelper.load[LibraryManifest](manifestPath).get - Some(loaded) - } else None - } - - /** Tries to detect the content type of the file to upload. - * - * If it is not a known type, it falls back to `application/octet-stream`. - */ - private def detectContentType(path: Path): ContentType = { - val filename = path.getFileName.toString - if (filename.endsWith(".tgz") || filename.endsWith(".tar.gz")) - ContentType(MediaTypes.`application/x-gtar`) - else if (filename.endsWith(".yaml") || filename.endsWith(".enso")) - ContentTypes.`text/plain(UTF-8)` - else ContentTypes.`application/octet-stream` - } - - /** Uploads the provided files to the provided url, using the provided token - * for authentication. - */ - private def uploadFiles( - uri: Uri, - authToken: auth.Token, - files: Seq[Path] - )(implicit ec: ExecutionContext): TaskProgress[Unit] = { - val future = createRequestEntity(files).map { entity => - val request = authToken - .alterRequest(HTTPRequestBuilder.fromURI(uri)) - .setEntity(entity) - .POST - // TODO [RW] upload progress - HTTPDownload.fetchString(request).force() - } - TaskProgress.fromFuture(future).flatMap { response => - if (response.statusCode == 200) { - logger.debug("Server responded with 200 OK.") - Success(()) - } else { - // TODO [RW] we may want to have more precise error messages to handle auth errors etc. (#1773) - val includedMessage = for { - json <- io.circe.parser.parse(response.content).toOption - obj <- json.asObject - message <- obj("error").flatMap(_.asString) - } yield message - val message = includedMessage.getOrElse("Unknown error") - val errorMessage = - s"Upload failed: $message (Status code: ${response.statusCode})." - logger.error(errorMessage) - Failure( - new RuntimeException( - errorMessage - ) - ) - } - } - } -} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/auth/Token.scala b/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/auth/Token.scala deleted file mode 100644 index f82b47fdfae9..000000000000 --- a/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/auth/Token.scala +++ /dev/null @@ -1,35 +0,0 @@ -package org.enso.libraryupload.auth - -import org.enso.downloader.http.HTTPRequestBuilder - -/** Represents an authentication method that can be used to authenticate - * requests to the library repository. - */ -trait Token { - - /** Alters the request adding any properties (like headers) necessary to - * successfully authenticate. - */ - def alterRequest(request: HTTPRequestBuilder): HTTPRequestBuilder -} - -/** A simple authentication method that adds an `Auth-Token` header to the - * request. - */ -case class SimpleHeaderToken(value: String) extends Token { - - /** @inheritdoc */ - override def alterRequest(request: HTTPRequestBuilder): HTTPRequestBuilder = - request.addHeader("Auth-Token", value) -} - -/** A dummy authentication method that does not do anything. - * - * It can be used for servers that do not require any authentication. - */ -case object NoAuthorization extends Token { - - /** @inheritdoc */ - override def alterRequest(request: HTTPRequestBuilder): HTTPRequestBuilder = - request -} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala new file mode 100644 index 000000000000..023f1b89a8e2 --- /dev/null +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala @@ -0,0 +1,50 @@ +package org.enso.librarymanager.published.repository + +import org.apache.commons.compress.archivers.tar.{ + TarArchiveEntry, + TarArchiveOutputStream +} +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream + +import java.io.{BufferedOutputStream, FileOutputStream} +import java.nio.file.Path +import scala.util.Using + +/** A helper class used for creating TAR-GZ archives in tests. */ +object ArchiveWriter { + + /** A file to add to the archive. */ + sealed trait FileToWrite { + + /** The path that this file should have within the archive. */ + def relativePath: String + } + + /** Represents a text file to be added to a test archive. + * + * @param relativePath the path in the archive + * @param content the text contents for the file + */ + case class TextFile(relativePath: String, content: String) extends FileToWrite + + /** Creates a tar archive at the given path, containing the provided files. */ + def writeTarArchive(path: Path, files: Seq[FileToWrite]): Unit = { + Using(new FileOutputStream(path.toFile)) { outputStream => + Using(new BufferedOutputStream(outputStream)) { bufferedStream => + Using(new GzipCompressorOutputStream(bufferedStream)) { gzipStream => + Using(new TarArchiveOutputStream(gzipStream)) { archive => + for (file <- files) { + file match { + case TextFile(relativePath, content) => + val entry = new TarArchiveEntry(relativePath) + archive.putArchiveEntry(entry) + archive.write(content.getBytes) + archive.closeArchiveEntry() + } + } + } + } + } + } + } +} diff --git a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala similarity index 88% rename from lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala rename to lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index 6c99ad5a33b9..3c163855ec8a 100644 --- a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -3,7 +3,6 @@ package org.enso.librarymanager.published.repository import nl.gn0s1s.bump.SemVer import org.enso.cli.OS import org.enso.distribution.FileSystem -import org.enso.downloader.archive.TarGzWriter import org.enso.editions.Editions.RawEdition import org.enso.editions.{Editions, LibraryName} import org.enso.pkg.{Package, PackageManager} @@ -49,13 +48,10 @@ abstract class DummyRepository { .resolve(lib.version.toString) Files.createDirectories(libraryRoot) createLibraryProject(libraryRoot, lib) - - TarGzWriter - .createArchive(libraryRoot.resolve("main.tgz")) { writer => - writer.writeTextFile("src/Main.enso", lib.mainContent) - } - .get - + val files = Seq( + ArchiveWriter.TextFile("src/Main.enso", lib.mainContent) + ) + ArchiveWriter.writeTarArchive(libraryRoot.resolve("main.tgz"), files) createManifest(libraryRoot) } } @@ -110,18 +106,12 @@ abstract class DummyRepository { * @param port port to listen on * @param root root of the library repository, the same as the argument to * [[createRepository]] - * @param uploads specifies whether to enable uploads in the server */ - def startServer( - port: Int, - root: Path, - uploads: Boolean = false - ): WrappedProcess = { + def startServer(port: Int, root: Path): WrappedProcess = { val serverDirectory = Path.of("tools/simple-library-server").toAbsolutePath.normalize - val preinstallCommand = - commandPrefix ++ Seq(npmCommand, "install") + val preinstallCommand = commandPrefix ++ Seq(npmCommand, "install") val preinstallExitCode = new ProcessBuilder() .command(preinstallCommand: _*) .directory(serverDirectory.toFile) @@ -135,7 +125,6 @@ abstract class DummyRepository { s"npm exited with code $preinstallCommand." ) - val uploadsArgs = if (uploads) Seq("--upload", "no-auth") else Seq() val command = commandPrefix ++ Seq( nodeCommand, "main.js", @@ -143,7 +132,7 @@ abstract class DummyRepository { port.toString, "--root", root.toAbsolutePath.normalize.toString - ) ++ uploadsArgs + ) val rawProcess = (new ProcessBuilder) .command(command: _*) .directory(serverDirectory.toFile) diff --git a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala similarity index 100% rename from lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala rename to lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala diff --git a/lib/scala/library-manager-test/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala similarity index 58% rename from lib/scala/library-manager-test/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala rename to lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala index 4c47ad016e11..882516116dab 100644 --- a/lib/scala/library-manager-test/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -1,8 +1,17 @@ package org.enso.librarymanager.published.repository +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.distribution.TemporaryDirectoryManager +import org.enso.distribution.locking.{ + LockUserInterface, + Resource, + ResourceManager, + ThreadSafeFileLockManager +} import org.enso.editions.Editions -import org.enso.loggingservice.{LogLevel, TestLogger} +import org.enso.librarymanager.published.cache.DownloadingLibraryCache import org.enso.loggingservice.TestLogger.TestLogMessage +import org.enso.loggingservice.{LogLevel, TestLogger} import org.enso.pkg.PackageManager import org.enso.testkit.WithTemporaryDirectory import org.scalatest.matchers.should.Matchers @@ -13,8 +22,7 @@ import java.nio.file.Files class LibraryDownloadTest extends AnyWordSpec with Matchers - with WithTemporaryDirectory - with DownloaderTest { + with WithTemporaryDirectory { val port: Int = 47306 @@ -24,7 +32,32 @@ class LibraryDownloadTest val repoRoot = getTestDirectory.resolve("repo") repo.createRepository(repoRoot) - withDownloader { cache => + val lockManager = + new ThreadSafeFileLockManager(getTestDirectory.resolve("locks")) + val resourceManager = new ResourceManager(lockManager) + try { + val cache = new DownloadingLibraryCache( + cacheRoot = getTestDirectory.resolve("cache"), + temporaryDirectoryManager = new TemporaryDirectoryManager( + getTestDirectory.resolve("tmp"), + resourceManager + ), + resourceManager = resourceManager, + lockUserInterface = new LockUserInterface { + override def startWaitingForResource(resource: Resource): Unit = + println(s"Waiting for ${resource.name}") + + override def finishWaitingForResource(resource: Resource): Unit = + println(s"${resource.name} is ready") + }, + progressReporter = new ProgressReporter { + override def trackProgress( + message: String, + task: TaskProgress[_] + ): Unit = {} + } + ) + val server = repo.startServer(port, repoRoot) try { cache.findCachedLibrary( @@ -62,6 +95,9 @@ class LibraryDownloadTest server.kill(killDescendants = true) server.join(waitForDescendants = true) } + } finally { + resourceManager.releaseMainLock() + resourceManager.unlockTemporaryDirectory() } } } diff --git a/tools/simple-library-server/main.js b/tools/simple-library-server/main.js index 39d056609de6..d33ce4079d0d 100755 --- a/tools/simple-library-server/main.js +++ b/tools/simple-library-server/main.js @@ -1,13 +1,7 @@ #!/usr/bin/env node const express = require("express"); -const path = require("path"); -const os = require("os"); -const fs = require("fs"); -const fsPromises = require("fs/promises"); -const multer = require("multer"); const compression = require("compression"); const yargs = require("yargs"); -const semverValid = require("semver/functions/valid"); const argv = yargs .usage( @@ -25,44 +19,11 @@ const argv = yargs type: "string", default: ".", }) - .option("upload", { - description: - "Specifies whether to allow uploading libraries and which authentication model to choose.", - choices: ["disabled", "no-auth", "constant-token"], - default: "disabled", - }) .help() .alias("help", "h").argv; -const libraryRoot = path.join(argv.root, "libraries"); - const app = express(); -const tmpDir = path.join(os.tmpdir(), "enso-library-repo-uploads"); -const upload = multer({ dest: tmpDir }); app.use(compression({ filter: shouldCompress })); - -/** The token to compare against for simple authentication. - * - * If it is not set, no authentication checks are made. - */ -let token = null; -if (argv.upload == "disabled") { - console.log("Uploads are disabled."); -} else { - app.post("/upload", upload.any(), handleUpload); - - if (argv.upload == "constant-token") { - const envVar = "ENSO_AUTH_TOKEN"; - token = process.env[envVar]; - if (!token) { - throw `${envVar} is not defined.`; - } else { - console.log(`Checking the ${envVar} to authorize requests.`); - } - } else { - console.log("WARNING: Uploads are enabled without any authentication."); - } -} app.use(express.static(argv.root)); console.log( @@ -71,7 +32,6 @@ console.log( app.listen(argv.port); -/// Specifies if a particular file can be compressed in transfer, if supported. function shouldCompress(req, res) { if (req.path.endsWith(".yaml")) { return true; @@ -79,121 +39,3 @@ function shouldCompress(req, res) { return compression.filter(req, res); } - -/** Handles upload of a library. */ -async function handleUpload(req, res) { - function fail(code, message) { - res.status(code).json({ error: message }); - cleanFiles(req.files); - } - - if (token !== null) { - const userToken = req.get("Auth-Token"); - if (userToken != token) { - return fail(403, "Authorization failed."); - } - } - - const version = req.query.version; - const namespace = req.query.namespace; - const name = req.query.name; - - if (version === undefined || namespace == undefined || name === undefined) { - return fail(400, "One or more required fields were missing."); - } - - if (!isVersionValid(version)) { - return fail(400, `Invalid semver version string [${version}].`); - } - - if (!isNamespaceValid(namespace)) { - return fail(400, `Invalid username [${namespace}].`); - } - - if (!isNameValid(name)) { - return fail(400, `Invalid library name [${name}].`); - } - - for (var i = 0; i < req.files.length; ++i) { - const filename = req.files[i].originalname; - if (!isFilenameValid(filename)) { - return fail(400, `Invalid filename: ${filename}.`); - } - } - - const libraryPath = path.join(libraryRoot, namespace, name, version); - - if (fs.existsSync(libraryPath)) { - return fail( - 409, - "A library with the given name and version " + - "combination already exists. Versions are immutable, so you must " + - "bump the library version when uploading a newer version." - ); - } - - await fsPromises.mkdir(libraryPath, { recursive: true }); - - console.log(`Uploading library [${namespace}.${name}:${version}].`); - try { - await putFiles(libraryPath, req.files); - } catch (error) { - console.log(`Upload failed: [${error}].`); - console.error(error.stack); - return fail(500, "Upload failed due to an internal error."); - } - - console.log("Upload complete."); - res.status(200).json({ message: "Successfully uploaded the library." }); -} - -/// Checks if a version complies with the semver specification. -function isVersionValid(version) { - return semverValid(version) !== null; -} - -/// Checks if the namespace/username is valid. -function isNamespaceValid(namespace) { - return /^[a-z][a-z0-9]*$/.test(namespace) && namespace.length >= 3; -} - -/** Checks if the library name is valid. - * - * It may actually accept more identifiers as valid than Enso would, the actual - * check should be done when creating the library. This is just a sanity check - * for safety. - */ -function isNameValid(name) { - return /^[A-Za-z0-9_]+$/.test(name); -} - -// TODO [RW] for now slashes are not permitted to avoid attacks; later on at least the `meta` directory should be allowed, but not much besides that -/// Checks if the uploaded filename is valid. -function isFilenameValid(name) { - return /^[A-Za-z0-9][A-Za-z0-9\._\-]*$/.test(name); -} - -/// Schedules to remove the files, if they still exist. -function cleanFiles(files) { - files.forEach((file) => { - if (fs.existsSync(file.path)) { - fs.unlink(file.path, (err) => { - if (err) { - console.error( - `Failed to remove ${file.path} ($file.originalname) from a failed upload: ${err}.` - ); - } - }); - } - }); -} - -/// Moves the files to the provided destination directory. -async function putFiles(directory, files) { - for (var i = 0; i < files.length; ++i) { - const file = files[i]; - const filename = file.originalname; - const destination = path.join(directory, filename); - await fsPromises.rename(file.path, destination); - } -} diff --git a/tools/simple-library-server/package.json b/tools/simple-library-server/package.json index 77d9c71af66b..14a14e70cdc0 100644 --- a/tools/simple-library-server/package.json +++ b/tools/simple-library-server/package.json @@ -16,8 +16,6 @@ "dependencies": { "compression": "^1.7.4", "express": "^4.17.1", - "multer": "^1.4.2", - "semver": "^7.3.5", "yargs": "^17.0.1" } }