From 630d26c2ab40e13b6c2b99b8c0a25816754e82cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Sat, 26 Jun 2021 18:46:54 +0200 Subject: [PATCH 01/30] Refactor: create separate downloader module, move OS again --- build.sbt | 38 +++++++++++++++++-- .../org/enso/launcher/cli/InternalOpts.scala | 3 +- .../installation/DistributionInstaller.scala | 3 +- .../DistributionUninstaller.scala | 3 +- .../releases/LauncherRepository.scala | 3 +- .../staticwebsite/StaticWebsite.scala | 10 ++--- ...StaticWebsiteFallbackReleaseProvider.scala | 2 +- .../launcher/upgrade/LauncherUpgrader.scala | 6 +-- .../scala/org/enso/launcher/NativeTest.scala | 2 +- .../launcher/installation/InstallerSpec.scala | 3 +- .../installation/UninstallerSpec.scala | 3 +- .../enso/launcher/upgrade/UpgradeSpec.scala | 3 +- .../src/main/scala/org/enso/cli}/OS.scala | 2 +- .../distribution/DistributionManager.scala | 1 + .../org/enso/distribution/Environment.scala | 1 + .../org/enso/distribution/FileSystem.scala | 1 + .../enso/downloader}/archive/Archive.scala | 22 +++++------ .../downloader}/archive/ArchiveEntry.scala | 2 +- .../downloader}/archive/ArchiveFormat.scala | 2 +- .../archive/FileProgressInputStream.scala | 6 +-- .../archive/POSIXPermissions.scala | 2 +- .../archive/internal/ArchiveIterator.scala | 2 +- .../archive/internal/BaseRenamer.scala | 2 +- .../internal/ProgressInputStream.scala | 2 +- .../enso/downloader}/http/HTTPDownload.scala | 7 ++-- .../enso/downloader}/http/HTTPException.scala | 2 +- .../enso/downloader}/http/HTTPRequest.scala | 2 +- .../downloader}/http/HTTPRequestBuilder.scala | 7 +++- .../enso/downloader}/http/URIBuilder.scala | 2 +- .../archive/POSIXPermissionsSpec.scala | 5 ++- .../downloader}/http/HTTPDownloadSpec.scala | 2 +- .../enso/projectmanager/BaseServerSpec.scala | 3 +- .../test/NativeTestHelper.scala | 2 +- .../RuntimeVersionManagerSpec.scala | 3 +- .../components/GraalRuntime.scala | 2 +- .../GraalVMComponentConfiguration.scala | 2 +- .../components/Manifest.scala | 2 +- .../RuntimeComponentConfiguration.scala | 2 +- .../components/RuntimeVersionManager.scala | 5 ++- .../releases/EnsoReleaseProvider.scala | 2 +- .../releases/github/GithubAPI.scala | 2 +- .../graalvm/GraalCEReleaseProvider.scala | 3 +- .../testing/TestArchivePackager.scala | 3 +- .../GraalVMComponentConfigurationSpec.scala | 2 +- .../GraalVMComponentUpdaterSpec.scala | 2 +- 45 files changed, 112 insertions(+), 74 deletions(-) rename lib/scala/{distribution-manager/src/main/scala/org/enso/distribution => cli/src/main/scala/org/enso/cli}/OS.scala (99%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/archive/Archive.scala (98%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/archive/ArchiveEntry.scala (93%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/archive/ArchiveFormat.scala (93%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/archive/FileProgressInputStream.scala (85%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/archive/POSIXPermissions.scala (96%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/archive/internal/ArchiveIterator.scala (95%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/archive/internal/BaseRenamer.scala (96%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader/archive}/internal/ProgressInputStream.scala (97%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/http/HTTPDownload.scala (99%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/http/HTTPException.scala (75%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/http/HTTPRequest.scala (83%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/http/HTTPRequestBuilder.scala (91%) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => downloader/src/main/scala/org/enso/downloader}/http/URIBuilder.scala (97%) rename lib/scala/{runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager => downloader/src/test/scala/org/enso/downloader}/archive/POSIXPermissionsSpec.scala (91%) rename lib/scala/{runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager => downloader/src/test/scala/org/enso/downloader}/http/HTTPDownloadSpec.scala (92%) diff --git a/build.sbt b/build.sbt index 2a1ba226b92d..931a44f34185 100644 --- a/build.sbt +++ b/build.sbt @@ -710,9 +710,10 @@ lazy val cli = project .configs(Test) .settings( version := "0.1", - libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % scalatestVersion % Test, - "org.typelevel" %% "cats-core" % catsVersion + libraryDependencies ++= circe ++ Seq( + "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + "org.typelevel" %% "cats-core" % catsVersion ), Test / parallelExecution := false ) @@ -1337,6 +1338,7 @@ lazy val `distribution-manager` = project ) ) .dependsOn(editions) + .dependsOn(cli) .dependsOn(pkg) .dependsOn(`logging-utils`) @@ -1354,11 +1356,37 @@ lazy val editions = project ) .dependsOn(testkit % Test) +lazy val downloader = (project in file("lib/scala/downloader")) + .settings( + version := "0.1", + libraryDependencies ++= circe ++ Seq( + "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, + "commons-io" % "commons-io" % commonsIoVersion, + "org.apache.commons" % "commons-compress" % commonsCompressVersion, + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + akkaActor, + akkaStream, + akkaHttp + ) + ) + .dependsOn(cli) + +lazy val `edition-updater` = project + .in(file("lib/scala/edition-updater")) + .configs(Test) + .settings( + libraryDependencies ++= Seq( + "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, + "org.scalatest" %% "scalatest" % scalatestVersion % Test + ) + ) + .dependsOn(editions) + .dependsOn(downloader) + lazy val `library-manager` = project .in(file("lib/scala/library-manager")) .configs(Test) .settings( - resolvers += Resolver.bintrayRepo("gn0s1s", "releases"), libraryDependencies ++= Seq( "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, "org.scalatest" %% "scalatest" % scalatestVersion % Test @@ -1368,6 +1396,7 @@ lazy val `library-manager` = project .dependsOn(editions) .dependsOn(cli) .dependsOn(`distribution-manager`) + .dependsOn(downloader) .dependsOn(testkit % Test) lazy val `runtime-version-manager` = project @@ -1385,6 +1414,7 @@ lazy val `runtime-version-manager` = project ) ) .dependsOn(pkg) + .dependsOn(downloader) .dependsOn(`logging-service`) .dependsOn(cli) .dependsOn(`version-output`) diff --git a/engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala b/engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala index b101eb3440ad..e249772c05bf 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala @@ -4,9 +4,10 @@ import java.io.IOException import java.nio.file.{Files, NoSuchFileException, Path} import cats.implicits._ import nl.gn0s1s.bump.SemVer +import org.enso.cli.OS import org.enso.cli.arguments.Opts import org.enso.cli.arguments.Opts.implicits._ -import org.enso.distribution.{FileSystem, OS} +import org.enso.distribution.FileSystem import org.enso.runtimeversionmanager.CurrentVersion import org.enso.distribution.FileSystem.PathSyntax import org.enso.runtimeversionmanager.cli.Arguments._ diff --git a/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionInstaller.scala b/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionInstaller.scala index 2cde4028dae2..bcbdba9678a0 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionInstaller.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionInstaller.scala @@ -2,11 +2,10 @@ package org.enso.launcher.installation import java.nio.file.{Files, Path} import com.typesafe.scalalogging.Logger -import org.enso.cli.CLIOutput +import org.enso.cli.{CLIOutput, OS} import org.enso.distribution.{ DistributionManager, FileSystem, - OS, PortableDistributionManager } import org.enso.distribution.locking.ResourceManager diff --git a/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionUninstaller.scala b/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionUninstaller.scala index 36651980cc9c..20fc0f0b51e3 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionUninstaller.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionUninstaller.scala @@ -3,11 +3,10 @@ package org.enso.launcher.installation import java.nio.file.{Files, Path} import com.typesafe.scalalogging.Logger import org.apache.commons.io.FileUtils -import org.enso.cli.CLIOutput +import org.enso.cli.{CLIOutput, OS} import org.enso.distribution.{ DistributionManager, FileSystem, - OS, PortableDistributionManager } import org.enso.distribution.locking.ResourceManager diff --git a/engine/launcher/src/main/scala/org/enso/launcher/releases/LauncherRepository.scala b/engine/launcher/src/main/scala/org/enso/launcher/releases/LauncherRepository.scala index 50e96ad0bcb1..e74492f6b968 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/releases/LauncherRepository.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/releases/LauncherRepository.scala @@ -1,8 +1,8 @@ package org.enso.launcher.releases import java.nio.file.Path - import com.typesafe.scalalogging.Logger +import org.enso.downloader.http.URIBuilder import org.enso.launcher.distribution.DefaultManagers import org.enso.launcher.releases.fallback.SimpleReleaseProviderWithFallback import org.enso.launcher.releases.fallback.staticwebsite.StaticWebsiteFallbackReleaseProvider @@ -10,7 +10,6 @@ import org.enso.launcher.releases.launcher.{ LauncherRelease, LauncherReleaseProvider } -import org.enso.runtimeversionmanager.http.URIBuilder import org.enso.runtimeversionmanager.releases.engine.EngineRepository import org.enso.runtimeversionmanager.releases.testing.FakeReleaseProvider import org.enso.runtimeversionmanager.releases.{ diff --git a/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsite.scala b/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsite.scala index 0e8bd12c1e3e..6ef2527f4494 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsite.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsite.scala @@ -1,13 +1,9 @@ package org.enso.launcher.releases.fallback.staticwebsite -import java.nio.file.Path - import org.enso.cli.task.TaskProgress -import org.enso.runtimeversionmanager.http.{ - HTTPDownload, - HTTPRequestBuilder, - URIBuilder -} +import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} + +import java.nio.file.Path /** Provides [[FileStorage]] backed by a static HTTPS website. * diff --git a/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsiteFallbackReleaseProvider.scala b/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsiteFallbackReleaseProvider.scala index 82e6a85c6e35..d30cc7e71ac9 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsiteFallbackReleaseProvider.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsiteFallbackReleaseProvider.scala @@ -1,6 +1,6 @@ package org.enso.launcher.releases.fallback.staticwebsite -import org.enso.runtimeversionmanager.http.URIBuilder +import org.enso.downloader.http.URIBuilder import org.enso.runtimeversionmanager.releases.Release import org.enso.launcher.releases.fallback.FallbackReleaseProvider diff --git a/engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala b/engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala index edb5ff488cef..3080e753246d 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala @@ -3,8 +3,8 @@ package org.enso.launcher.upgrade import java.nio.file.{Files, Path} import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer -import org.enso.cli.CLIOutput -import org.enso.distribution.{DistributionManager, FileSystem, OS} +import org.enso.cli.{CLIOutput, OS} +import org.enso.distribution.{DistributionManager, FileSystem} import org.enso.distribution.locking.{ LockType, LockUserInterface, @@ -13,7 +13,7 @@ import org.enso.distribution.locking.{ } import org.enso.runtimeversionmanager.CurrentVersion import org.enso.distribution.FileSystem.PathSyntax -import org.enso.runtimeversionmanager.archive.Archive +import org.enso.downloader.archive.Archive import org.enso.runtimeversionmanager.components.UpgradeRequiredError import org.enso.launcher.cli.{ CLIProgressReporter, diff --git a/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala b/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala index 9f8f871eb26b..2f85ff42e9f2 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala @@ -1,6 +1,6 @@ package org.enso.launcher -import org.enso.distribution.OS +import org.enso.cli.OS import java.nio.file.{Files, Path} import org.enso.runtimeversionmanager.test.NativeTestHelper diff --git a/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala index d0dce631600d..787bc746d7b2 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala @@ -1,9 +1,10 @@ package org.enso.launcher.installation -import org.enso.distribution.{FileSystem, OS} +import org.enso.distribution.FileSystem import java.nio.file.{Files, Path} import FileSystem.PathSyntax +import org.enso.cli.OS import org.enso.runtimeversionmanager.test.WithTemporaryDirectory import org.enso.launcher._ diff --git a/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala index 9cebe1478317..1fc09a274485 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala @@ -1,9 +1,10 @@ package org.enso.launcher.installation -import org.enso.distribution.{FileSystem, OS} +import org.enso.distribution.FileSystem import java.nio.file.{Files, Path} import FileSystem.PathSyntax +import org.enso.cli.OS import org.enso.runtimeversionmanager.test.WithTemporaryDirectory import org.enso.launcher.NativeTest diff --git a/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala index 73175a3e0b57..a79a5b55abcc 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala @@ -3,9 +3,10 @@ package org.enso.launcher.upgrade import java.nio.file.{Files, Path, StandardCopyOption} import io.circe.parser import nl.gn0s1s.bump.SemVer -import org.enso.distribution.{FileSystem, OS} +import org.enso.distribution.FileSystem import org.enso.distribution.locking.{FileLockManager, LockType} import FileSystem.PathSyntax +import org.enso.cli.OS import org.enso.launcher._ import org.enso.runtimeversionmanager.test.WithTemporaryDirectory import org.enso.testkit.RetrySpec diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/OS.scala b/lib/scala/cli/src/main/scala/org/enso/cli/OS.scala similarity index 99% rename from lib/scala/distribution-manager/src/main/scala/org/enso/distribution/OS.scala rename to lib/scala/cli/src/main/scala/org/enso/cli/OS.scala index 3288020dc972..a1e537726e32 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/OS.scala +++ b/lib/scala/cli/src/main/scala/org/enso/cli/OS.scala @@ -1,4 +1,4 @@ -package org.enso.distribution +package org.enso.cli import com.typesafe.scalalogging.Logger import io.circe.{Decoder, DecodingFailure} diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala index 056d029ff6db..ce7a31f5aaa5 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala @@ -1,6 +1,7 @@ package org.enso.distribution import com.typesafe.scalalogging.Logger +import org.enso.cli.OS import org.enso.distribution.FileSystem.PathSyntax import org.enso.logger.masking.{MaskedPath, ToLogString} diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/Environment.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/Environment.scala index a7a4f8a9b92e..eced083686e6 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/Environment.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/Environment.scala @@ -1,6 +1,7 @@ package org.enso.distribution import com.typesafe.scalalogging.Logger +import org.enso.cli.OS import org.enso.logger.masking.MaskedString import java.io.File diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/FileSystem.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/FileSystem.scala index a1edcada6618..271ade1cc279 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/FileSystem.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/FileSystem.scala @@ -2,6 +2,7 @@ package org.enso.distribution import com.typesafe.scalalogging.Logger import org.apache.commons.io.FileUtils +import org.enso.cli.OS import java.io.PrintWriter import java.nio.file.attribute.PosixFilePermissions diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/Archive.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/Archive.scala similarity index 98% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/Archive.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/Archive.scala index baadc5044933..8fc9618bb1f5 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/Archive.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/Archive.scala @@ -1,12 +1,6 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive -import java.io.BufferedInputStream -import java.nio.file.{Files, Path} import com.typesafe.scalalogging.Logger -import org.apache.commons.compress.archivers.{ - ArchiveInputStream, - ArchiveEntry => ApacheArchiveEntry -} import org.apache.commons.compress.archivers.tar.{ TarArchiveEntry, TarArchiveInputStream @@ -15,20 +9,26 @@ import org.apache.commons.compress.archivers.zip.{ ZipArchiveEntry, ZipArchiveInputStream } +import org.apache.commons.compress.archivers.{ + ArchiveInputStream, + ArchiveEntry => ApacheArchiveEntry +} import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream import org.apache.commons.io.IOUtils +import org.enso.cli.OS import org.enso.cli.task.{ ProgressUnit, TaskProgress, TaskProgressImplementation } -import org.enso.distribution.OS -import org.enso.runtimeversionmanager.archive.internal.{ +import org.enso.downloader.archive.internal.{ ArchiveIterator, - BaseRenamer + BaseRenamer, + ReadProgress } -import org.enso.runtimeversionmanager.internal.ReadProgress +import java.io.BufferedInputStream +import java.nio.file.{Files, Path} import scala.util.{Try, Using} /** Contains utilities related to the extraction of various archive file diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveEntry.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveEntry.scala similarity index 93% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveEntry.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveEntry.scala index cdd82a5e78b5..e4f7c05e8de7 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveEntry.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveEntry.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive import java.nio.file.Path diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveFormat.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveFormat.scala similarity index 93% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveFormat.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveFormat.scala index cc2a3fc0fc13..dfd3798c5b6a 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveFormat.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveFormat.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive import java.nio.file.Path diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/FileProgressInputStream.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/FileProgressInputStream.scala similarity index 85% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/FileProgressInputStream.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/FileProgressInputStream.scala index deb8c0470b41..e5dcddb144dd 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/FileProgressInputStream.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/FileProgressInputStream.scala @@ -1,10 +1,10 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive + +import org.enso.downloader.archive.internal.ProgressInputStream import java.io.FileInputStream import java.nio.file.{Files, Path} -import org.enso.runtimeversionmanager.internal.ProgressInputStream - /** A helper that allows to create a [[ProgressInputStream]] for a file located * at the given path. */ diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/POSIXPermissions.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/POSIXPermissions.scala similarity index 96% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/POSIXPermissions.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/POSIXPermissions.scala index 3caee1331183..d3258895c2c4 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/POSIXPermissions.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/POSIXPermissions.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive import java.nio.file.attribute.PosixFilePermission import java.util diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/ArchiveIterator.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ArchiveIterator.scala similarity index 95% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/ArchiveIterator.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ArchiveIterator.scala index 64b25b68e67c..e4d74fdc9e11 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/ArchiveIterator.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ArchiveIterator.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.archive.internal +package org.enso.downloader.archive.internal import org.apache.commons.compress.archivers.{ArchiveEntry, ArchiveInputStream} diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/BaseRenamer.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/BaseRenamer.scala similarity index 96% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/BaseRenamer.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/BaseRenamer.scala index 678ae0670ffd..bc754c8acb16 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/BaseRenamer.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/BaseRenamer.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.archive.internal +package org.enso.downloader.archive.internal import java.nio.file.Path diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/internal/ProgressInputStream.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ProgressInputStream.scala similarity index 97% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/internal/ProgressInputStream.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ProgressInputStream.scala index 10d85b882c05..28c0d1eb1bee 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/internal/ProgressInputStream.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ProgressInputStream.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.internal +package org.enso.downloader.archive.internal import java.io.InputStream diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPDownload.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala similarity index 99% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPDownload.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala index 9c63d0183ea7..42cf2da96e46 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPDownload.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala @@ -1,7 +1,4 @@ -package org.enso.runtimeversionmanager.http - -import java.nio.charset.{Charset, StandardCharsets} -import java.nio.file.Path +package org.enso.downloader.http import akka.actor.ActorSystem import akka.http.scaladsl.Http @@ -17,6 +14,8 @@ import org.enso.cli.task.{ TaskProgressImplementation } +import java.nio.charset.{Charset, StandardCharsets} +import java.nio.file.Path import scala.concurrent.Future /** Represents a HTTP header. */ diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPException.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPException.scala similarity index 75% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPException.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPException.scala index a66cf24c05af..600b0cd7097c 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPException.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPException.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.http +package org.enso.downloader.http /** Indicates an error when processing a HTTP request. */ diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequest.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequest.scala similarity index 83% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequest.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequest.scala index 6578bc3d34d9..134d5e32318e 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequest.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequest.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.http +package org.enso.downloader.http import akka.http.scaladsl.model.HttpRequest diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequestBuilder.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala similarity index 91% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequestBuilder.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala index 5f1b730683d3..f0633d531212 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequestBuilder.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala @@ -1,7 +1,8 @@ -package org.enso.runtimeversionmanager.http +package org.enso.downloader.http import akka.http.scaladsl.model.HttpHeader.ParsingResult import akka.http.scaladsl.model._ +import org.enso.downloader.http /** A simple immutable builder for HTTP requests. * @@ -39,7 +40,9 @@ case class HTTPRequestBuilder private ( ) } } - HTTPRequest(HttpRequest(method = method, uri = uri, headers = httpHeaders)) + http.HTTPRequest( + HttpRequest(method = method, uri = uri, headers = httpHeaders) + ) } } diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/URIBuilder.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/URIBuilder.scala similarity index 97% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/URIBuilder.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/http/URIBuilder.scala index 1f2f3ab81c57..5986b58cdf5d 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/URIBuilder.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/URIBuilder.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.http +package org.enso.downloader.http import akka.http.scaladsl.model.Uri diff --git a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/archive/POSIXPermissionsSpec.scala b/lib/scala/downloader/src/test/scala/org/enso/downloader/archive/POSIXPermissionsSpec.scala similarity index 91% rename from lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/archive/POSIXPermissionsSpec.scala rename to lib/scala/downloader/src/test/scala/org/enso/downloader/archive/POSIXPermissionsSpec.scala index ea2daa488786..8ba829587dd4 100644 --- a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/archive/POSIXPermissionsSpec.scala +++ b/lib/scala/downloader/src/test/scala/org/enso/downloader/archive/POSIXPermissionsSpec.scala @@ -1,7 +1,8 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive -import java.nio.file.attribute.PosixFilePermissions +import org.enso.downloader.archive.POSIXPermissions +import java.nio.file.attribute.PosixFilePermissions import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/http/HTTPDownloadSpec.scala b/lib/scala/downloader/src/test/scala/org/enso/downloader/http/HTTPDownloadSpec.scala similarity index 92% rename from lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/http/HTTPDownloadSpec.scala rename to lib/scala/downloader/src/test/scala/org/enso/downloader/http/HTTPDownloadSpec.scala index 3cbea72525f8..d88c19c37d78 100644 --- a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/http/HTTPDownloadSpec.scala +++ b/lib/scala/downloader/src/test/scala/org/enso/downloader/http/HTTPDownloadSpec.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.http +package org.enso.downloader.http import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/BaseServerSpec.scala b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/BaseServerSpec.scala index 4dd99519e59a..c769ea0531ad 100644 --- a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/BaseServerSpec.scala +++ b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/BaseServerSpec.scala @@ -11,8 +11,9 @@ import io.circe.parser.parse import nl.gn0s1s.bump.SemVer import org.apache.commons.io.FileUtils import org.enso.distribution.FileSystem.PathSyntax -import org.enso.distribution.{FileSystem, OS} +import org.enso.distribution.FileSystem import org.enso.editions.Editions +import org.enso.cli.OS import org.enso.jsonrpc.test.JsonRpcServerTestKit import org.enso.jsonrpc.{ClientControllerFactory, Protocol} import org.enso.loggingservice.printers.StderrPrinterWithColors diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala index 242aea79c10d..e511d9f4ee0c 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.test -import org.enso.distribution.OS +import org.enso.cli.OS import java.io.{ BufferedReader, diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala index 1ca9a80801bd..4fb6561663bd 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala @@ -2,7 +2,8 @@ package org.enso.runtimeversionmanager.components import java.nio.file.{Files, Path} import nl.gn0s1s.bump.SemVer -import org.enso.distribution.{FileSystem, OS} +import org.enso.cli.OS +import org.enso.distribution.FileSystem import org.enso.distribution.FileSystem.PathSyntax import org.enso.runtimeversionmanager.config.GlobalConfigurationManager import org.enso.runtimeversionmanager.releases.ReleaseNotFound diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala index a9a40ed67e37..77ba54d4c82c 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.components -import org.enso.distribution.OS +import org.enso.cli.OS import java.nio.file.{Files, Path} import org.enso.distribution.FileSystem.PathSyntax diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfiguration.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfiguration.scala index d656f34730ce..808e012cde16 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfiguration.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfiguration.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.components -import org.enso.distribution.OS +import org.enso.cli.OS /** Component configuration of the GraalVM distribution. */ class GraalVMComponentConfiguration extends RuntimeComponentConfiguration { diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Manifest.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Manifest.scala index 171e7171fa6a..478a9803a553 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Manifest.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Manifest.scala @@ -5,7 +5,7 @@ import java.nio.file.Path import cats.Show import io.circe.{yaml, Decoder} import nl.gn0s1s.bump.SemVer -import org.enso.distribution.OS +import org.enso.cli.OS import org.enso.editions.SemVerJson._ import org.enso.runtimeversionmanager.components.Manifest.{ JVMOption, diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeComponentConfiguration.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeComponentConfiguration.scala index a7533d0f2d43..f57cd56edb5a 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeComponentConfiguration.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeComponentConfiguration.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.components -import org.enso.distribution.OS +import org.enso.cli.OS /** Provides configuration of the runtime components. */ trait RuntimeComponentConfiguration { diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala index b3a8d8404928..fc46171acc30 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala @@ -3,12 +3,13 @@ package org.enso.runtimeversionmanager.components import java.nio.file.{Files, Path, StandardOpenOption} import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer -import org.enso.distribution.{DistributionManager, FileSystem, OS} +import org.enso.cli.OS +import org.enso.distribution.{DistributionManager, FileSystem} import org.enso.distribution.locking.{LockType, ResourceManager} import org.enso.runtimeversionmanager.CurrentVersion import org.enso.distribution.FileSystem.PathSyntax import org.enso.logger.masking.MaskedPath -import org.enso.runtimeversionmanager.archive.Archive +import org.enso.downloader.archive.Archive import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.locking.Resources import org.enso.runtimeversionmanager.releases.ReleaseProvider diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/EnsoReleaseProvider.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/EnsoReleaseProvider.scala index e9c18dc4b43b..94e8c71cf148 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/EnsoReleaseProvider.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/EnsoReleaseProvider.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.releases import nl.gn0s1s.bump.SemVer -import org.enso.distribution.OS +import org.enso.cli.OS import scala.util.{Failure, Success, Try} diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala index 94ed977a83bf..2d4babd9dfff 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala @@ -4,7 +4,7 @@ import java.nio.file.Path import io.circe._ import io.circe.parser._ import org.enso.cli.task.TaskProgress -import org.enso.runtimeversionmanager.http.{ +import org.enso.downloader.http.{ APIResponse, HTTPDownload, HTTPRequestBuilder, diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala index 15a5d149e8ed..5dc403bc2175 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala @@ -1,8 +1,9 @@ package org.enso.runtimeversionmanager.releases.graalvm +import org.enso.cli.OS + import java.nio.file.Path import org.enso.cli.task.TaskProgress -import org.enso.distribution.OS import org.enso.runtimeversionmanager.components.GraalVMVersion import org.enso.runtimeversionmanager.releases.github.GithubReleaseProvider import org.enso.runtimeversionmanager.releases.{ diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/TestArchivePackager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/TestArchivePackager.scala index c873c868f1df..d9678421c179 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/TestArchivePackager.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/TestArchivePackager.scala @@ -1,6 +1,7 @@ package org.enso.runtimeversionmanager.releases.testing -import org.enso.distribution.{FileSystem, OS} +import org.enso.cli.OS +import org.enso.distribution.FileSystem import java.nio.file.Path import scala.sys.process.Process diff --git a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfigurationSpec.scala b/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfigurationSpec.scala index a835db55d170..1fca23b0cf45 100644 --- a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfigurationSpec.scala +++ b/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfigurationSpec.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.components -import org.enso.distribution.OS +import org.enso.cli.OS import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala b/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala index 9bd888568492..2b54d5109be8 100644 --- a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala +++ b/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.components -import org.enso.distribution.OS +import org.enso.cli.OS import java.nio.file.Path import org.scalatest.matchers.should.Matchers From b11e683a3839934cf0f907046bda3fd17d9da226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Sat, 26 Jun 2021 22:22:03 +0200 Subject: [PATCH 02/30] Implement EditionUpdater --- .../downloader/http/HTTPRequestBuilder.scala | 2 +- .../org/enso/downloader/http/URIBuilder.scala | 15 +++- .../editions/updater/EditionUpdater.scala | 83 +++++++++++++++++++ .../scala/org/enso/editions/EditionName.scala | 6 +- .../enso/editions/repository/Manifest.scala | 5 ++ 5 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala index f0633d531212..f525d20da772 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala @@ -57,5 +57,5 @@ object HTTPRequestBuilder { * builder that will send the request to the given `uri`. */ def fromURIString(uri: String): HTTPRequestBuilder = - fromURI(Uri.parseAbsolute(uri)) + fromURI(URIBuilder.fromUri(uri).build()) } diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/URIBuilder.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/URIBuilder.scala index 5986b58cdf5d..82f7d1a95a22 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/URIBuilder.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/URIBuilder.scala @@ -6,9 +6,6 @@ import akka.http.scaladsl.model.Uri * * It contains very limited functionality that is needed by the APIs used in * the launcher. It can be easily extended if necessary. - * - * As all APIs we use support HTTPS, it does not allow to create a non-HTTPS - * URL. */ case class URIBuilder private (uri: Uri) { @@ -43,6 +40,18 @@ object URIBuilder { def fromHost(host: String): URIBuilder = new URIBuilder(Uri.from(scheme = "https", host = host)) + /** Creates a builder from an arbitrary [[Uri]] instance. */ + def fromUri(uri: Uri): URIBuilder = + new URIBuilder(uri) + + /** Creates a builder from an arbitrary URI represented as string. + * + * If the string is invalid, it throws + * [[akka.http.scaladsl.model.IllegalUriException]]. + */ + def fromUri(uri: String): URIBuilder = + new URIBuilder(Uri.parseAbsolute(uri)) + /** A simple DSL for the URIBuilder. */ implicit class URIBuilderSyntax(builder: URIBuilder) { diff --git a/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala b/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala new file mode 100644 index 000000000000..9e55fc4fea1f --- /dev/null +++ b/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala @@ -0,0 +1,83 @@ +package org.enso.editions.updater + +import com.typesafe.scalalogging.Logger +import org.enso.distribution.FileSystem.PathSyntax +import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} +import org.enso.editions.EditionName +import org.enso.editions.repository.Manifest +import org.enso.yaml.YamlHelper + +import java.nio.file.{Files, Path} +import scala.util.{Failure, Try} + +/** A helper class that handles updating available editions. + * + * It downloads lists of available editions from all sources and downloads any + * missing (thus new) editions. Any editions that are already cached are not + * re-downloaded, as we assume that, once published, the edition is immutable. + * + * If two sources provide an edition with the same name, the one that is first + * on the sources list will take precedence. + * + * @param cachePath the path to the directory that contains the cached editions + * and where the new editions will be downloaded to + * @param sources the list of URLs indicating roots of edition repositories + * that should be queried for new editions + */ +class EditionUpdater(cachePath: Path, sources: Seq[String]) { + private lazy val logger = Logger[EditionUpdater] + + /** Downloads edition lists from the [[sources]] and downloads any missing + * editions to the [[cachePath]]. + * + * If there are errors when processing one of the edition sources or + * downloading the editions, the errors are logged as warnings, but other + * sources proceed as normal. + */ + def updateEditions(): Try[Unit] = Try { + for { + source <- sources + repositoryRoot <- Try { URIBuilder.fromUri(source) } + .recoverWith { error => + logger.warn(s"Failed to parse the source URI [$source]: $error") + Failure(error) + } + manifest <- downloadEditionRepositoryManifest(repositoryRoot) + .recoverWith { error => + logger.warn(s"Failed to fetch editions from [$source]: $error") + Failure(error) + } + edition <- manifest.editions + if !isEditionAlreadyCached(edition) + } { + downloadEdition(repositoryRoot, edition).getOrElse { + logger.warn(s"Failed to download edition [$edition] from [$source].") + } + } + } + + private def downloadEditionRepositoryManifest( + repositoryRoot: URIBuilder + ): Try[Manifest] = + Try { + val uri = repositoryRoot.addPathSegment(Manifest.fileName).build() + val request = HTTPRequestBuilder.fromURI(uri).GET + val response = HTTPDownload.fetchString(request).force() + YamlHelper.parseString[Manifest](response.content).toTry.get + } + + private def downloadEdition( + repositoryRoot: URIBuilder, + editionName: EditionName + ): Try[Unit] = Try { + val destinationPath = cachePath / editionName.toFileName + + val uri = repositoryRoot.addPathSegment(editionName.toFileName).build() + val request = HTTPRequestBuilder.fromURI(uri).GET + + HTTPDownload.download(request, destinationPath).force() + } + + private def isEditionAlreadyCached(editionName: EditionName): Boolean = + Files.exists(cachePath / editionName.toFileName) +} diff --git a/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala b/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala index 6e50577f5a30..a51dbdf27bcc 100644 --- a/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala +++ b/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala @@ -8,7 +8,11 @@ import io.circe.Decoder * unquoted inside of a YAML file, that is treated as a floating point * number, so special care must be taken to correctly parse it. */ -case class EditionName(name: String) extends AnyVal +case class EditionName(name: String) extends AnyVal { + + /** Returns the name of the file that is associated with the edition name. */ + def toFileName: String = s"$name.yaml" +} object EditionName { diff --git a/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala b/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala index b154be295d38..1aa3d42a5e38 100644 --- a/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala +++ b/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala @@ -19,4 +19,9 @@ object Manifest { editions <- json.get[Seq[EditionName]](Fields.editions) } yield Manifest(editions) } + + /** The name of the manifest file that should be present at the root of + * editions repository. + */ + val fileName = "manifest.yaml" } From 6eda2a57c7aa3328bddf4192b123395de9545971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Sat, 26 Jun 2021 23:25:59 +0200 Subject: [PATCH 03/30] WIP downloading libraries --- .../enso/editions/EditionResolverSpec.scala | 6 +-- .../published/LibraryCache.scala | 24 +++++++++ .../published/LibraryDownloader.scala | 52 +++++++++++++++++++ .../repository/LibraryManifest.scala | 5 ++ .../librarymanager/LibraryResolverSpec.scala | 4 +- 5 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryCache.scala create mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala diff --git a/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala b/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala index 7c1e1fa52875..7d17994ccbcc 100644 --- a/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala +++ b/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala @@ -16,7 +16,7 @@ class EditionResolverSpec with Inside with OptionValues { object FakeEditionProvider extends EditionProvider { - val mainRepo = Repository.make("main", "https://example.com/main").get + val mainRepo = Repository("main", "https://example.com/main") val editions: Map[String, Editions.RawEdition] = Map( "2021.0" -> Editions.Raw.Edition( parent = None, @@ -60,7 +60,7 @@ class EditionResolverSpec "EditionResolver" should { "resolve a simple edition" in { - val repo = Repository.make("foo", "http://example.com").get + val repo = Repository("foo", "http://example.com") val edition = Editions.Raw.Edition( parent = None, engineVersion = Some(SemVer(1, 2, 3)), @@ -127,7 +127,7 @@ class EditionResolverSpec } "correctly handle repository shadowing when resolving libraries" in { - val localRepo = Repository.make("main", "http://example.com/local").get + val localRepo = Repository("main", "http://example.com/local") localRepo should not equal FakeEditionProvider.mainRepo diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryCache.scala new file mode 100644 index 000000000000..4ad12c9553e7 --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryCache.scala @@ -0,0 +1,24 @@ +package org.enso.librarymanager.published + +import nl.gn0s1s.bump.SemVer +import org.enso.distribution.FileSystem.PathSyntax +import org.enso.editions.LibraryName + +import java.nio.file.{Files, Path} + +class LibraryCache(root: Path) { + def getLibrary(libraryName: LibraryName, version: SemVer): Option[Path] = { + val path = LibraryCache.resolvePath(root, libraryName, version) + if (Files.exists(path)) Some(path) + else None + } +} + +object LibraryCache { + def resolvePath( + cacheRoot: Path, + libraryName: LibraryName, + libraryVersion: SemVer + ): Path = + cacheRoot / libraryName.namespace / libraryName.name / libraryVersion.toString +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala new file mode 100644 index 000000000000..c8129d7a9c1e --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala @@ -0,0 +1,52 @@ +package org.enso.librarymanager.published + +import nl.gn0s1s.bump.SemVer +import org.enso.cli.task.TaskProgress +import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} +import org.enso.editions.Editions.Repository +import org.enso.editions.LibraryName +import org.enso.librarymanager.published.repository.LibraryManifest +import org.enso.yaml.YamlHelper + +import java.nio.file.Path + +object LibraryDownloader { + // TODO [RW]how do we report progress? + // we could do as described below (but it is rather complex and hard to implement) or just use the ProgressReporter interface and report progress of each sub-package separately - I think this will be much better as it is much easier to implement and in the future we can modify that if need be + // but apart from List[TaskProgress[A]] => TaskProgress[List[A]] we will also need TaskProgress[A] => (A => TaskProgress[B]) => TaskProgress[B] + // I need to discuss how progress reporting should be handled - do we need to combine this all into a single taskprogress instance? (really complicated and will have uneven percentages) or maybe we could change the interface altogether and rely on something like + def downloadLibrary( + name: LibraryName, + version: SemVer, + repository: Repository, + destinationRoot: Path + ): Unit = { + val libraryRoot = resolveLibraryRoot(name, version, repository) + val manifest = downloadManifest(libraryRoot) + val packages = manifest.archives + for (pkg <- packages) { + // TODO filtering + } + + // TODO tmp directory, unpack extract + // TODO locking + // TODO progress reporting + } + + private def resolveLibraryRoot( + name: LibraryName, + version: SemVer, + repository: Repository + ): URIBuilder = URIBuilder + .fromUri(repository.url) + .addPathSegment(name.namespace) + .addPathSegment(name.name) + .addPathSegment(version.toString) + + private def downloadManifest(libraryRoot: URIBuilder): LibraryManifest = { + val uri = libraryRoot.addPathSegment(LibraryManifest.fileName).build() + val request = HTTPRequestBuilder.fromURI(uri).GET + val response = HTTPDownload.fetchString(request).force() + YamlHelper.parseString[LibraryManifest](response.content).toTry.get + } +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala index a711fdb4826a..1ff0a74df116 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala @@ -42,4 +42,9 @@ object LibraryManifest { description = description ) } + + /** The name of the manifest file as included in the directory associated with + * a given library in the library repository. + */ + val fileName = "manifest.yaml" } diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/LibraryResolverSpec.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/LibraryResolverSpec.scala index 2cdc407f1d83..567f1711de1d 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/LibraryResolverSpec.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/LibraryResolverSpec.scala @@ -17,7 +17,7 @@ class LibraryResolverSpec with EitherValue with Inside { "LibraryResolver" should { - val mainRepo = Repository.make("main", "https://example.com/main").get + val mainRepo = Repository("main", "https://example.com/main") val parentEdition = Editions.Resolved.Edition( parent = None, engineVersion = Some(SemVer(0, 0, 0)), @@ -31,7 +31,7 @@ class LibraryResolverSpec ) ) ) - val customRepo = Repository.make("custom", "https://example.com/custom").get + val customRepo = Repository("custom", "https://example.com/custom") val currentEdition = Editions.Resolved.Edition( parent = Some(parentEdition), engineVersion = None, From 323f8cb3cc1a47ac632bd6f13d2199913bb53fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Mon, 28 Jun 2021 17:54:51 +0200 Subject: [PATCH 04/30] Design concurrency model for library cache --- .../published/LibraryCache.scala | 24 -------- .../published/LibraryDownloader.scala | 56 ++++++++----------- .../published/PublishedLibraryProvider.scala | 3 + .../published/cache/LibraryCache.scala | 46 +++++++++++++++ .../published/cache/LibraryResource.scala | 14 +++++ .../repository/RepositoryHelper.scala | 17 ++++++ 6 files changed, 102 insertions(+), 58 deletions(-) delete mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryCache.scala create mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala create mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryCache.scala deleted file mode 100644 index 4ad12c9553e7..000000000000 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryCache.scala +++ /dev/null @@ -1,24 +0,0 @@ -package org.enso.librarymanager.published - -import nl.gn0s1s.bump.SemVer -import org.enso.distribution.FileSystem.PathSyntax -import org.enso.editions.LibraryName - -import java.nio.file.{Files, Path} - -class LibraryCache(root: Path) { - def getLibrary(libraryName: LibraryName, version: SemVer): Option[Path] = { - val path = LibraryCache.resolvePath(root, libraryName, version) - if (Files.exists(path)) Some(path) - else None - } -} - -object LibraryCache { - def resolvePath( - cacheRoot: Path, - libraryName: LibraryName, - libraryVersion: SemVer - ): Path = - cacheRoot / libraryName.namespace / libraryName.name / libraryVersion.toString -} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala index c8129d7a9c1e..97a3447de878 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala @@ -1,7 +1,7 @@ package org.enso.librarymanager.published import nl.gn0s1s.bump.SemVer -import org.enso.cli.task.TaskProgress +import org.enso.cli.task.{ProgressReporter, TaskProgress} import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} import org.enso.editions.Editions.Repository import org.enso.editions.LibraryName @@ -10,43 +10,31 @@ import org.enso.yaml.YamlHelper import java.nio.file.Path -object LibraryDownloader { - // TODO [RW]how do we report progress? - // we could do as described below (but it is rather complex and hard to implement) or just use the ProgressReporter interface and report progress of each sub-package separately - I think this will be much better as it is much easier to implement and in the future we can modify that if need be - // but apart from List[TaskProgress[A]] => TaskProgress[List[A]] we will also need TaskProgress[A] => (A => TaskProgress[B]) => TaskProgress[B] - // I need to discuss how progress reporting should be handled - do we need to combine this all into a single taskprogress instance? (really complicated and will have uneven percentages) or maybe we could change the interface altogether and rely on something like - def downloadLibrary( - name: LibraryName, - version: SemVer, - repository: Repository, - destinationRoot: Path - ): Unit = { - val libraryRoot = resolveLibraryRoot(name, version, repository) - val manifest = downloadManifest(libraryRoot) - val packages = manifest.archives - for (pkg <- packages) { - // TODO filtering - } +class LibraryDownloader(progressReporter: ProgressReporter) { +// def downloadLibrary( +// name: LibraryName, +// version: SemVer, +// repository: Repository, +// destinationRoot: Path +// ): Unit = { +// val libraryRoot = resolveLibraryRoot(name, version, repository) +// val manifest = downloadManifest(libraryRoot) +// val packages = manifest.archives +// for (pkg <- packages) { +// // TODO filtering +// } +// +// // TODO tmp directory, unpack extract +// // TODO locking +// // TODO progress reporting +// } - // TODO tmp directory, unpack extract - // TODO locking - // TODO progress reporting - } - - private def resolveLibraryRoot( - name: LibraryName, - version: SemVer, - repository: Repository - ): URIBuilder = URIBuilder - .fromUri(repository.url) - .addPathSegment(name.namespace) - .addPathSegment(name.name) - .addPathSegment(version.toString) - - private def downloadManifest(libraryRoot: URIBuilder): LibraryManifest = { + def downloadManifest(libraryRoot: URIBuilder): LibraryManifest = { val uri = libraryRoot.addPathSegment(LibraryManifest.fileName).build() val request = HTTPRequestBuilder.fromURI(uri).GET val response = HTTPDownload.fetchString(request).force() YamlHelper.parseString[LibraryManifest](response.content).toTry.get } + + def downloadPac } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala index 3b730c6ff057..5883fc24e94f 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala @@ -7,6 +7,9 @@ import org.enso.editions.{LibraryName, LibraryVersion} import java.nio.file.Path import scala.util.Try +import java.nio.file.Path +import scala.util.Try + /** A provider of published libraries. * * It usually should use some kind of a cache to keep already downloaded diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala index a993e7fb18a5..a8c7f3079a83 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala @@ -67,3 +67,49 @@ object LibraryCache { .resolve(libraryName.name) .resolve(version.toString) } + +// TODO [RW] move this elsewhere +/* Note [Library Cache Concurrency Model] + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * The library cache may be used by multiple instances of the engine and other + * tools running concurrently and so it needs to handle concurrent access + * scenarios. + * + * Currently our tools do not provide a way to uninstall libraries from the + * cache, which will simplify the logic significantly, in the future it may be + * extended to allow for clearing the cache. (Currently the user can just + * manually clean the cache directory when no instances are running.) + * + * Thanks to the mentioned assumption, once a library is present in the cache, + * we can assume that it will not disappear, so we do not need to synchronize + * read access. The only thing that needs to be synchronized is installing + * libraries - to make sure that if two processes try to install the same + * library, only one of them actually performs the action. We also need to be + * sure that when one process checks if the library exists, and if another + * process is in the middle of installing it, it will not yet report it as + * existing (as this could lead to loading an only-partially installed library). + * The primary way of ensuring that will be to install the libraries to a + * temporary cache directory next to the true cache and atomically move it at + * the end of the operation. However as we do not have real guarantees that the + * filesystem move is atomic (although most of the time it should be if it is + * within a single filesystem), we will use locking to ensure consistency. + * Obviously, every client that tries to install a library will acquire a write + * lock for it, so that only one client is actually installing; but also every + * client checking for the existence of the library will briefly acquire a read + * lock for that library - thus if the library is currently being installed, the + * client will need to wait for this read lock until the library installation is + * finished and so will never encounter it in a 'partially installed' state. If + * the library is not installed, the client can release the read lock and + * re-acquire the write lock to try to install it. After acquiring the write + * lock, it should check again if the library is available, to make sure that no + * other process installed it in the meantime. This solution is efficient + * because every library is locked independently and read locks can be acquired + * by multiple clients at the same time, so the synchronization overhead for + * already installed libraries is negligible, and for libraries that need to be + * installed it is too negligible in comparison to the time required to download + * the libraries. + * + * A single LockManager (and its locks directory) should be associated with at + * most one library cache directory, as it makes sense for the distribution to + * have only one cache, so the lock entries are not disambiguated in any way. + */ diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala new file mode 100644 index 000000000000..fa23a78eac0d --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala @@ -0,0 +1,14 @@ +package org.enso.librarymanager.published.cache + +import nl.gn0s1s.bump.SemVer +import org.enso.distribution.locking.Resource +import org.enso.editions.LibraryName + +case class LibraryResource(libraryName: LibraryName, version: SemVer) + extends Resource { + override def name: String = + s"cached-library-${libraryName.qualifiedName}-$version" + override def waitMessage: String = + s"Another Enso instance is currently installing $libraryName ($version), " + + s"so this action must wait until the installation is complete." +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala new file mode 100644 index 000000000000..916200146648 --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala @@ -0,0 +1,17 @@ +package org.enso.librarymanager.published.repository + +import nl.gn0s1s.bump.SemVer +import org.enso.downloader.http.URIBuilder +import org.enso.editions.Editions.Repository +import org.enso.editions.LibraryName + +object RepositoryHelper { + implicit class RepositoryMethods(val repository: Repository) { + def resolveLibraryRoot(name: LibraryName, version: SemVer): URIBuilder = + URIBuilder + .fromUri(repository.url) + .addPathSegment(name.namespace) + .addPathSegment(name.name) + .addPathSegment(version.toString) + } +} From d290fe231751ae6cd3b3c8e62463788eab4aa874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Mon, 28 Jun 2021 20:20:01 +0200 Subject: [PATCH 05/30] checkpoint --- .../published/LibraryDownloader.scala | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala index 97a3447de878..165abbabb01e 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala @@ -1,9 +1,8 @@ package org.enso.librarymanager.published -import nl.gn0s1s.bump.SemVer -import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.cli.task.ProgressReporter +import org.enso.distribution.FileSystem.PathSyntax import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} -import org.enso.editions.Editions.Repository import org.enso.editions.LibraryName import org.enso.librarymanager.published.repository.LibraryManifest import org.enso.yaml.YamlHelper @@ -36,5 +35,24 @@ class LibraryDownloader(progressReporter: ProgressReporter) { YamlHelper.parseString[LibraryManifest](response.content).toTry.get } - def downloadPac + /** Downloads a sub-package and extracts it to the given location, it will + * merge the contents with other packages that were extracted there earlier. + */ + def downloadAndExtractPackage( + libraryName: LibraryName, + libraryRoot: URIBuilder, + packageName: String, + destination: Path + ): Unit = { + val uri = libraryRoot.addPathSegment(packageName).build() + val request = HTTPRequestBuilder.fromURI(uri).GET + // TODO download to temporary location, not to the final destination! + val download = HTTPDownload.download(request, destination / packageName) + progressReporter.trackProgress( + s"Downloading $libraryName ($packageName).", + download + ) + download.force() + // TODO WIP + } } From 12eb550778e6c933ba380348523b23a955febb3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Sat, 17 Jul 2021 18:10:28 +0200 Subject: [PATCH 06/30] update stuff after rebase --- build.sbt | 2 + .../editions/updater/EditionUpdater.scala | 5 +- .../published/PublishedLibraryProvider.scala | 3 - .../cache/DownloadingLibraryCache.scala | 66 +++++++++++++++++++ .../published/cache/LibraryCache.scala | 46 ------------- 5 files changed, 70 insertions(+), 52 deletions(-) create mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala diff --git a/build.sbt b/build.sbt index 931a44f34185..89bf355df7fc 100644 --- a/build.sbt +++ b/build.sbt @@ -242,10 +242,12 @@ lazy val enso = (project in file(".")) runtime, searcher, launcher, + downloader, `runtime-version-manager`, `runtime-version-manager-test`, editions, `distribution-manager`, + `edition-updater`, `library-manager`, syntax.jvm, testkit diff --git a/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala b/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala index 9e55fc4fea1f..683b267fb7eb 100644 --- a/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala +++ b/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala @@ -1,7 +1,6 @@ package org.enso.editions.updater import com.typesafe.scalalogging.Logger -import org.enso.distribution.FileSystem.PathSyntax import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} import org.enso.editions.EditionName import org.enso.editions.repository.Manifest @@ -70,7 +69,7 @@ class EditionUpdater(cachePath: Path, sources: Seq[String]) { repositoryRoot: URIBuilder, editionName: EditionName ): Try[Unit] = Try { - val destinationPath = cachePath / editionName.toFileName + val destinationPath = cachePath.resolve(editionName.toFileName) val uri = repositoryRoot.addPathSegment(editionName.toFileName).build() val request = HTTPRequestBuilder.fromURI(uri).GET @@ -79,5 +78,5 @@ class EditionUpdater(cachePath: Path, sources: Seq[String]) { } private def isEditionAlreadyCached(editionName: EditionName): Boolean = - Files.exists(cachePath / editionName.toFileName) + Files.exists(cachePath.resolve(editionName.toFileName)) } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala index 5883fc24e94f..3b730c6ff057 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala @@ -7,9 +7,6 @@ import org.enso.editions.{LibraryName, LibraryVersion} import java.nio.file.Path import scala.util.Try -import java.nio.file.Path -import scala.util.Try - /** A provider of published libraries. * * It usually should use some kind of a cache to keep already downloaded diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala new file mode 100644 index 000000000000..48071ff72821 --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -0,0 +1,66 @@ +package org.enso.librarymanager.published.cache +import nl.gn0s1s.bump.SemVer +import org.enso.editions.{Editions, LibraryName, LibraryVersion} + +import java.nio.file.Path +import scala.util.Try + +/** A [[LibraryCache]] that will try to download missing libraries. */ +class DownloadingLibraryCache extends LibraryCache { + override def findCachedLibrary( + libraryName: LibraryName, + version: SemVer + ): Option[Path] = ??? + + override def findOrInstallLibrary( + libraryName: LibraryName, + version: SemVer, + recommendedRepository: Editions.Repository, + dependencyResolver: LibraryName => Option[LibraryVersion] + ): Try[Path] = ??? +} + +/* Note [Library Cache Concurrency Model] + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * The library cache may be used by multiple instances of the engine and other + * tools running concurrently and so it needs to handle concurrent access + * scenarios. + * + * Currently our tools do not provide a way to uninstall libraries from the + * cache, which will simplify the logic significantly, in the future it may be + * extended to allow for clearing the cache. (Currently the user can just + * manually clean the cache directory when no instances are running.) + * + * Thanks to the mentioned assumption, once a library is present in the cache, + * we can assume that it will not disappear, so we do not need to synchronize + * read access. The only thing that needs to be synchronized is installing + * libraries - to make sure that if two processes try to install the same + * library, only one of them actually performs the action. We also need to be + * sure that when one process checks if the library exists, and if another + * process is in the middle of installing it, it will not yet report it as + * existing (as this could lead to loading an only-partially installed library). + * The primary way of ensuring that will be to install the libraries to a + * temporary cache directory next to the true cache and atomically move it at + * the end of the operation. However as we do not have real guarantees that the + * filesystem move is atomic (although most of the time it should be if it is + * within a single filesystem), we will use locking to ensure consistency. + * Obviously, every client that tries to install a library will acquire a write + * lock for it, so that only one client is actually installing; but also every + * client checking for the existence of the library will briefly acquire a read + * lock for that library - thus if the library is currently being installed, the + * client will need to wait for this read lock until the library installation is + * finished and so will never encounter it in a 'partially installed' state. If + * the library is not installed, the client can release the read lock and + * re-acquire the write lock to try to install it. After acquiring the write + * lock, it should check again if the library is available, to make sure that no + * other process installed it in the meantime. This solution is efficient + * because every library is locked independently and read locks can be acquired + * by multiple clients at the same time, so the synchronization overhead for + * already installed libraries is negligible, and for libraries that need to be + * installed it is too negligible in comparison to the time required to download + * the libraries. + * + * A single LockManager (and its locks directory) should be associated with at + * most one library cache directory, as it makes sense for the distribution to + * have only one cache, so the lock entries are not disambiguated in any way. + */ diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala index a8c7f3079a83..a993e7fb18a5 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala @@ -67,49 +67,3 @@ object LibraryCache { .resolve(libraryName.name) .resolve(version.toString) } - -// TODO [RW] move this elsewhere -/* Note [Library Cache Concurrency Model] - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * The library cache may be used by multiple instances of the engine and other - * tools running concurrently and so it needs to handle concurrent access - * scenarios. - * - * Currently our tools do not provide a way to uninstall libraries from the - * cache, which will simplify the logic significantly, in the future it may be - * extended to allow for clearing the cache. (Currently the user can just - * manually clean the cache directory when no instances are running.) - * - * Thanks to the mentioned assumption, once a library is present in the cache, - * we can assume that it will not disappear, so we do not need to synchronize - * read access. The only thing that needs to be synchronized is installing - * libraries - to make sure that if two processes try to install the same - * library, only one of them actually performs the action. We also need to be - * sure that when one process checks if the library exists, and if another - * process is in the middle of installing it, it will not yet report it as - * existing (as this could lead to loading an only-partially installed library). - * The primary way of ensuring that will be to install the libraries to a - * temporary cache directory next to the true cache and atomically move it at - * the end of the operation. However as we do not have real guarantees that the - * filesystem move is atomic (although most of the time it should be if it is - * within a single filesystem), we will use locking to ensure consistency. - * Obviously, every client that tries to install a library will acquire a write - * lock for it, so that only one client is actually installing; but also every - * client checking for the existence of the library will briefly acquire a read - * lock for that library - thus if the library is currently being installed, the - * client will need to wait for this read lock until the library installation is - * finished and so will never encounter it in a 'partially installed' state. If - * the library is not installed, the client can release the read lock and - * re-acquire the write lock to try to install it. After acquiring the write - * lock, it should check again if the library is available, to make sure that no - * other process installed it in the meantime. This solution is efficient - * because every library is locked independently and read locks can be acquired - * by multiple clients at the same time, so the synchronization overhead for - * already installed libraries is negligible, and for libraries that need to be - * installed it is too negligible in comparison to the time required to download - * the libraries. - * - * A single LockManager (and its locks directory) should be associated with at - * most one library cache directory, as it makes sense for the distribution to - * have only one cache, so the lock entries are not disambiguated in any way. - */ From 0f5ad31e47db6369522c60a14831a25049c1797e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Sat, 17 Jul 2021 18:22:28 +0200 Subject: [PATCH 07/30] checkpoint --- build.sbt | 2 + .../cache/DownloadingLibraryCache.scala | 81 +++++++++++++++++-- .../published/cache/LibraryCache.scala | 2 +- 3 files changed, 79 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index 89bf355df7fc..f527fcc1c407 100644 --- a/build.sbt +++ b/build.sbt @@ -236,6 +236,8 @@ lazy val enso = (project in file(".")) `task-progress-notifications`, `logging-utils`, `logging-service`, + `logging-truffle-connector`, + `locking-test-helper`, `akka-native`, `version-output`, `engine-runner`, diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala index 48071ff72821..5ed62d01b088 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -1,23 +1,94 @@ package org.enso.librarymanager.published.cache +import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer +import org.enso.cli.task.ProgressReporter +import org.enso.distribution.locking.{ + LockType, + LockUserInterface, + ResourceManager +} import org.enso.editions.{Editions, LibraryName, LibraryVersion} +import org.enso.logger.masking.MaskedPath -import java.nio.file.Path -import scala.util.Try +import java.nio.file.{Files, Path} +import scala.util.{Success, Try} /** A [[LibraryCache]] that will try to download missing libraries. */ -class DownloadingLibraryCache extends LibraryCache { +class DownloadingLibraryCache( + cacheRoot: Path, + resourceManager: ResourceManager, + lockUserInterface: LockUserInterface, + progressReporter: ProgressReporter +) extends LibraryCache { + private val logger = Logger[DownloadingLibraryCache] + + /** @inheritdoc */ override def findCachedLibrary( libraryName: LibraryName, version: SemVer - ): Option[Path] = ??? + ): Option[Path] = { + val path = LibraryCache.resolvePath(cacheRoot, libraryName, version) + resourceManager.withResource( + lockUserInterface, + LibraryResource(libraryName, version), + LockType.Shared + ) { + if (Files.isDirectory(path)) { + logger.trace( + s"Library [$libraryName:$version] found cached at " + + s"[${MaskedPath(path).applyMasking()}]." + ) + Some(path) + } else None + } + } + // TODO dependency resolution should be a whole separate mechanism from this, remove that for now + /** @inheritdoc */ override def findOrInstallLibrary( libraryName: LibraryName, version: SemVer, recommendedRepository: Editions.Repository, dependencyResolver: LibraryName => Option[LibraryVersion] - ): Try[Path] = ??? + ): Try[Path] = { + val _ = progressReporter // TODO + val cached = findCachedLibrary(libraryName, version) + cached match { + case Some(result) => Success(result) + case None => + installLibrary( + libraryName, + version, + recommendedRepository, + dependencyResolver + ) + } + } + + private def installLibrary( + libraryName: LibraryName, + version: SemVer, + recommendedRepository: Editions.Repository, + dependencyResolver: LibraryName => Option[LibraryVersion] + ): Try[Path] = { + logger.trace(s"Trying to install [$libraryName:$version].") + resourceManager.withResource( + lockUserInterface, + LibraryResource(libraryName, version), + LockType.Shared + ) { + val path = LibraryCache.resolvePath(cacheRoot, libraryName, version) + if (Files.exists(path)) { + logger.info( + s"Another process has just installed [$libraryName:$version]." + ) + Success(path) + } else { + // TODO download and install + ??? + } + } + } } /* Note [Library Cache Concurrency Model] diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala index a993e7fb18a5..cabc9bf35486 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala @@ -11,7 +11,7 @@ import scala.util.Try */ trait LibraryCache extends ReadOnlyLibraryCache { - /** Returns the path to the library it is already cached. + /** Returns the path to the library if it is already cached. * * This method should not attempt to download the library if it is missing, * because other providers may have it. From 693df31cda7111cac799d1145a8f2062a775a0ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Mon, 19 Jul 2021 12:46:06 +0200 Subject: [PATCH 08/30] checkpoint: tmp management, repository helpers --- .../distribution/DefaultManagers.scala | 6 +- .../TemporaryDirectoryManager.scala | 39 ++++++++++--- .../enso/downloader/http/HTTPDownload.scala | 42 +++++++++++--- .../DefaultLibraryProvider.scala | 13 +---- .../DefaultPublishedLibraryProvider.scala | 13 ++--- .../published/PublishedLibraryProvider.scala | 5 +- .../cache/DownloadingLibraryCache.scala | 58 ++++++++++++++----- .../published/cache/LibraryCache.scala | 26 +++++++-- .../published/cache/NoOpCache.scala | 10 +++- .../repository/LibraryNotFoundException.scala | 15 +++++ .../repository/RepositoryHelper.scala | 57 +++++++++++++++++- .../DefaultDistributionConfiguration.scala | 8 ++- .../DistributionConfiguration.scala | 8 ++- .../TestDistributionConfiguration.scala | 7 ++- .../test/RuntimeVersionManagerTest.scala | 4 +- .../locking/ConcurrencyTest.scala | 7 ++- .../components/RuntimeVersionManager.scala | 34 ++++++----- 17 files changed, 268 insertions(+), 84 deletions(-) rename lib/scala/{runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager => distribution-manager/src/main/scala/org/enso}/distribution/TemporaryDirectoryManager.scala (66%) create mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryNotFoundException.scala diff --git a/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala b/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala index c8aa1968d40e..0c80405f96c5 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala @@ -1,6 +1,9 @@ package org.enso.launcher.distribution -import org.enso.distribution.PortableDistributionManager +import org.enso.distribution.{ + PortableDistributionManager, + TemporaryDirectoryManager +} import org.enso.distribution.locking.{ ResourceManager, ThreadSafeFileLockManager @@ -16,7 +19,6 @@ import org.enso.runtimeversionmanager.components.{ RuntimeComponentUpdaterFactory, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.engine.EngineRepository import org.enso.runtimeversionmanager.releases.graalvm.GraalCEReleaseProvider diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/TemporaryDirectoryManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala similarity index 66% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/TemporaryDirectoryManager.scala rename to lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala index f399e2e3260e..48ce8612b262 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/TemporaryDirectoryManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala @@ -1,22 +1,47 @@ -package org.enso.runtimeversionmanager.distribution +package org.enso.distribution -import java.nio.file.{Files, Path} import com.typesafe.scalalogging.Logger -import org.enso.distribution.{DistributionManager, FileSystem} import org.enso.distribution.locking.ResourceManager -/** Manages safe access to the temporary directory. +import java.nio.file.{FileAlreadyExistsException, Files, Path} +import scala.util.Random + +/** Manages safe access to the local temporary directory. * * The temporary directory is created on demand and automatically removed if it * is empty. Temporary files from previous runs are removed when the temporary * directory is first accessed. Locking mechanism is used to ensure that the * old files are no longer used by any other instances running in parallel. + * + * The local temporary directory is located inside of ENSO_DATA_ROOT, which + * means it should be on the same disk partition as directories keeping + * engines, runtimes and cached libraries. + * + * This directory is used as a destination for extracting component packages, + * so that they can be moved all at once at the last step of the installation. */ class TemporaryDirectoryManager( distribution: DistributionManager, resourceManager: ResourceManager ) { private val logger = Logger[TemporaryDirectoryManager] + private val random = new Random() + + /** Creates a unique temporary subdirectory. */ + def temporarySubdirectory(prefix: String = ""): Path = { + val path = + safeTemporaryDirectory.resolve(prefix + random.nextInt().toString) + if (Files.exists(path)) + temporarySubdirectory(prefix) + else { + try { + Files.createDirectory(path) + } catch { + case _: FileAlreadyExistsException => + temporarySubdirectory(prefix) + } + } + } /** Returns path to a directory for storing temporary files that is located on * the same filesystem as `runtimes` and `engines`. @@ -29,11 +54,11 @@ class TemporaryDirectoryManager( * should ensure that). If that fails, it is also cleaned before any future * accesses. */ - def accessTemporaryDirectory(): Path = safeTemporaryDirectory - private lazy val safeTemporaryDirectory = { resourceManager.startUsingTemporaryDirectory() - distribution.paths.unsafeTemporaryDirectory + val path = distribution.paths.unsafeTemporaryDirectory + Files.createDirectories(path) + path } /** Tries to clean the temporary files directory. diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala index 42cf2da96e46..9c01f1f3ebf8 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala @@ -3,7 +3,7 @@ package org.enso.downloader.http import akka.actor.ActorSystem import akka.http.scaladsl.Http import akka.http.scaladsl.model.headers.Location -import akka.http.scaladsl.model.{HttpRequest, HttpResponse} +import akka.http.scaladsl.model.{HttpRequest, HttpResponse, Uri} import akka.stream.scaladsl.{FileIO, Sink} import akka.util.ByteString import com.typesafe.config.{ConfigFactory, ConfigValueFactory} @@ -35,15 +35,13 @@ case class Header(name: String, value: String) { * @param content the response decoded as a string * @param headers sequence of headers included in the response */ -case class APIResponse(content: String, headers: Seq[Header]) +case class APIResponse(content: String, headers: Seq[Header], statusCode: Int) -/** Contains utility functions for fetching data using the HTTP(S) protocol. - */ +/** Contains utility functions for fetching data using the HTTP(S) protocol. */ object HTTPDownload { private val logger = Logger[HTTPDownload.type] - /** Determines how many redirects are taken until an error is thrown. - */ + /** Determines how many redirects are taken until an error is thrown. */ val maximumRedirects: Int = 20 /** Fetches the `request` and tries to decode is as a [[String]]. @@ -79,11 +77,26 @@ object HTTPDownload { (response, chunks: Seq[ByteString]) => APIResponse( combineChunks(chunks), - response.headers.map(header => Header(header.name, header.value)) + response.headers.map(header => Header(header.name, header.value)), + response.status.intValue() ) ) } + /** Fetches the `uri` and tries to decode is as a [[String]]. + * + * It is a shorthand for the other variant of [[fetchString]] that creates a + * simple GET request from the URI. + * + * @param uri the URI to query + * @return a [[TaskProgress]] that tracks progress of the download and can + * be used to get the final result + */ + def fetchString(uri: Uri): TaskProgress[APIResponse] = { + val request = HTTPRequestBuilder.fromURI(uri).GET + fetchString(request) + } + /** Downloads the `request` and saves the response in the file pointed by the * `destination`. * @@ -120,6 +133,21 @@ object HTTPDownload { ) } + /** Downloads the `uri` and saves the response in the file pointed by the + * `destination`. + * + * It is a shorthand for the other variant of [[download]] that creates a + * simple GET request from the URI. + * + * @param uri the uri to download + * @return a [[TaskProgress]] that tracks progress of the download and can + * be used to wait for the completion of the download. + */ + def download(uri: Uri, destination: Path): TaskProgress[Path] = { + val request = HTTPRequestBuilder.fromURI(uri).GET + download(request, destination) + } + implicit private lazy val actorSystem: ActorSystem = { val config = ConfigFactory .load() diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala index fbb0fb2b3dbc..7fb7ac35b6d8 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala @@ -95,19 +95,8 @@ class DefaultLibraryProvider( } case Right(version @ LibraryVersion.Published(semver, repository)) => - val dependencyResolver = { name: LibraryName => - resolver - .resolveLibraryVersion(name, edition, preferLocalLibraries) - .toOption - } - publishedLibraryProvider - .findLibrary( - libraryName, - semver, - repository, - dependencyResolver - ) + .findLibrary(libraryName, semver, repository) .map(ResolvedLibrary(libraryName, version, _)) .toEither .left diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala index ba8bd0ab3b78..2d5a603c7d2a 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala @@ -2,7 +2,7 @@ package org.enso.librarymanager.published import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer -import org.enso.editions.{Editions, LibraryName, LibraryVersion} +import org.enso.editions.{Editions, LibraryName} import org.enso.librarymanager.published.cache.{ LibraryCache, ReadOnlyLibraryCache @@ -42,8 +42,7 @@ class DefaultPublishedLibraryProvider( override def findLibrary( libraryName: LibraryName, version: SemVer, - recommendedRepository: Editions.Repository, - dependencyResolver: LibraryName => Option[LibraryVersion] + recommendedRepository: Editions.Repository ): Try[Path] = { val cached = findCached(libraryName, version, caches) cached.map(Success(_)).getOrElse { @@ -51,12 +50,8 @@ class DefaultPublishedLibraryProvider( s"$libraryName was not found in any caches, it will need to be " + s"downloaded." ) - primaryCache.findOrInstallLibrary( - libraryName, - version, - recommendedRepository, - dependencyResolver - ) + primaryCache + .findOrInstallLibrary(libraryName, version, recommendedRepository) } } } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala index 3b730c6ff057..c224116f111e 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala @@ -2,7 +2,7 @@ package org.enso.librarymanager.published import nl.gn0s1s.bump.SemVer import org.enso.editions.Editions.Repository -import org.enso.editions.{LibraryName, LibraryVersion} +import org.enso.editions.LibraryName import java.nio.file.Path import scala.util.Try @@ -23,7 +23,6 @@ trait PublishedLibraryProvider { def findLibrary( libraryName: LibraryName, version: SemVer, - recommendedRepository: Repository, - dependencyResolver: LibraryName => Option[LibraryVersion] + recommendedRepository: Repository ): Try[Path] } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala index 5ed62d01b088..4565578e0d78 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -7,15 +7,30 @@ import org.enso.distribution.locking.{ LockUserInterface, ResourceManager } +import org.enso.downloader.http.{HTTPDownload, URIBuilder} import org.enso.editions.{Editions, LibraryName, LibraryVersion} +import org.enso.librarymanager.published.repository.LibraryManifest +import org.enso.librarymanager.published.repository.RepositoryHelper.{ + manifestFilename, + RepositoryMethods +} import org.enso.logger.masking.MaskedPath +import org.enso.yaml.YamlHelper import java.nio.file.{Files, Path} import scala.util.{Success, Try} -/** A [[LibraryCache]] that will try to download missing libraries. */ +/** A [[LibraryCache]] that will try to download missing libraries. + * + * @param cacheRoot + * @param temporaryDirectoryRoot + * @param resourceManager + * @param lockUserInterface + * @param progressReporter + */ class DownloadingLibraryCache( cacheRoot: Path, + temporaryDirectoryRoot: Path, resourceManager: ResourceManager, lockUserInterface: LockUserInterface, progressReporter: ProgressReporter @@ -43,34 +58,26 @@ class DownloadingLibraryCache( } } - // TODO dependency resolution should be a whole separate mechanism from this, remove that for now /** @inheritdoc */ override def findOrInstallLibrary( libraryName: LibraryName, version: SemVer, - recommendedRepository: Editions.Repository, - dependencyResolver: LibraryName => Option[LibraryVersion] + recommendedRepository: Editions.Repository ): Try[Path] = { val _ = progressReporter // TODO val cached = findCachedLibrary(libraryName, version) cached match { case Some(result) => Success(result) case None => - installLibrary( - libraryName, - version, - recommendedRepository, - dependencyResolver - ) + installLibrary(libraryName, version, recommendedRepository) } } private def installLibrary( libraryName: LibraryName, version: SemVer, - recommendedRepository: Editions.Repository, - dependencyResolver: LibraryName => Option[LibraryVersion] - ): Try[Path] = { + recommendedRepository: Editions.Repository + ): Try[Path] = Try { logger.trace(s"Trying to install [$libraryName:$version].") resourceManager.withResource( lockUserInterface, @@ -84,11 +91,36 @@ class DownloadingLibraryCache( ) Success(path) } else { + val access = recommendedRepository.accessLibrary(libraryName, version) + val manifestDownload = access.downloadManifest() + progressReporter.trackProgress( + s"Downloading library manifest of [$libraryName].", + manifestDownload + ) + val manifest = manifestDownload.force() + // TODO download and install ??? } } } + + private def shouldDownloadArchive(archiveName: String): Boolean = { + val isTestData = archiveName.startsWith("tests") + !isTestData + } + + override def preinstallLibrary( + libraryName: LibraryName, + version: SemVer, + recommendedRepository: Editions.Repository, + dependencyResolver: LibraryName => Option[LibraryVersion] + ): Try[Unit] = { + logger.warn("Predownloading dependencies is not yet implemented.") + val _ = dependencyResolver + findOrInstallLibrary(libraryName, version, recommendedRepository) + .map(_ => ()) + } } /* Note [Library Cache Concurrency Model] diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala index cabc9bf35486..ef28655277f8 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala @@ -31,23 +31,39 @@ trait LibraryCache extends ReadOnlyLibraryCache { /** If the cache contains the library, it is returned immediately, otherwise, * it tries to download the missing library. * + * It does not need to install the library's dependencies. However once the + * library is being compiled, installation of its dependencies will be + * triggered automatically by the compiler. + * * @param libraryName the name of the library to search for * @param version the library version * @param recommendedRepository the repository that should be used to * download the library from, if it is missing - * @param dependencyResolver a function that will specify what versions of - * dependencies should be also downloaded when - * installing the missing library (if any) - * TODO [RW] the design of this function should be refined in #1772 * @return the path to the library or a failure if the library could not be * installed */ def findOrInstallLibrary( + libraryName: LibraryName, + version: SemVer, + recommendedRepository: Editions.Repository + ): Try[Path] + + /** Ensures that the given library and all of its dependencies are installed. + * + * @param libraryName the name of the library to search for + * @param version the library version + * @param recommendedRepository the repository that should be used to + * download the library from, if it is missing + * @param dependencyResolver a function that will specify what versions of + * dependencies should be also downloaded when + * installing the missing library (if any) + */ + def preinstallLibrary( libraryName: LibraryName, version: SemVer, recommendedRepository: Editions.Repository, dependencyResolver: LibraryName => Option[LibraryVersion] - ): Try[Path] + ): Try[Unit] } object LibraryCache { diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala index c7625f6d1f0e..425cdc6defa7 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala @@ -20,11 +20,19 @@ class NoOpCache extends LibraryCache { /** @inheritdoc */ override def findOrInstallLibrary( + libraryName: LibraryName, + version: SemVer, + recommendedRepository: Editions.Repository + ): Try[Path] = Failure( + new NotImplementedError("Downloading libraries is not yet implemented.") + ) + + override def preinstallLibrary( libraryName: LibraryName, version: SemVer, recommendedRepository: Editions.Repository, dependencyResolver: LibraryName => Option[LibraryVersion] - ): Try[Path] = Failure( + ): Try[Unit] = Failure( new NotImplementedError("Downloading libraries is not yet implemented.") ) } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryNotFoundException.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryNotFoundException.scala new file mode 100644 index 000000000000..848ea4ad9deb --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryNotFoundException.scala @@ -0,0 +1,15 @@ +package org.enso.librarymanager.published.repository + +import nl.gn0s1s.bump.SemVer +import org.enso.editions.LibraryName + +sealed class LibraryDownloadFailure(message: String) + extends RuntimeException(message) + +case class LibraryNotFoundException( + libraryName: LibraryName, + version: SemVer, + uri: String +) extends LibraryDownloadFailure( + s"Library [$libraryName:$version] was not found at [$uri]." + ) diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala index 916200146648..c565fbce20c9 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala @@ -1,11 +1,59 @@ package org.enso.librarymanager.published.repository import nl.gn0s1s.bump.SemVer -import org.enso.downloader.http.URIBuilder +import org.enso.cli.task.TaskProgress +import org.enso.downloader.http.{APIResponse, HTTPDownload, URIBuilder} import org.enso.editions.Editions.Repository import org.enso.editions.LibraryName +import org.enso.yaml.YamlHelper + +import java.nio.file.Path +import scala.util.{Failure, Success} object RepositoryHelper { + class LibraryAccess( + libraryName: LibraryName, + version: SemVer, + libraryRoot: URIBuilder + ) { + def downloadManifest(): TaskProgress[LibraryManifest] = { + val url = (libraryRoot / manifestFilename).build() + HTTPDownload.fetchString(url).flatMap { response => + response.statusCode match { + case 200 => + YamlHelper.parseString[LibraryManifest](response.content).toTry + case 404 => + Failure( + LibraryNotFoundException(libraryName, version, url.toString) + ) + case _ => + Failure( + new LibraryDownloadFailure("Could not download the manifest") + ) + } + } + } + + private def downloadArtifact( + artifactName: String, + destination: Path + ): TaskProgress[Unit] = { + val url = (libraryRoot / artifactName).build() + HTTPDownload.download(url, destination).map(_ => ()) + } + + def downloadLicense(destination: Path): TaskProgress[Unit] = + downloadArtifact(licenseFilename, destination) + + def downloadPackageConfig(destination: Path): TaskProgress[Unit] = + downloadArtifact(packageFileName, destination) + + def downloadArchive( + archiveName: String, + destination: Path + ): TaskProgress[Unit] = downloadArtifact(archiveName, destination) + } + implicit class RepositoryMethods(val repository: Repository) { def resolveLibraryRoot(name: LibraryName, version: SemVer): URIBuilder = URIBuilder @@ -13,5 +61,12 @@ object RepositoryHelper { .addPathSegment(name.namespace) .addPathSegment(name.name) .addPathSegment(version.toString) + + def accessLibrary(name: LibraryName, version: SemVer): LibraryAccess = + new LibraryAccess(resolveLibraryRoot(name, version)) } + + val manifestFilename = "manifest.yaml" + val licenseFilename = "LICENSE.md" + val packageFileName = "package.yaml" } diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala index 832ce6b7e4c9..abfc3dcaca19 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala @@ -1,7 +1,12 @@ package org.enso.projectmanager.versionmanagement import com.typesafe.scalalogging.LazyLogging -import org.enso.distribution.{DistributionManager, EditionManager, Environment} +import org.enso.distribution.{ + DistributionManager, + EditionManager, + Environment, + TemporaryDirectoryManager +} import org.enso.distribution.locking.{ ResourceManager, ThreadSafeFileLockManager @@ -14,7 +19,6 @@ import org.enso.runtimeversionmanager.components.{ RuntimeVersionManagementUserInterface, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.ReleaseProvider import org.enso.runtimeversionmanager.releases.engine.{ EngineRelease, diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DistributionConfiguration.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DistributionConfiguration.scala index 970b21f236a3..cb8dac791d80 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DistributionConfiguration.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DistributionConfiguration.scala @@ -1,12 +1,16 @@ package org.enso.projectmanager.versionmanagement import org.enso.distribution.locking.ResourceManager -import org.enso.distribution.{DistributionManager, EditionManager, Environment} +import org.enso.distribution.{ + DistributionManager, + EditionManager, + Environment, + TemporaryDirectoryManager +} import org.enso.runtimeversionmanager.components.{ RuntimeVersionManagementUserInterface, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.ReleaseProvider import org.enso.runtimeversionmanager.releases.engine.EngineRelease import org.enso.runtimeversionmanager.runner.JVMSettings diff --git a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala index 24564a78d081..7df57cf139f0 100644 --- a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala @@ -1,6 +1,10 @@ package org.enso.projectmanager -import org.enso.distribution.{DistributionManager, EditionManager} +import org.enso.distribution.{ + DistributionManager, + EditionManager, + TemporaryDirectoryManager +} import org.enso.distribution.locking.ResourceManager import java.nio.file.Path @@ -11,7 +15,6 @@ import org.enso.runtimeversionmanager.components.{ RuntimeVersionManagementUserInterface, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.engine.{ EngineRelease, EngineReleaseProvider diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala index f22e491f17d3..c4162d655a66 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala @@ -4,7 +4,8 @@ import nl.gn0s1s.bump.SemVer import org.enso.distribution.{ DistributionManager, Environment, - PortableDistributionManager + PortableDistributionManager, + TemporaryDirectoryManager } import org.enso.pkg.{Config, PackageManager} import org.enso.runtimeversionmanager.components.{ @@ -13,7 +14,6 @@ import org.enso.runtimeversionmanager.components.{ RuntimeVersionManagementUserInterface, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.engine.EngineReleaseProvider import org.enso.runtimeversionmanager.releases.graalvm.GraalVMRuntimeReleaseProvider import org.scalatest.OptionValues diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala index 2bb651b48f65..0335f3524c90 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala @@ -3,7 +3,11 @@ package org.enso.distribution.locking import java.nio.file.{Files, Path} import nl.gn0s1s.bump.SemVer import org.enso.cli.task.TaskProgress -import org.enso.distribution.{DistributionManager, FileSystem} +import org.enso.distribution.{ + DistributionManager, + FileSystem, + TemporaryDirectoryManager +} import org.enso.distribution.locking.{ LockManager, LockType, @@ -19,7 +23,6 @@ import org.enso.runtimeversionmanager.components.{ Manifest, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.engine.{ EngineRelease, EngineReleaseProvider diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala index fc46171acc30..b1fbd3613d84 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala @@ -4,13 +4,16 @@ import java.nio.file.{Files, Path, StandardOpenOption} import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer import org.enso.cli.OS -import org.enso.distribution.{DistributionManager, FileSystem} +import org.enso.distribution.{ + DistributionManager, + FileSystem, + TemporaryDirectoryManager +} import org.enso.distribution.locking.{LockType, ResourceManager} import org.enso.runtimeversionmanager.CurrentVersion import org.enso.distribution.FileSystem.PathSyntax import org.enso.logger.masking.MaskedPath import org.enso.downloader.archive.Archive -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.locking.Resources import org.enso.runtimeversionmanager.releases.ReleaseProvider import org.enso.runtimeversionmanager.releases.engine.EngineRelease @@ -430,9 +433,9 @@ class RuntimeVersionManager( ) } } - FileSystem.withTemporaryDirectory("enso-install") { directory => - logger.debug("Downloading packages to [{}].", directory) - val enginePackage = directory / engineRelease.packageFileName + FileSystem.withTemporaryDirectory("enso-install") { globalTmpDirectory => + logger.debug("Downloading packages to [{}].", globalTmpDirectory) + val enginePackage = globalTmpDirectory / engineRelease.packageFileName val downloadTask = engineRelease.downloadPackage(enginePackage) userInterface.trackProgress( s"Downloading ${enginePackage.getFileName}.", @@ -443,18 +446,19 @@ class RuntimeVersionManager( val engineDirectoryName = engineDirectoryNameForVersion(engineRelease.version) + val localTmpDirectory = + temporaryDirectoryManager.temporarySubdirectory(s"engine-$version") + val extractionTask = Archive .extractArchive( enginePackage, - temporaryDirectoryManager.accessTemporaryDirectory(), + localTmpDirectory, Some(engineDirectoryName) ) userInterface.trackProgress("Extracting the engine.", extractionTask) extractionTask.force() - val engineTemporaryPath = - temporaryDirectoryManager - .accessTemporaryDirectory() / engineDirectoryName + val engineTemporaryPath = localTmpDirectory / engineDirectoryName def undoTemporaryEngine(): Unit = { if (Files.exists(engineTemporaryPath)) { FileSystem.removeDirectory(engineTemporaryPath) @@ -695,19 +699,21 @@ class RuntimeVersionManager( downloadTask.force() val runtimeDirectoryName = graalDirectoryForVersion(runtimeVersion) + val localTmpDirectory = + temporaryDirectoryManager.temporarySubdirectory( + s"runtime-${runtimeVersion.graalVersion}-java${runtimeVersion.java}" + ) val extractionTask = Archive.extractArchive( runtimePackage, - temporaryDirectoryManager.accessTemporaryDirectory(), + localTmpDirectory, Some(runtimeDirectoryName) ) logger.debug("Extracting [{}].", runtimePackage) userInterface.trackProgress("Extracting the runtime.", extractionTask) extractionTask.force() - val runtimeTemporaryPath = - temporaryDirectoryManager - .accessTemporaryDirectory() / runtimeDirectoryName + val runtimeTemporaryPath = localTmpDirectory / runtimeDirectoryName def undoTemporaryRuntime(): Unit = { if (Files.exists(runtimeTemporaryPath)) { @@ -843,7 +849,7 @@ class RuntimeVersionManager( */ private def safelyRemoveComponent(path: Path): Unit = { val temporaryPath = - temporaryDirectoryManager.accessTemporaryDirectory() / path.getFileName + temporaryDirectoryManager.temporarySubdirectory(path.getFileName.toString) FileSystem.atomicMove(path, temporaryPath) FileSystem.removeDirectory(temporaryPath) } From 0333ba60192ab36244700d22e7c0a8775f901b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Mon, 19 Jul 2021 17:33:44 +0200 Subject: [PATCH 09/30] checkpoint --- .../cache/DownloadingLibraryCache.scala | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala index 4565578e0d78..3960cd119479 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -2,6 +2,7 @@ package org.enso.librarymanager.published.cache import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer import org.enso.cli.task.ProgressReporter +import org.enso.distribution.FileSystem import org.enso.distribution.locking.{ LockType, LockUserInterface, @@ -98,9 +99,12 @@ class DownloadingLibraryCache( manifestDownload ) val manifest = manifestDownload.force() - - // TODO download and install - ??? + // See [Temporary Directories for Installation] + FileSystem.withTemporaryDirectory(s"enso-$libraryName") { + globalTmpDir => + // TODO download and install + ??? + } } } } @@ -167,3 +171,26 @@ class DownloadingLibraryCache( * most one library cache directory, as it makes sense for the distribution to * have only one cache, so the lock entries are not disambiguated in any way. */ + +/* Note [Temporary Directories for Installation] + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * When installing libraries (however the system is very similar for engines and + * runtimes too), we extract the files to a temporary directory, to minimize the + * risk of another process seeing a library in an invalid semi-installed state. + * To achieve that, we extract the archives into a temporary directory that + * resides next to the actual libraries directory, to ensure that they are on + * the same disk partition - this way, after the extraction is complete, the + * move from the temporary location to the final location is likely to be + * atomic. (However even if the move is not atomic, everything should be + * correct, as we also use file locks.) + * + * The temporary directory that is next to the destination directory is called + * local temporary directory. + * + * However we also need a place to download the archives too, and as this place + * does not necessarily need to be on the same partition as the destination, we + * can use the default system-provided temporary directory. This one is called + * the global temporary directory. The benefit of using it is that it is usually + * automatically cleaned by the OS, so we do not need to be as careful about + * cleanup in failure scenarios as with the local temporary directory. + */ From 23ecabab639d54a2e6bdccc9613c6c70b6cccd4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Mon, 19 Jul 2021 19:07:37 +0200 Subject: [PATCH 10/30] Implement library installation --- .../cache/DownloadingLibraryCache.scala | 147 +++++++++++++++--- .../repository/RepositoryHelper.scala | 19 +-- 2 files changed, 137 insertions(+), 29 deletions(-) diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala index 3960cd119479..1f460b40c539 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -1,24 +1,26 @@ package org.enso.librarymanager.published.cache import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer -import org.enso.cli.task.ProgressReporter -import org.enso.distribution.FileSystem +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.distribution.FileSystem.PathSyntax import org.enso.distribution.locking.{ LockType, LockUserInterface, ResourceManager } -import org.enso.downloader.http.{HTTPDownload, URIBuilder} +import org.enso.distribution.{FileSystem, TemporaryDirectoryManager} +import org.enso.downloader.archive.Archive import org.enso.editions.{Editions, LibraryName, LibraryVersion} import org.enso.librarymanager.published.repository.LibraryManifest import org.enso.librarymanager.published.repository.RepositoryHelper.{ - manifestFilename, + LibraryAccess, RepositoryMethods } import org.enso.logger.masking.MaskedPath -import org.enso.yaml.YamlHelper +import org.enso.pkg.PackageManager import java.nio.file.{Files, Path} +import scala.util.control.NonFatal import scala.util.{Success, Try} /** A [[LibraryCache]] that will try to download missing libraries. @@ -31,7 +33,7 @@ import scala.util.{Success, Try} */ class DownloadingLibraryCache( cacheRoot: Path, - temporaryDirectoryRoot: Path, + temporaryDirectoryManager: TemporaryDirectoryManager, resourceManager: ResourceManager, lockUserInterface: LockUserInterface, progressReporter: ProgressReporter @@ -85,30 +87,135 @@ class DownloadingLibraryCache( LibraryResource(libraryName, version), LockType.Shared ) { - val path = LibraryCache.resolvePath(cacheRoot, libraryName, version) - if (Files.exists(path)) { + val cachedLibraryPath = + LibraryCache.resolvePath(cacheRoot, libraryName, version) + if (Files.exists(cachedLibraryPath)) { logger.info( s"Another process has just installed [$libraryName:$version]." ) - Success(path) + cachedLibraryPath } else { - val access = recommendedRepository.accessLibrary(libraryName, version) - val manifestDownload = access.downloadManifest() - progressReporter.trackProgress( - s"Downloading library manifest of [$libraryName].", - manifestDownload - ) - val manifest = manifestDownload.force() + val access = recommendedRepository.accessLibrary(libraryName, version) + val manifest = downloadManifest(libraryName, access) + // See [Temporary Directories for Installation] - FileSystem.withTemporaryDirectory(s"enso-$libraryName") { - globalTmpDir => - // TODO download and install - ??? + val localTmpDir = temporaryDirectoryManager.temporarySubdirectory( + s"$libraryName-$version" + ) + + try { + downloadLooseFiles(libraryName, version, access, localTmpDir) + downloadAndExtractArchives(libraryName, access, manifest, localTmpDir) + verifyPackageIntegrity(localTmpDir) + + FileSystem.atomicMove( + source = localTmpDir, + destination = cachedLibraryPath + ) + + cachedLibraryPath + } catch { + case NonFatal(exception) => + logger.error( + s"Installation of library [$libraryName:$version] failed with " + + s"error: [$exception].", + exception + ) + FileSystem.removeDirectoryIfExists(localTmpDir) + throw exception } } } } + private def downloadManifest( + libraryName: LibraryName, + access: LibraryAccess + ): LibraryManifest = { + val manifestDownload = access.downloadManifest() + progressReporter.trackProgress( + s"Downloading library manifest of [$libraryName].", + manifestDownload + ) + manifestDownload.force() + } + + /** Verifies that the downloaded package can even be loaded. + * + * For now it only checks if the `package.yaml` file is not corrupted. + * + * In the future, additional checks, like checksums, could be added. + */ + private def verifyPackageIntegrity(packageRoot: Path): Unit = + PackageManager.Default.loadPackage(packageRoot.toFile).get + + private def downloadLooseFiles( + libraryName: LibraryName, + version: SemVer, + access: LibraryAccess, + localTmpDir: Path + ): Unit = { + val pkgDownload = access.downloadPackageConfig(localTmpDir) + progressReporter.trackProgress( + s"Downloading package file of [$libraryName].", + pkgDownload + ) + pkgDownload.force() + + val licenseDownload = access.downloadLicense(localTmpDir) + progressReporter.trackProgress( + s"Downloading license of [$libraryName].", + licenseDownload + ) + TaskProgress.waitForTask(licenseDownload).getOrElse { + // TODO [RW] Once warnings are reported to the IDE (#1860), + // inform that the license file is missing. + logger.warn( + s"License file for library [$libraryName:$version] was missing." + ) + } + } + + private def downloadAndExtractArchives( + libraryName: LibraryName, + access: LibraryAccess, + manifest: LibraryManifest, + destinationDirectory: Path + ): Unit = FileSystem.withTemporaryDirectory(s"enso-$libraryName") { + // See [Temporary Directories for Installation] + globalTmpDir => + for (archiveName <- manifest.archives) { + if (shouldDownloadArchive(archiveName)) { + val tmpArchivePath = globalTmpDir / archiveName + + val download = access.downloadArchive( + archiveName, + tmpArchivePath / archiveName + ) + progressReporter.trackProgress( + s"Downloading [$archiveName] of [$libraryName].", + download + ) + download.force() + + val extraction = + Archive.extractArchive(tmpArchivePath, destinationDirectory, None) + progressReporter.trackProgress( + s"Extracting [$archiveName] of [$libraryName].", + extraction + ) + extraction.force() + + Files.delete(tmpArchivePath) + } else { + logger.info( + s"Sub-package [$archiveName] of [$libraryName] is " + + s"skipped, as it is optional." + ) + } + } + } + private def shouldDownloadArchive(archiveName: String): Boolean = { val isTestData = archiveName.startsWith("tests") !isTestData diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala index c565fbce20c9..97717a0313b8 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala @@ -2,13 +2,14 @@ package org.enso.librarymanager.published.repository import nl.gn0s1s.bump.SemVer import org.enso.cli.task.TaskProgress -import org.enso.downloader.http.{APIResponse, HTTPDownload, URIBuilder} +import org.enso.distribution.FileSystem.PathSyntax +import org.enso.downloader.http.{HTTPDownload, URIBuilder} import org.enso.editions.Editions.Repository import org.enso.editions.LibraryName import org.enso.yaml.YamlHelper import java.nio.file.Path -import scala.util.{Failure, Success} +import scala.util.Failure object RepositoryHelper { class LibraryAccess( @@ -42,16 +43,16 @@ object RepositoryHelper { HTTPDownload.download(url, destination).map(_ => ()) } - def downloadLicense(destination: Path): TaskProgress[Unit] = - downloadArtifact(licenseFilename, destination) + def downloadLicense(destinationDirectory: Path): TaskProgress[Unit] = + downloadArtifact(licenseFilename, destinationDirectory / licenseFilename) - def downloadPackageConfig(destination: Path): TaskProgress[Unit] = - downloadArtifact(packageFileName, destination) + def downloadPackageConfig(destinationDirectory: Path): TaskProgress[Unit] = + downloadArtifact(packageFileName, destinationDirectory / packageFileName) def downloadArchive( archiveName: String, - destination: Path - ): TaskProgress[Unit] = downloadArtifact(archiveName, destination) + destinationDirectory: Path + ): TaskProgress[Unit] = downloadArtifact(archiveName, destinationDirectory) } implicit class RepositoryMethods(val repository: Repository) { @@ -63,7 +64,7 @@ object RepositoryHelper { .addPathSegment(version.toString) def accessLibrary(name: LibraryName, version: SemVer): LibraryAccess = - new LibraryAccess(resolveLibraryRoot(name, version)) + new LibraryAccess(name, version, resolveLibraryRoot(name, version)) } val manifestFilename = "manifest.yaml" From 3db23294137c40743d5580f4132010f6db55cf0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Mon, 19 Jul 2021 23:38:03 +0200 Subject: [PATCH 11/30] WIP on test --- .../published/LibraryDownloader.scala | 58 -------------- .../repository/DummyRepository.scala | 79 +++++++++++++++++++ .../repository/ExampleRepository.scala | 17 ++++ .../repository/LibraryDownloadTest.scala | 42 ++++++++++ 4 files changed, 138 insertions(+), 58 deletions(-) delete mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala create mode 100644 lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala create mode 100644 lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala create mode 100644 lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala deleted file mode 100644 index 165abbabb01e..000000000000 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/LibraryDownloader.scala +++ /dev/null @@ -1,58 +0,0 @@ -package org.enso.librarymanager.published - -import org.enso.cli.task.ProgressReporter -import org.enso.distribution.FileSystem.PathSyntax -import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} -import org.enso.editions.LibraryName -import org.enso.librarymanager.published.repository.LibraryManifest -import org.enso.yaml.YamlHelper - -import java.nio.file.Path - -class LibraryDownloader(progressReporter: ProgressReporter) { -// def downloadLibrary( -// name: LibraryName, -// version: SemVer, -// repository: Repository, -// destinationRoot: Path -// ): Unit = { -// val libraryRoot = resolveLibraryRoot(name, version, repository) -// val manifest = downloadManifest(libraryRoot) -// val packages = manifest.archives -// for (pkg <- packages) { -// // TODO filtering -// } -// -// // TODO tmp directory, unpack extract -// // TODO locking -// // TODO progress reporting -// } - - def downloadManifest(libraryRoot: URIBuilder): LibraryManifest = { - val uri = libraryRoot.addPathSegment(LibraryManifest.fileName).build() - val request = HTTPRequestBuilder.fromURI(uri).GET - val response = HTTPDownload.fetchString(request).force() - YamlHelper.parseString[LibraryManifest](response.content).toTry.get - } - - /** Downloads a sub-package and extracts it to the given location, it will - * merge the contents with other packages that were extracted there earlier. - */ - def downloadAndExtractPackage( - libraryName: LibraryName, - libraryRoot: URIBuilder, - packageName: String, - destination: Path - ): Unit = { - val uri = libraryRoot.addPathSegment(packageName).build() - val request = HTTPRequestBuilder.fromURI(uri).GET - // TODO download to temporary location, not to the final destination! - val download = HTTPDownload.download(request, destination / packageName) - progressReporter.trackProgress( - s"Downloading $libraryName ($packageName).", - download - ) - download.force() - // TODO WIP - } -} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala new file mode 100644 index 000000000000..fca783321e2c --- /dev/null +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -0,0 +1,79 @@ +package org.enso.librarymanager.published.repository + +import nl.gn0s1s.bump.SemVer +import org.enso.distribution.FileSystem +import org.enso.editions.Editions.RawEdition +import org.enso.editions.{Editions, LibraryName} +import org.enso.pkg.{Package, PackageManager} + +import java.io.File +import java.nio.file.{Files, Path} + +abstract class DummyRepository { + + case class DummyLibrary( + libraryName: LibraryName, + version: SemVer, + mainContent: String + ) + + def repoName: String = "test_repo" + + def libraries: Seq[DummyLibrary] + + def createRepository(root: Path): Unit = { + for (lib <- libraries) { + val libraryRoot = root + .resolve("libraries") + .resolve(lib.libraryName.namespace) + .resolve(lib.libraryName.name) + .resolve(lib.version.toString) + Files.createDirectories(libraryRoot) + val pkg = createLibraryProject(libraryRoot, lib) + FileSystem.writeTextFile( + pkg.sourceDir.toPath.resolve("Main.enso"), + lib.mainContent + ) + } + } + + private def createLibraryProject( + path: Path, + lib: DummyLibrary + ): Package[File] = { + val pkg = PackageManager.Default.create( + path.toFile, + name = lib.libraryName.name, + namespace = lib.libraryName.namespace, + version = lib.version.toString() + ) + pkg.save().get + pkg + } + + def createEdition(repoUrl: String): RawEdition = { + Editions.Raw.Edition( + parent = Some(buildinfo.Info.currentEdition), + repositories = Map(repoName -> Editions.Repository(repoName, repoUrl)), + libraries = Map.from(libraries.map { lib => + lib.libraryName -> Editions.Raw + .PublishedLibrary(lib.libraryName, lib.version, repoName) + }) + ) + } + + def runServer(port: Int, root: Path): Process = { + val serverDirectory = Path.of("../../../tools/simple-library-server") + (new ProcessBuilder) + .command( + "node", + "main.js", + "--port", + port.toString, + "--root", + root.toAbsolutePath.normalize.toString + ) + .directory(serverDirectory.toFile) + .start() + } +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala new file mode 100644 index 000000000000..7fd64d44cddc --- /dev/null +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala @@ -0,0 +1,17 @@ +package org.enso.librarymanager.published.repository + +import nl.gn0s1s.bump.SemVer +import org.enso.editions.LibraryName + +class ExampleRepository extends DummyRepository { + val testLib = DummyLibrary( + LibraryName("Foo", "Bar"), + SemVer(1, 0, 0), + """baz = 42 + | + |quux = "foobar" + |""".stripMargin + ) + + override def libraries: Seq[DummyLibrary] = Seq(testLib) +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala new file mode 100644 index 000000000000..0463166acb97 --- /dev/null +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -0,0 +1,42 @@ +package org.enso.librarymanager.published.repository + +import org.enso.distribution.FileSystem +import org.enso.editions.Editions +import org.enso.librarymanager.published.cache.DownloadingLibraryCache +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +class LibraryDownloadTest extends AnyWordSpec with Matchers { + + val port: Int = 54325 + + "DownloadingLibraryCache" should { + "be able to download and install libraries from a repository" in { + val repo = new ExampleRepository + FileSystem.withTemporaryDirectory("enso-test-repo") { repoRoot => + repo.createRepository(repoRoot) + val server = repo.runServer(port, repoRoot) + + // TODO + val cache = new DownloadingLibraryCache(???, ???, ???, ???, ???) + + cache.findCachedLibrary( + repo.testLib.libraryName, + repo.testLib.version + ) shouldBe empty + + cache + .findOrInstallLibrary( + repo.testLib.libraryName, + repo.testLib.version, + Editions + .Repository("test_repo", s"http://localhost:$port/libraries") + ) + .get + + server.destroy() + server.waitFor() + } + } + } +} From 3f2d706231e9225a193cd9421b659850b853f97d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 20 Jul 2021 12:30:41 +0200 Subject: [PATCH 12/30] Implement a basic download test --- .../distribution/DefaultManagers.scala | 2 +- .../TemporaryDirectoryManager.scala | 30 +++++--- .../cache/DownloadingLibraryCache.scala | 5 +- .../published/repository/ArchiveWriter.scala | 39 ++++++++++ .../repository/DummyRepository.scala | 23 ++++-- .../repository/LibraryDownloadTest.scala | 72 ++++++++++++++----- .../DefaultDistributionConfiguration.scala | 2 +- .../TestDistributionConfiguration.scala | 2 +- .../test/RuntimeVersionManagerTest.scala | 2 +- .../locking/ConcurrencyTest.scala | 2 +- 10 files changed, 134 insertions(+), 45 deletions(-) create mode 100644 lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala diff --git a/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala b/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala index 0c80405f96c5..c9516d414bc0 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala @@ -44,7 +44,7 @@ object DefaultManagers { /** Default [[TemporaryDirectoryManager]]. */ lazy val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, defaultResourceManager) + TemporaryDirectoryManager(distributionManager, defaultResourceManager) /** Default [[RuntimeComponentConfiguration]]. */ lazy val componentConfig: RuntimeComponentConfiguration = diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala index 48ce8612b262..6b18bcac0af2 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala @@ -21,7 +21,7 @@ import scala.util.Random * so that they can be moved all at once at the last step of the installation. */ class TemporaryDirectoryManager( - distribution: DistributionManager, + unsafeRoot: Path, resourceManager: ResourceManager ) { private val logger = Logger[TemporaryDirectoryManager] @@ -54,11 +54,10 @@ class TemporaryDirectoryManager( * should ensure that). If that fails, it is also cleaned before any future * accesses. */ - private lazy val safeTemporaryDirectory = { + private lazy val safeTemporaryDirectory: Path = { resourceManager.startUsingTemporaryDirectory() - val path = distribution.paths.unsafeTemporaryDirectory - Files.createDirectories(path) - path + Files.createDirectories(unsafeRoot) + unsafeRoot } /** Tries to clean the temporary files directory. @@ -70,18 +69,27 @@ class TemporaryDirectoryManager( * marked as in-use and will not be cleaned. */ def tryCleaningTemporaryDirectory(): Unit = { - val tmp = distribution.paths.unsafeTemporaryDirectory - if (Files.exists(tmp)) { + if (Files.exists(unsafeRoot)) { resourceManager.tryWithExclusiveTemporaryDirectory { - if (!FileSystem.isDirectoryEmpty(tmp)) { + if (!FileSystem.isDirectoryEmpty(unsafeRoot)) { logger.info( "Cleaning up temporary files from a previous installation." ) } - FileSystem.removeDirectory(tmp) - Files.createDirectories(tmp) - FileSystem.removeEmptyDirectoryOnExit(tmp) + FileSystem.removeDirectory(unsafeRoot) + Files.createDirectories(unsafeRoot) + FileSystem.removeEmptyDirectoryOnExit(unsafeRoot) } } } } + +object TemporaryDirectoryManager { + def apply( + distribution: DistributionManager, + resourceManager: ResourceManager + ): TemporaryDirectoryManager = new TemporaryDirectoryManager( + distribution.paths.unsafeTemporaryDirectory, + resourceManager + ) +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala index 1f460b40c539..2d15d98781fe 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -188,10 +188,7 @@ class DownloadingLibraryCache( if (shouldDownloadArchive(archiveName)) { val tmpArchivePath = globalTmpDir / archiveName - val download = access.downloadArchive( - archiveName, - tmpArchivePath / archiveName - ) + val download = access.downloadArchive(archiveName, tmpArchivePath) progressReporter.trackProgress( s"Downloading [$archiveName] of [$libraryName].", download diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala new file mode 100644 index 000000000000..1fde03cde031 --- /dev/null +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala @@ -0,0 +1,39 @@ +package org.enso.librarymanager.published.repository + +import org.apache.commons.compress.archivers.tar.{ + TarArchiveEntry, + TarArchiveOutputStream +} +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream + +import java.io.{BufferedOutputStream, FileOutputStream} +import java.nio.file.Path +import scala.util.Using + +object ArchiveWriter { + sealed trait FileToWrite { + def relativePath: String + } + + case class TextFile(relativePath: String, content: String) extends FileToWrite + + def writeTarArchive(path: Path, files: Seq[FileToWrite]): Unit = { + Using(new FileOutputStream(path.toFile)) { outputStream => + Using(new BufferedOutputStream(outputStream)) { bufferedStream => + Using(new GzipCompressorOutputStream(bufferedStream)) { gzipStream => + Using(new TarArchiveOutputStream(gzipStream)) { archive => + for (file <- files) { + file match { + case TextFile(relativePath, content) => + val entry = new TarArchiveEntry(relativePath) + archive.putArchiveEntry(entry) + archive.write(content.getBytes) + archive.closeArchiveEntry() + } + } + } + } + } + } + } +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index fca783321e2c..f83a292b84e2 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -29,11 +29,12 @@ abstract class DummyRepository { .resolve(lib.libraryName.name) .resolve(lib.version.toString) Files.createDirectories(libraryRoot) - val pkg = createLibraryProject(libraryRoot, lib) - FileSystem.writeTextFile( - pkg.sourceDir.toPath.resolve("Main.enso"), - lib.mainContent + createLibraryProject(libraryRoot, lib) + val files = Seq( + ArchiveWriter.TextFile("src/Main.enso", lib.mainContent) ) + ArchiveWriter.writeTarArchive(libraryRoot.resolve("main.tgz"), files) + createManifest(libraryRoot) } } @@ -51,6 +52,15 @@ abstract class DummyRepository { pkg } + private def createManifest(path: Path): Unit = { + FileSystem.writeTextFile( + path.resolve("manifest.yaml"), + s"""archives: + | - main.tgz + |""".stripMargin + ) + } + def createEdition(repoUrl: String): RawEdition = { Editions.Raw.Edition( parent = Some(buildinfo.Info.currentEdition), @@ -62,8 +72,9 @@ abstract class DummyRepository { ) } - def runServer(port: Int, root: Path): Process = { - val serverDirectory = Path.of("../../../tools/simple-library-server") + def startServer(port: Int, root: Path): Process = { + val serverDirectory = + Path.of("tools/simple-library-server").toAbsolutePath.normalize (new ProcessBuilder) .command( "node", diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala index 0463166acb97..d1200d0cfcd5 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -1,8 +1,16 @@ package org.enso.librarymanager.published.repository -import org.enso.distribution.FileSystem +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.distribution.locking.{ + LockUserInterface, + Resource, + ResourceManager, + ThreadSafeFileLockManager +} +import org.enso.distribution.{FileSystem, TemporaryDirectoryManager} import org.enso.editions.Editions import org.enso.librarymanager.published.cache.DownloadingLibraryCache +import org.enso.pkg.PackageManager import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -13,29 +21,55 @@ class LibraryDownloadTest extends AnyWordSpec with Matchers { "DownloadingLibraryCache" should { "be able to download and install libraries from a repository" in { val repo = new ExampleRepository - FileSystem.withTemporaryDirectory("enso-test-repo") { repoRoot => + FileSystem.withTemporaryDirectory("enso-test-lib") { tmp => + val repoRoot = tmp.resolve("repo") repo.createRepository(repoRoot) - val server = repo.runServer(port, repoRoot) - - // TODO - val cache = new DownloadingLibraryCache(???, ???, ???, ???, ???) + val lockManager = new ThreadSafeFileLockManager(tmp.resolve("locks")) + val resourceManager = new ResourceManager(lockManager) + val cache = new DownloadingLibraryCache( + cacheRoot = tmp.resolve("cache"), + temporaryDirectoryManager = + new TemporaryDirectoryManager(tmp.resolve("tmp"), resourceManager), + resourceManager = resourceManager, + lockUserInterface = new LockUserInterface { + override def startWaitingForResource(resource: Resource): Unit = + println(s"Waiting for ${resource.name}") - cache.findCachedLibrary( - repo.testLib.libraryName, - repo.testLib.version - ) shouldBe empty + override def finishWaitingForResource(resource: Resource): Unit = + println(s"${resource.name} is ready") + }, + progressReporter = new ProgressReporter { + override def trackProgress( + message: String, + task: TaskProgress[_] + ): Unit = {} + } + ) - cache - .findOrInstallLibrary( + val server = repo.startServer(port, repoRoot) + try { + cache.findCachedLibrary( repo.testLib.libraryName, - repo.testLib.version, - Editions - .Repository("test_repo", s"http://localhost:$port/libraries") - ) - .get + repo.testLib.version + ) shouldBe empty - server.destroy() - server.waitFor() + val libPath = cache + .findOrInstallLibrary( + repo.testLib.libraryName, + repo.testLib.version, + Editions + .Repository("test_repo", s"http://localhost:$port/libraries") + ) + .get + val pkg = PackageManager.Default.loadPackage(libPath.toFile).get + pkg.name shouldEqual "Bar" + val sources = pkg.listSources + sources should have size 1 + sources.head.file.getName shouldEqual "Main.enso" + } finally { + server.destroy() + server.waitFor() + } } } } diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala index abfc3dcaca19..bc1a52f56391 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala @@ -57,7 +57,7 @@ object DefaultDistributionConfiguration /** @inheritdoc */ lazy val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) lazy val componentConfiguration: RuntimeComponentConfiguration = new GraalVMComponentConfiguration diff --git a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala index 7df57cf139f0..da5069f3d275 100644 --- a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala @@ -70,7 +70,7 @@ class TestDistributionConfiguration( lazy val editionManager: EditionManager = EditionManager(distributionManager) lazy val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) lazy val componentConfig = new GraalVMComponentConfiguration diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala index c4162d655a66..8f5efcaa1483 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala @@ -53,7 +53,7 @@ class RuntimeVersionManagerTest val resourceManager = TestLocalResourceManager.create() val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) val componentConfig = new GraalVMComponentConfiguration val runtimeVersionManager = new RuntimeVersionManager( diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala index 0335f3524c90..b94c2fda0e46 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala @@ -152,7 +152,7 @@ class ConcurrencyTest } val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) val componentConfig = new GraalVMComponentConfiguration val componentsManager = new RuntimeVersionManager( TestRuntimeVersionManagementUserInterface.default, From 68eaed6150fc019eb7d54dee07bc8791a44de48f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 20 Jul 2021 13:58:01 +0200 Subject: [PATCH 13/30] Integrate downloading cache with PackageRepository --- .../org/enso/interpreter/runtime/Context.java | 11 ++++- .../org/enso/compiler/PackageRepository.scala | 5 +++ .../instrument/NotificationHandler.scala | 32 +++++++++----- ...ProgressAndLockNotificationForwarder.scala | 44 +++++++++++++++++++ .../locking/ResourceManager.scala | 6 +-- .../DefaultLibraryProvider.scala | 23 ++++++++-- .../published/cache/NoOpCache.scala | 38 ---------------- .../ControllerInterface.scala | 44 +++---------------- 8 files changed, 109 insertions(+), 94 deletions(-) create mode 100644 lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala delete mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java index 15fa10bcb566..93d02381605e 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java @@ -15,6 +15,7 @@ import org.enso.compiler.Compiler; import org.enso.compiler.PackageRepository; import org.enso.compiler.data.CompilerConfig; +import org.enso.distribution.locking.ThreadSafeFileLockManager; import org.enso.editions.LibraryName; import org.enso.interpreter.Language; import org.enso.interpreter.OptionsHelper; @@ -101,11 +102,19 @@ public void initialize() { var languageHome = OptionsHelper.getLanguageHomeOverride(environment).or(() -> Optional.ofNullable(home)); + var distributionManager = RuntimeDistributionManager$.MODULE$; + + // TODO [RW] Once #1890 is implemented, this will need to connect to the Language Server's + // LockManager. + var lockManager = new ThreadSafeFileLockManager(distributionManager.paths().locks()); + var resourceManager = new org.enso.distribution.locking.ResourceManager(lockManager); + packageRepository = PackageRepository.initializeRepository( OptionConverters.toScala(projectPackage), OptionConverters.toScala(languageHome), - RuntimeDistributionManager$.MODULE$, + distributionManager, + resourceManager, this, builtins, notificationHandler); diff --git a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala index 8a17740b7d85..1f41e204a26c 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala @@ -2,6 +2,7 @@ package org.enso.compiler import com.oracle.truffle.api.TruffleFile import com.typesafe.scalalogging.Logger +import org.enso.distribution.locking.ResourceManager import org.enso.distribution.{DistributionManager, EditionManager, LanguageHome} import org.enso.editions.{DefaultEdition, LibraryName, LibraryVersion} import org.enso.interpreter.instrument.NotificationHandler @@ -329,6 +330,7 @@ object PackageRepository { projectPackage: Option[Package[TruffleFile]], languageHome: Option[String], distributionManager: DistributionManager, + resourceManager: ResourceManager, context: Context, builtins: Builtins, notificationHandler: NotificationHandler @@ -344,6 +346,9 @@ object PackageRepository { val resolvingLibraryProvider = new DefaultLibraryProvider( distributionManager = distributionManager, + resourceManager = resourceManager, + lockUserInterface = notificationHandler, + progressReporter = notificationHandler, languageHome = homeManager, edition = edition, preferLocalLibraries = diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala index ef420792b902..0160a12069df 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala @@ -2,12 +2,9 @@ package org.enso.interpreter.instrument import com.typesafe.scalalogging.Logger import org.enso.cli.ProgressBar -import org.enso.cli.task.{ - ProgressNotification, - ProgressNotificationForwarder, - ProgressReporter, - TaskProgress -} +import org.enso.cli.task.{ProgressNotification, ProgressReporter, TaskProgress} +import org.enso.distribution.ProgressAndLockNotificationForwarder +import org.enso.distribution.locking.{LockUserInterface, Resource} import org.enso.editions.{LibraryName, LibraryVersion} import org.enso.polyglot.runtime.Runtime.{Api, ApiResponse} @@ -16,7 +13,7 @@ import java.nio.file.Path /** A class that forwards notifications about loaded libraries and long-running * tasks to the user interface. */ -trait NotificationHandler extends ProgressReporter { +trait NotificationHandler extends ProgressReporter with LockUserInterface { /** Called when a library has been loaded. * @@ -40,6 +37,8 @@ object NotificationHandler { */ object TextMode extends NotificationHandler { + private lazy val logger = Logger[TextMode.type] + /** @inheritdoc */ override def addedLibrary( libraryName: LibraryName, @@ -51,11 +50,16 @@ object NotificationHandler { /** @inheritdoc */ override def trackProgress(message: String, task: TaskProgress[_]): Unit = { - Logger[TextMode.type].info(message) + logger.info(message) if (System.console() != null) { ProgressBar.waitWithProgress(task) } } + + override def startWaitingForResource(resource: Resource): Unit = + logger.warn(resource.waitMessage) + + override def finishWaitingForResource(resource: Resource): Unit = () } /** A [[NotificationHandler]] that forwards messages to other @@ -76,6 +80,14 @@ object NotificationHandler { override def trackProgress(message: String, task: TaskProgress[_]): Unit = for (listener <- listeners) listener.trackProgress(message, task) + /** @inheritdoc */ + override def startWaitingForResource(resource: Resource): Unit = + for (listener <- listeners) listener.startWaitingForResource(resource) + + /** @inheritdoc */ + override def finishWaitingForResource(resource: Resource): Unit = + for (listener <- listeners) listener.finishWaitingForResource(resource) + /** Registers a new listener. */ def addListener(listener: NotificationHandler): Unit = listeners ::= listener @@ -86,8 +98,8 @@ object NotificationHandler { * the IDE. */ class InteractiveMode(endpoint: Endpoint) - extends NotificationHandler - with ProgressNotificationForwarder { + extends ProgressAndLockNotificationForwarder + with NotificationHandler { private val logger = Logger[InteractiveMode] private def sendMessage(message: ApiResponse): Unit = { diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala new file mode 100644 index 000000000000..e91f8fac0feb --- /dev/null +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala @@ -0,0 +1,44 @@ +package org.enso.distribution + +import org.enso.cli.task.{ + ProgressNotification, + ProgressNotificationForwarder, + ProgressUnit +} +import org.enso.distribution.locking.{LockUserInterface, Resource} + +import java.util.UUID + +abstract class ProgressAndLockNotificationForwarder + extends ProgressNotificationForwarder + with LockUserInterface { + private val waitingForResources = + collection.concurrent.TrieMap[String, UUID]() + + /** @inheritdoc */ + override def startWaitingForResource(resource: Resource): Unit = { + val uuid = UUID.randomUUID() + sendProgressNotification( + ProgressNotification.TaskStarted( + uuid, + None, + ProgressUnit.Unspecified + ) + ) + sendProgressNotification( + ProgressNotification.TaskUpdate( + uuid, + Some(resource.waitMessage), + 0 + ) + ) + waitingForResources.put(resource.name, uuid) + } + + /** @inheritdoc */ + override def finishWaitingForResource(resource: Resource): Unit = { + for (uuid <- waitingForResources.remove(resource.name)) { + sendProgressNotification(ProgressNotification.TaskSuccess(uuid)) + } + } +} diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/locking/ResourceManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/locking/ResourceManager.scala index 785077da6b5c..7234f7cefc02 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/locking/ResourceManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/locking/ResourceManager.scala @@ -151,10 +151,10 @@ class ResourceManager(lockManager: LockManager) { */ def startUsingTemporaryDirectory(): Unit = { if (temporaryDirectoryLock.isDefined) { - throw new IllegalStateException( - "Temporary directory lock has been acquired twice." - ) + logger.trace("The temporary directory was already in-use.") + return } + val lock = lockManager.acquireLockWithWaitingAction( TemporaryDirectory.name, LockType.Shared, diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala index 7fb7ac35b6d8..df4d0386ade8 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala @@ -1,11 +1,17 @@ package org.enso.librarymanager import com.typesafe.scalalogging.Logger -import org.enso.distribution.{DistributionManager, LanguageHome} +import org.enso.cli.task.ProgressReporter +import org.enso.distribution.locking.{LockUserInterface, ResourceManager} +import org.enso.distribution.{ + DistributionManager, + LanguageHome, + TemporaryDirectoryManager +} import org.enso.editions.{Editions, LibraryName, LibraryVersion} import org.enso.librarymanager.local.DefaultLocalLibraryProvider import org.enso.librarymanager.published.bundles.LocalReadOnlyRepository -import org.enso.librarymanager.published.cache.NoOpCache +import org.enso.librarymanager.published.cache.DownloadingLibraryCache import org.enso.librarymanager.published.{ DefaultPublishedLibraryProvider, PublishedLibraryProvider @@ -17,12 +23,16 @@ import java.nio.file.Path /** A helper class for loading libraries. * * @param distributionManager a distribution manager + * @param resourceManager a resource manager * @param languageHome a language home which may contain bundled libraries * @param edition the edition used in the project * @param preferLocalLibraries project setting whether to use local libraries */ class DefaultLibraryProvider( distributionManager: DistributionManager, + resourceManager: ResourceManager, + lockUserInterface: LockUserInterface, + progressReporter: ProgressReporter, languageHome: Option[LanguageHome], edition: Editions.ResolvedEdition, preferLocalLibraries: Boolean @@ -36,8 +46,13 @@ class DefaultLibraryProvider( private val resolver = LibraryResolver(localLibraryProvider) - // TODO [RW] actual cache that can download libraries will be implemented in #1772 - private val primaryCache = new NoOpCache + private val primaryCache = new DownloadingLibraryCache( + cacheRoot = distributionManager.paths.cachedLibraries, + TemporaryDirectoryManager(distributionManager, resourceManager), + resourceManager, + lockUserInterface, + progressReporter + ) private val additionalCacheLocations = { val engineBundleRoot = languageHome.map(_.libraries) val locations = diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala deleted file mode 100644 index 425cdc6defa7..000000000000 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala +++ /dev/null @@ -1,38 +0,0 @@ -package org.enso.librarymanager.published.cache -import nl.gn0s1s.bump.SemVer -import org.enso.editions.{Editions, LibraryName, LibraryVersion} - -import java.nio.file.Path -import scala.util.{Failure, Try} - -/** A temporary cache that provides no libraries. - * - * This is a temporary poly-fill which will later be replaced when the - * downloading mechanism is implemented. - */ -class NoOpCache extends LibraryCache { - - /** @inheritdoc */ - override def findCachedLibrary( - libraryName: LibraryName, - version: SemVer - ): Option[Path] = None - - /** @inheritdoc */ - override def findOrInstallLibrary( - libraryName: LibraryName, - version: SemVer, - recommendedRepository: Editions.Repository - ): Try[Path] = Failure( - new NotImplementedError("Downloading libraries is not yet implemented.") - ) - - override def preinstallLibrary( - libraryName: LibraryName, - version: SemVer, - recommendedRepository: Editions.Repository, - dependencyResolver: LibraryName => Option[LibraryVersion] - ): Try[Unit] = Failure( - new NotImplementedError("Downloading libraries is not yet implemented.") - ) -} diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/ControllerInterface.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/ControllerInterface.scala index 808c4c27d97f..09467e5285b4 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/ControllerInterface.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/ControllerInterface.scala @@ -3,34 +3,28 @@ package org.enso.projectmanager.service.versionmanagement import akka.actor.ActorRef import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer -import org.enso.cli.task.{ - ProgressNotification, - ProgressNotificationForwarder, - ProgressUnit -} -import org.enso.distribution.locking.Resource +import org.enso.cli.task.ProgressNotification +import org.enso.distribution.ProgressAndLockNotificationForwarder import org.enso.runtimeversionmanager.components.{ GraalVMVersion, RuntimeVersionManagementUserInterface } -import java.util.UUID - /** A [[RuntimeVersionManagementUserInterface]] that sends * [[ProgressNotification]] to the specified actors (both for usual tasks and * indeterminate progress when waiting on locks). * - * @param progressTracker the actor to send progress updates to + * @param progressTracker the actor to send progress updates to * @param allowMissingComponents specifies if missing components should be * automatically installed - * @param allowBrokenComponents specifies if broken components can be installed + * @param allowBrokenComponents specifies if broken components can be installed */ class ControllerInterface( progressTracker: ActorRef, allowMissingComponents: Boolean, allowBrokenComponents: Boolean -) extends RuntimeVersionManagementUserInterface - with ProgressNotificationForwarder { +) extends ProgressAndLockNotificationForwarder + with RuntimeVersionManagementUserInterface { /** @inheritdoc */ override def shouldInstallMissingEngine(version: SemVer): Boolean = @@ -48,32 +42,6 @@ class ControllerInterface( override def logInfo(message: => String): Unit = Logger[ControllerInterface].info(message) - private val waitingForResources = - collection.concurrent.TrieMap[String, UUID]() - - /** @inheritdoc */ - override def startWaitingForResource(resource: Resource): Unit = { - val uuid = UUID.randomUUID() - progressTracker ! ProgressNotification.TaskStarted( - uuid, - None, - ProgressUnit.Unspecified - ) - progressTracker ! ProgressNotification.TaskUpdate( - uuid, - Some(resource.waitMessage), - 0 - ) - waitingForResources.put(resource.name, uuid) - } - - /** @inheritdoc */ - override def finishWaitingForResource(resource: Resource): Unit = { - for (uuid <- waitingForResources.remove(resource.name)) { - progressTracker ! ProgressNotification.TaskSuccess(uuid) - } - } - /** @inheritdoc */ override def sendProgressNotification( notification: ProgressNotification From 701a5c5024c92158a0532fe1aec4abdd2ee4b8ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 20 Jul 2021 17:55:11 +0200 Subject: [PATCH 14/30] Fix Akka issues in built runtime --- build.sbt | 3 ++- .../enso/downloader/http/HTTPDownload.scala | 20 ++++++++++++++++++- .../DefaultLibraryProvider.scala | 18 +++++++++++------ .../repository/LibraryDownloadTest.scala | 1 + 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index f527fcc1c407..512031321569 100644 --- a/build.sbt +++ b/build.sbt @@ -1370,7 +1370,8 @@ lazy val downloader = (project in file("lib/scala/downloader")) "org.scalatest" %% "scalatest" % scalatestVersion % Test, akkaActor, akkaStream, - akkaHttp + akkaHttp, + akkaSLF4J ) ) .dependsOn(cli) diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala index 9c01f1f3ebf8..3415dc5cf4b4 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala @@ -17,6 +17,7 @@ import org.enso.cli.task.{ import java.nio.charset.{Charset, StandardCharsets} import java.nio.file.Path import scala.concurrent.Future +import scala.jdk.CollectionConverters.IterableHasAsJava /** Represents a HTTP header. */ case class Header(name: String, value: String) { @@ -149,12 +150,29 @@ object HTTPDownload { } implicit private lazy val actorSystem: ActorSystem = { + val loggers: java.lang.Iterable[String] = + Seq("akka.event.slf4j.Slf4jLogger").asJava val config = ConfigFactory .load() + .withValue( + "akka.extensions", + ConfigValueFactory.fromAnyRef(Seq.empty.asJava) + ) + .withValue( + "akka.library-extensions", + ConfigValueFactory.fromAnyRef(Seq.empty.asJava) + ) + .withValue("akka.loggers", ConfigValueFactory.fromAnyRef(loggers)) + .withValue( + "akka.logging-filter", + ConfigValueFactory.fromAnyRef("akka.event.DefaultLoggingFilter") + ) .withValue("akka.loglevel", ConfigValueFactory.fromAnyRef("WARNING")) + ActorSystem( "http-requests-actor-system", - config + config, + classLoader = getClass.getClassLoader // Note [Actor System Class Loader] ) } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala index df4d0386ade8..0cc90ff74d17 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala @@ -110,12 +110,18 @@ class DefaultLibraryProvider( } case Right(version @ LibraryVersion.Published(semver, repository)) => - publishedLibraryProvider - .findLibrary(libraryName, semver, repository) - .map(ResolvedLibrary(libraryName, version, _)) - .toEither - .left - .map(ResolvingLibraryProvider.Error.DownloadFailed) + val res = + publishedLibraryProvider + .findLibrary(libraryName, semver, repository) + .map(ResolvedLibrary(libraryName, version, _)) + .toEither + res match { + case Left(value) => + println(s"Download error: $value") + value.printStackTrace() + case Right(_) => + } + res.left.map(ResolvingLibraryProvider.Error.DownloadFailed) } } } diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala index d1200d0cfcd5..a35d56a18dd1 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -66,6 +66,7 @@ class LibraryDownloadTest extends AnyWordSpec with Matchers { val sources = pkg.listSources sources should have size 1 sources.head.file.getName shouldEqual "Main.enso" + // TODO [RW] check that the license is missing } finally { server.destroy() server.waitFor() From 21f3591fb01c3615157ebd984523b7d73c27f96d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 20 Jul 2021 17:59:22 +0200 Subject: [PATCH 15/30] Install node in test CI --- .github/workflows/scala.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index 5a2cc23a41ff..b988473a2877 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -15,6 +15,8 @@ env: sbtVersion: 1.5.2 # Please ensure that this is in sync with rustVersion in build.sbt rustToolchain: nightly-2021-05-12 + # Some moderately recent version of Node.JS is needed to run the library download tests. + nodeVersion: 12.18.4 jobs: test_and_publish: @@ -80,6 +82,10 @@ jobs: graalvm-version: ${{ env.graalVersion }} java-version: ${{ env.javaVersion }} native-image: true + - name: Install Node + uses: actions/setup-node@v1 + with: + node-version: ${{ env.nodeVersion }} - name: Set Up SBT shell: bash run: | From ca6010d3c51b01a653032de7d9a0ba6607e971d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 20 Jul 2021 18:07:29 +0200 Subject: [PATCH 16/30] Legal review --- distribution/engine/THIRD-PARTY/NOTICE | 20 +++---- .../NOTICES | 28 +++++----- .../NOTICES | 4 +- .../NOTICES | 4 +- .../NOTICES | 0 .../dev.zio.zio_2.13-1.0.1/NOTICES | 4 +- .../io.spray.spray-json_2.13-1.3.5/NOTICES | 4 +- .../NOTICE.txt | 55 +++++++++++++++++++ .../NOTICES | 20 +++++++ .../NOTICES | 10 ---- .../org.yaml.snakeyaml-1.26/NOTICES | 4 +- .../org.yaml.snakeyaml-1.26/NOTICES | 4 +- .../NOTICES | 4 +- .../NOTICES | 4 +- .../dev.zio.zio_2.13-1.0.1/NOTICES | 4 +- .../io.spray.spray-json_2.13-1.3.5/NOTICES | 4 +- .../org.yaml.snakeyaml-1.26/NOTICES | 4 +- tools/legal-review/Table/report-state | 2 +- .../copyright-keep-context | 2 +- .../copyright-ignore | 2 + .../copyright-keep-context | 1 + .../files-ignore | 1 + .../files-keep | 1 + .../copyright-keep-context | 1 - tools/legal-review/engine/report-state | 4 +- tools/legal-review/launcher/report-state | 4 +- .../legal-review/project-manager/report-state | 4 +- 27 files changed, 134 insertions(+), 65 deletions(-) rename distribution/engine/THIRD-PARTY/{com.typesafe.config-1.3.2 => com.typesafe.config-1.4.0}/NOTICES (100%) create mode 100644 distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICE.txt create mode 100644 distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICES delete mode 100644 distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.2/NOTICES rename tools/legal-review/engine/{com.typesafe.config-1.3.2 => com.typesafe.config-1.4.0}/copyright-keep-context (100%) create mode 100644 tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-ignore create mode 100644 tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-keep-context create mode 100644 tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-ignore create mode 100644 tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-keep delete mode 100644 tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.2/copyright-keep-context diff --git a/distribution/engine/THIRD-PARTY/NOTICE b/distribution/engine/THIRD-PARTY/NOTICE index 2f5e6caf8ed9..80eafe959ff1 100644 --- a/distribution/engine/THIRD-PARTY/NOTICE +++ b/distribution/engine/THIRD-PARTY/NOTICE @@ -11,6 +11,11 @@ The license information can be found along with the copyright notices. Copyright notices related to this dependency can be found in the directory `com.lihaoyi.sourcecode_2.13-0.2.1`. +'commons-compress', licensed under the Apache License, Version 2.0, is distributed with the engine. +The license file can be found at `licenses/APACHE2.0`. +Copyright notices related to this dependency can be found in the directory `org.apache.commons.commons-compress-1.20`. + + 'checker-qual', licensed under the The MIT License, is distributed with the engine. The license information can be found along with the copyright notices. Copyright notices related to this dependency can be found in the directory `org.checkerframework.checker-qual-2.11.1`. @@ -96,11 +101,6 @@ The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `com.fasterxml.jackson.core.jackson-core-2.11.1`. -'config', licensed under the Apache License, Version 2.0, is distributed with the engine. -The license file can be found at `licenses/APACHE2.0`. -Copyright notices related to this dependency can be found in the directory `com.typesafe.config-1.3.2`. - - 'zio_2.13', licensed under the Apache-2.0, is distributed with the engine. The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `dev.zio.zio_2.13-1.0.1`. @@ -161,11 +161,6 @@ The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `org.scala-lang.scala-compiler-2.13.6`. -'reactive-streams', licensed under the CC0, is distributed with the engine. -The license file can be found at `licenses/CC0`. -Copyright notices related to this dependency can be found in the directory `org.reactivestreams.reactive-streams-1.0.2`. - - 'config', licensed under the Apache-2.0, is distributed with the engine. The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `com.typesafe.config-1.4.1`. @@ -206,6 +201,11 @@ The license information can be found along with the copyright notices. Copyright notices related to this dependency can be found in the directory `org.typelevel.jawn-parser_2.13-1.0.0`. +'config', licensed under the Apache-2.0, is distributed with the engine. +The license file can be found at `licenses/APACHE2.0`. +Copyright notices related to this dependency can be found in the directory `com.typesafe.config-1.4.0`. + + 'circe-literal_2.13', licensed under the Apache 2.0, is distributed with the engine. The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `io.circe.circe-literal_2.13-0.14.0-M1`. diff --git a/distribution/engine/THIRD-PARTY/com.google.auto.service.auto-service-1.0-rc7/NOTICES b/distribution/engine/THIRD-PARTY/com.google.auto.service.auto-service-1.0-rc7/NOTICES index 3b5e809ce4c8..3db01d150f9f 100644 --- a/distribution/engine/THIRD-PARTY/com.google.auto.service.auto-service-1.0-rc7/NOTICES +++ b/distribution/engine/THIRD-PARTY/com.google.auto.service.auto-service-1.0-rc7/NOTICES @@ -1,3 +1,17 @@ +/* + * Copyright 2013 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + /* * Copyright 2008 Google LLC * @@ -14,20 +28,6 @@ * limitations under the License. */ -/* - * Copyright 2013 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - /* * Copyright 2008 Google LLC * diff --git a/distribution/engine/THIRD-PARTY/com.google.errorprone.error_prone_annotations-2.3.4/NOTICES b/distribution/engine/THIRD-PARTY/com.google.errorprone.error_prone_annotations-2.3.4/NOTICES index da6f3cd143cb..d345d4fd67c2 100644 --- a/distribution/engine/THIRD-PARTY/com.google.errorprone.error_prone_annotations-2.3.4/NOTICES +++ b/distribution/engine/THIRD-PARTY/com.google.errorprone.error_prone_annotations-2.3.4/NOTICES @@ -31,7 +31,7 @@ */ /* - * Copyright 2014 The Error Prone Authors. + * Copyright 2017 The Error Prone Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,7 +47,7 @@ */ /* - * Copyright 2017 The Error Prone Authors. + * Copyright 2014 The Error Prone Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/distribution/engine/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES b/distribution/engine/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES index 10e8435459c1..c4c899bb76e1 100644 --- a/distribution/engine/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES +++ b/distribution/engine/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES @@ -1,7 +1,7 @@ /* - * Copyright (C) 2009-2020 Lightbend Inc. + * Copyright (C) 2017-2020 Lightbend Inc. */ /* - * Copyright (C) 2017-2020 Lightbend Inc. + * Copyright (C) 2009-2020 Lightbend Inc. */ diff --git a/distribution/engine/THIRD-PARTY/com.typesafe.config-1.3.2/NOTICES b/distribution/engine/THIRD-PARTY/com.typesafe.config-1.4.0/NOTICES similarity index 100% rename from distribution/engine/THIRD-PARTY/com.typesafe.config-1.3.2/NOTICES rename to distribution/engine/THIRD-PARTY/com.typesafe.config-1.4.0/NOTICES diff --git a/distribution/engine/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES b/distribution/engine/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES index d39289b94fc7..1e3906f673f8 100644 --- a/distribution/engine/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES +++ b/distribution/engine/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES @@ -15,8 +15,6 @@ * limitations under the License. */ -Copyright 2017-2019 John A. De Goes and the ZIO Contributors - /* * Copyright 2017-2020 John A. De Goes and the ZIO Contributors * Copyright 2017-2018 Łukasz Biały, Paul Chiusano, Michael Pilquist, @@ -36,3 +34,5 @@ Copyright 2017-2019 John A. De Goes and the ZIO Contributors * See the License for the specific language governing permissions and * limitations under the License. */ + +Copyright 2017-2019 John A. De Goes and the ZIO Contributors diff --git a/distribution/engine/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES b/distribution/engine/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES index fea49c9eb3b8..ff981cbc6e70 100644 --- a/distribution/engine/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES +++ b/distribution/engine/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011,2012 Mathias Doenitz, Johannes Rudolph + * Copyright (C) 2011 Mathias Doenitz * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ /* - * Copyright (C) 2011 Mathias Doenitz + * Copyright (C) 2011,2012 Mathias Doenitz, Johannes Rudolph * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICE.txt b/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICE.txt new file mode 100644 index 000000000000..132b0897babc --- /dev/null +++ b/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICE.txt @@ -0,0 +1,55 @@ +Apache Commons Compress +Copyright 2002-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +--- + +The files in the package org.apache.commons.compress.archivers.sevenz +were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), +which has been placed in the public domain: + +"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) + +--- + +The test file lbzip2_32767.bz2 has been copied from libbzip2's source +repository: + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2019 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org diff --git a/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICES b/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICES new file mode 100644 index 000000000000..ea8ae0ca227f --- /dev/null +++ b/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICES @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Some portions of this file Copyright (c) 2004-2006 Intel Corportation + * and licensed under the BSD license. + */ diff --git a/distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.2/NOTICES b/distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.2/NOTICES deleted file mode 100644 index bd0a85c0c769..000000000000 --- a/distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.2/NOTICES +++ /dev/null @@ -1,10 +0,0 @@ -/************************************************************************ - * Licensed under Public Domain (CC0) * - * * - * To the extent possible under law, the person who associated CC0 with * - * this code has waived all copyright and related or neighboring * - * rights to this code. * - * * - * You should have received a copy of the CC0 legalcode along with this * - * work. If not, see .* - ************************************************************************/ diff --git a/distribution/engine/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES b/distribution/engine/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES index 788494853651..c2bba08c1748 100644 --- a/distribution/engine/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES +++ b/distribution/engine/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES @@ -14,7 +14,7 @@ * limitations under the License. */ -/* Copyright (c) 2008 Google Inc. - // Copyright 2003-2010 Christian d'Heureuse, Inventec Informatik AG, Zurich, Switzerland // www.source-code.biz, www.inventec.ch/chdh + +/* Copyright (c) 2008 Google Inc. diff --git a/distribution/launcher/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES b/distribution/launcher/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES index 788494853651..c2bba08c1748 100644 --- a/distribution/launcher/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES +++ b/distribution/launcher/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES @@ -14,7 +14,7 @@ * limitations under the License. */ -/* Copyright (c) 2008 Google Inc. - // Copyright 2003-2010 Christian d'Heureuse, Inventec Informatik AG, Zurich, Switzerland // www.source-code.biz, www.inventec.ch/chdh + +/* Copyright (c) 2008 Google Inc. diff --git a/distribution/lib/Standard/Table/0.1.0/THIRD-PARTY/com.fasterxml.woodstox.woodstox-core-5.2.1/NOTICES b/distribution/lib/Standard/Table/0.1.0/THIRD-PARTY/com.fasterxml.woodstox.woodstox-core-5.2.1/NOTICES index a214ba7bd6e3..8ef17d630a31 100644 --- a/distribution/lib/Standard/Table/0.1.0/THIRD-PARTY/com.fasterxml.woodstox.woodstox-core-5.2.1/NOTICES +++ b/distribution/lib/Standard/Table/0.1.0/THIRD-PARTY/com.fasterxml.woodstox.woodstox-core-5.2.1/NOTICES @@ -1,5 +1,5 @@ Copyright (c) 2005 Tatu Saloranta, tatu.saloranta@iki.fi -Copyright (c) 2004- Tatu Saloranta, tatu.saloranta@iki.fi - Copyright (c) 2004 Tatu Saloranta, tatu.saloranta@iki.fi + +Copyright (c) 2004- Tatu Saloranta, tatu.saloranta@iki.fi diff --git a/distribution/project-manager/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES b/distribution/project-manager/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES index 10e8435459c1..c4c899bb76e1 100644 --- a/distribution/project-manager/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES +++ b/distribution/project-manager/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES @@ -1,7 +1,7 @@ /* - * Copyright (C) 2009-2020 Lightbend Inc. + * Copyright (C) 2017-2020 Lightbend Inc. */ /* - * Copyright (C) 2017-2020 Lightbend Inc. + * Copyright (C) 2009-2020 Lightbend Inc. */ diff --git a/distribution/project-manager/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES b/distribution/project-manager/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES index d39289b94fc7..1e3906f673f8 100644 --- a/distribution/project-manager/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES +++ b/distribution/project-manager/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES @@ -15,8 +15,6 @@ * limitations under the License. */ -Copyright 2017-2019 John A. De Goes and the ZIO Contributors - /* * Copyright 2017-2020 John A. De Goes and the ZIO Contributors * Copyright 2017-2018 Łukasz Biały, Paul Chiusano, Michael Pilquist, @@ -36,3 +34,5 @@ Copyright 2017-2019 John A. De Goes and the ZIO Contributors * See the License for the specific language governing permissions and * limitations under the License. */ + +Copyright 2017-2019 John A. De Goes and the ZIO Contributors diff --git a/distribution/project-manager/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES b/distribution/project-manager/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES index fea49c9eb3b8..ff981cbc6e70 100644 --- a/distribution/project-manager/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES +++ b/distribution/project-manager/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011,2012 Mathias Doenitz, Johannes Rudolph + * Copyright (C) 2011 Mathias Doenitz * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ /* - * Copyright (C) 2011 Mathias Doenitz + * Copyright (C) 2011,2012 Mathias Doenitz, Johannes Rudolph * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/distribution/project-manager/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES b/distribution/project-manager/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES index 788494853651..c2bba08c1748 100644 --- a/distribution/project-manager/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES +++ b/distribution/project-manager/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES @@ -14,7 +14,7 @@ * limitations under the License. */ -/* Copyright (c) 2008 Google Inc. - // Copyright 2003-2010 Christian d'Heureuse, Inventec Informatik AG, Zurich, Switzerland // www.source-code.biz, www.inventec.ch/chdh + +/* Copyright (c) 2008 Google Inc. diff --git a/tools/legal-review/Table/report-state b/tools/legal-review/Table/report-state index a2c9c1222358..944ed7826a3f 100644 --- a/tools/legal-review/Table/report-state +++ b/tools/legal-review/Table/report-state @@ -1,3 +1,3 @@ B39E86F4C9F95AE99476021C49ED7121CA82A4F734D1D1E493F2A00773D753C2 -440AAA28949F11A1839DEBDB7B4BEEDC6D95E50287A2C852D68427195C1277BE +E19E5AB2117CED2FE672C2C43F47181E21DB10D59506779505DE197A85E2D6F7 0 diff --git a/tools/legal-review/engine/com.typesafe.config-1.3.2/copyright-keep-context b/tools/legal-review/engine/com.typesafe.config-1.4.0/copyright-keep-context similarity index 100% rename from tools/legal-review/engine/com.typesafe.config-1.3.2/copyright-keep-context rename to tools/legal-review/engine/com.typesafe.config-1.4.0/copyright-keep-context index 77674fa31387..6d007f558b96 100644 --- a/tools/legal-review/engine/com.typesafe.config-1.3.2/copyright-keep-context +++ b/tools/legal-review/engine/com.typesafe.config-1.4.0/copyright-keep-context @@ -1,3 +1,3 @@ Copyright (C) 2011-2012 Typesafe Inc. -Copyright (C) 2014 Typesafe Inc. Copyright (C) 2015 Typesafe Inc. +Copyright (C) 2014 Typesafe Inc. diff --git a/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-ignore b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-ignore new file mode 100644 index 000000000000..47f54b72cf57 --- /dev/null +++ b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-ignore @@ -0,0 +1,2 @@ +regarding copyright ownership. The ASF licenses this file +this work for additional information regarding copyright ownership. diff --git a/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-keep-context b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-keep-context new file mode 100644 index 000000000000..6ab681a5b3bd --- /dev/null +++ b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-keep-context @@ -0,0 +1 @@ +Some portions of this file Copyright (c) 2004-2006 Intel Corportation diff --git a/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-ignore b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-ignore new file mode 100644 index 000000000000..0256724c8d06 --- /dev/null +++ b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-ignore @@ -0,0 +1 @@ +META-INF/LICENSE.txt diff --git a/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-keep b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-keep new file mode 100644 index 000000000000..f9a3ec844f02 --- /dev/null +++ b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-keep @@ -0,0 +1 @@ +META-INF/NOTICE.txt diff --git a/tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.2/copyright-keep-context b/tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.2/copyright-keep-context deleted file mode 100644 index 2d1291cd5315..000000000000 --- a/tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.2/copyright-keep-context +++ /dev/null @@ -1 +0,0 @@ -this code has waived all copyright and related or neighboring * diff --git a/tools/legal-review/engine/report-state b/tools/legal-review/engine/report-state index 56d111c2ff2b..de78f167d4c4 100644 --- a/tools/legal-review/engine/report-state +++ b/tools/legal-review/engine/report-state @@ -1,3 +1,3 @@ -61713B297871FFB55B9BDF492049F90A7756BBEDF91DA31B72F0EFB98221754D -C309ED3582802FAA0EA238BB40A24F34CACC4FC4A36334C260232FF49DCD9C72 +038976FC3077212393B2513AB30E376A4E496A9A23EAF71F2CA8B4721ED512E9 +31BAA7838578F7775841BE7BDA52C0BDFDB4F59766C5FA57110D312AE6138651 0 diff --git a/tools/legal-review/launcher/report-state b/tools/legal-review/launcher/report-state index 1a22d8ada645..383a001dff56 100644 --- a/tools/legal-review/launcher/report-state +++ b/tools/legal-review/launcher/report-state @@ -1,3 +1,3 @@ -2D9E29668392299BAE4C1D0D0D7562556712E78CF8FA1022853E42C9E4C05FA1 -0F4AF9C887470D371F850DF94EF5155B3872A8537D0B753FE5ADD94171DFCC35 +56351584030E6947A34374E8672DB955C5A7077228182A1053BE489E9B076315 +28E1F445EA0515C2F3E73ABD9DDB73395B5E0F7A84D07038A1117B782E32A1C6 0 diff --git a/tools/legal-review/project-manager/report-state b/tools/legal-review/project-manager/report-state index df9d55110fee..213a01e574f1 100644 --- a/tools/legal-review/project-manager/report-state +++ b/tools/legal-review/project-manager/report-state @@ -1,3 +1,3 @@ -0AED341E22D16C5722BF722FD5F92039464E9FE3CCD3288D3643F05E09AF62FB -E4B0E3E0602F9227B8D9334913B2D903C652DADA95732D0F55E26B469E82B89C +4A89A53D6497DFF6A8B647FAD7B63844C15C22020B1AC1CC9B260C9BA0819F1F +B7948AB0E996317E7F073260BA28A63D14215436079C09E793F803CF999D607C 0 From 83271a69946e2b69027e7b8d048ed38ff1a82658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 11:51:21 +0200 Subject: [PATCH 17/30] Handle 404 in HTTPDownload, add a test for license warning, add diagnostics to server test --- build.sbt | 1 + .../scala/org/enso/launcher/NativeTest.scala | 5 +- .../components/LauncherRunnerSpec.scala | 2 +- .../enso/launcher/upgrade/UpgradeSpec.scala | 1 + .../enso/downloader/http/HTTPDownload.scala | 49 +++-- .../enso/downloader/http/HTTPException.scala | 15 +- .../cache/DownloadingLibraryCache.scala | 19 +- .../repository/DummyRepository.scala | 33 +++- .../repository/LibraryDownloadTest.scala | 40 ++-- .../org/enso/loggingservice/TestLogger.scala | 6 +- .../test/NativeTestHelper.scala | 186 +----------------- .../org/enso/testkit/process/RunResult.scala | 9 + .../enso/testkit/process/WrappedProcess.scala | 185 +++++++++++++++++ tools/simple-library-server/main.js | 7 +- 14 files changed, 318 insertions(+), 240 deletions(-) create mode 100644 lib/scala/testkit/src/main/scala/org/enso/testkit/process/RunResult.scala create mode 100644 lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala diff --git a/build.sbt b/build.sbt index 512031321569..1c65bc6a5051 100644 --- a/build.sbt +++ b/build.sbt @@ -1403,6 +1403,7 @@ lazy val `library-manager` = project .dependsOn(`distribution-manager`) .dependsOn(downloader) .dependsOn(testkit % Test) + .dependsOn(`logging-service` % Test) lazy val `runtime-version-manager` = project .in(file("lib/scala/runtime-version-manager")) diff --git a/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala b/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala index 2f85ff42e9f2..08a289c53f49 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala @@ -1,9 +1,8 @@ package org.enso.launcher import org.enso.cli.OS - -import java.nio.file.{Files, Path} import org.enso.runtimeversionmanager.test.NativeTestHelper +import org.enso.testkit.process.RunResult import org.scalatest.concurrent.{Signaler, TimeLimitedTests} import org.scalatest.matchers.should.Matchers import org.scalatest.matchers.{MatchResult, Matcher} @@ -11,6 +10,8 @@ import org.scalatest.time.Span import org.scalatest.time.SpanSugar._ import org.scalatest.wordspec.AnyWordSpec +import java.nio.file.{Files, Path} + /** Contains helper methods for creating tests that need to run the native * launcher binary. */ diff --git a/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala index 39c1602175ec..18d0ab35e688 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala @@ -145,7 +145,7 @@ class LauncherRunnerSpec extends RuntimeVersionManagerTest { val runner = makeFakeRunner() val projectPath = getTestDirectory / "project2" val nightlyVersion = SemVer(0, 0, 0, Some("SNAPSHOT.2000-01-01")) - val logs = TestLogger.gatherLogs { + val (_, logs) = TestLogger.gatherLogs { runner .newProject( path = projectPath, diff --git a/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala index a79a5b55abcc..34b3baabd3af 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala @@ -10,6 +10,7 @@ import org.enso.cli.OS import org.enso.launcher._ import org.enso.runtimeversionmanager.test.WithTemporaryDirectory import org.enso.testkit.RetrySpec +import org.enso.testkit.process.{RunResult, WrappedProcess} import org.scalatest.exceptions.TestFailedException import org.scalatest.{BeforeAndAfterAll, OptionValues} diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala index 3415dc5cf4b4..391c5c51240d 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala @@ -18,6 +18,7 @@ import java.nio.charset.{Charset, StandardCharsets} import java.nio.file.Path import scala.concurrent.Future import scala.jdk.CollectionConverters.IterableHasAsJava +import scala.util.{Failure, Success, Try} /** Represents a HTTP header. */ case class Header(name: String, value: String) { @@ -72,14 +73,17 @@ object HTTPDownload { def combineChunks(chunks: Seq[ByteString]): String = chunks.reduceOption(_ ++ _).map(_.decodeString(encoding)).getOrElse("") runRequest( - request.requestImpl, - sizeHint, - Sink.seq, - (response, chunks: Seq[ByteString]) => - APIResponse( - combineChunks(chunks), - response.headers.map(header => Header(header.name, header.value)), - response.status.intValue() + request = request.requestImpl, + sizeHint = sizeHint, + earlyResponseMapping = response => Success(response), + sink = Sink.seq, + resultMapping = (response, chunks: Seq[ByteString]) => + Success( + APIResponse( + combineChunks(chunks), + response.headers.map(header => Header(header.name, header.value)), + response.status.intValue() + ) ) ) } @@ -127,10 +131,20 @@ object HTTPDownload { destination ) runRequest( - request.requestImpl, - sizeHint, - FileIO.toPath(destination), - (_, _: Any) => destination + request = request.requestImpl, + sizeHint = sizeHint, + earlyResponseMapping = { response => + if (response.status.isSuccess) + Success(response) + else if (response.status.intValue == 404) + Failure(ResourceNotFound()) + else + Failure( + HTTPException(s"Server responded with: [${response.status.value}].") + ) + }, + sink = FileIO.toPath(destination), + resultMapping = (_, _: Any) => Success(destination) ) } @@ -186,6 +200,11 @@ object HTTPDownload { * @param sizeHint an optional hint indicating the expected size of the * response. It is used if the response does not include * explicit Content-Length header. + * @param earlyResponseMapping a mapping that can be used to alter the + * response or handle any early errors; it is run + * before passing the response through the + * `sink`; thus it can be used to avoid creating + * downloaded files on failure * @param sink specifies how the response content should be handled, it * receives chunks of [[ByteString]] and should produce a * [[Future]] with some result @@ -199,8 +218,9 @@ object HTTPDownload { private def runRequest[A, B]( request: HttpRequest, sizeHint: Option[Long], + earlyResponseMapping: HttpResponse => Try[HttpResponse], sink: Sink[ByteString, Future[A]], - resultMapping: (HttpResponse, A) => B + resultMapping: (HttpResponse, A) => Try[B] ): TaskProgress[B] = { // TODO [RW] Add optional stream encoding allowing for compression - // add headers and decode the stream if necessary (#1805). @@ -265,8 +285,9 @@ object HTTPDownload { http .singleRequest(request) .flatMap(handleRedirects(maximumRedirects)) + .flatMap(earlyResponseMapping andThen Future.fromTry) .flatMap(handleFinalResponse) - .map(resultMapping.tupled) + .flatMap(resultMapping.tupled andThen Future.fromTry) .onComplete(taskProgress.setComplete) taskProgress } diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPException.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPException.scala index 600b0cd7097c..42cc86da6d78 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPException.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPException.scala @@ -1,5 +1,14 @@ package org.enso.downloader.http -/** Indicates an error when processing a HTTP request. - */ -case class HTTPException(message: String) extends RuntimeException(message) +/** Indicates an error when processing a HTTP request. */ +class HTTPException(message: String) extends RuntimeException(message) + +object HTTPException { + + /** A helper constructor for [[HTTPException]]. */ + def apply(message: String): HTTPException = new HTTPException(message) +} + +/** Indicates that the HTTP request failed with 404 status. */ +case class ResourceNotFound() + extends HTTPException("The server has responded with 404 status.") diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala index 2d15d98781fe..05ef587099cf 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -10,6 +10,7 @@ import org.enso.distribution.locking.{ } import org.enso.distribution.{FileSystem, TemporaryDirectoryManager} import org.enso.downloader.archive.Archive +import org.enso.downloader.http.ResourceNotFound import org.enso.editions.{Editions, LibraryName, LibraryVersion} import org.enso.librarymanager.published.repository.LibraryManifest import org.enso.librarymanager.published.repository.RepositoryHelper.{ @@ -167,13 +168,17 @@ class DownloadingLibraryCache( s"Downloading license of [$libraryName].", licenseDownload ) - TaskProgress.waitForTask(licenseDownload).getOrElse { - // TODO [RW] Once warnings are reported to the IDE (#1860), - // inform that the license file is missing. - logger.warn( - s"License file for library [$libraryName:$version] was missing." - ) - } + TaskProgress + .waitForTask(licenseDownload) + .recoverWith { case ResourceNotFound() => + // TODO [RW] Once warnings are reported to the IDE (#1860), + // inform that the license file is missing. + logger.warn( + s"License file for library [$libraryName:$version] was missing." + ) + Success(()) + } + .get } private def downloadAndExtractArchives( diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index f83a292b84e2..1e0051eda791 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -5,9 +5,11 @@ import org.enso.distribution.FileSystem import org.enso.editions.Editions.RawEdition import org.enso.editions.{Editions, LibraryName} import org.enso.pkg.{Package, PackageManager} +import org.enso.testkit.process.WrappedProcess import java.io.File import java.nio.file.{Files, Path} +import scala.util.control.NonFatal abstract class DummyRepository { @@ -72,19 +74,30 @@ abstract class DummyRepository { ) } - def startServer(port: Int, root: Path): Process = { + def startServer(port: Int, root: Path): WrappedProcess = { val serverDirectory = Path.of("tools/simple-library-server").toAbsolutePath.normalize - (new ProcessBuilder) - .command( - "node", - "main.js", - "--port", - port.toString, - "--root", - root.toAbsolutePath.normalize.toString - ) + val command = Seq( + "node", + "main.js", + "--port", + port.toString, + "--root", + root.toAbsolutePath.normalize.toString + ) + val rawProcess = (new ProcessBuilder) + .command(command: _*) .directory(serverDirectory.toFile) .start() + val process = new WrappedProcess(command, rawProcess) + try { + process.printIO() + process.waitForMessage("Serving the repository", 5, process.StdOut) + } catch { + case NonFatal(e) => + process.kill() + throw e + } + process } } diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala index a35d56a18dd1..930d1fce47bc 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -10,13 +10,17 @@ import org.enso.distribution.locking.{ import org.enso.distribution.{FileSystem, TemporaryDirectoryManager} import org.enso.editions.Editions import org.enso.librarymanager.published.cache.DownloadingLibraryCache +import org.enso.loggingservice.{LogLevel, TestLogger} +import org.enso.loggingservice.TestLogger.TestLogMessage import org.enso.pkg.PackageManager import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import java.nio.file.Files + class LibraryDownloadTest extends AnyWordSpec with Matchers { - val port: Int = 54325 + val port: Int = 47306 "DownloadingLibraryCache" should { "be able to download and install libraries from a repository" in { @@ -53,23 +57,35 @@ class LibraryDownloadTest extends AnyWordSpec with Matchers { repo.testLib.version ) shouldBe empty - val libPath = cache - .findOrInstallLibrary( - repo.testLib.libraryName, - repo.testLib.version, - Editions - .Repository("test_repo", s"http://localhost:$port/libraries") - ) - .get + val (libPath, logs) = TestLogger.gatherLogs { + cache + .findOrInstallLibrary( + repo.testLib.libraryName, + repo.testLib.version, + Editions + .Repository("test_repo", s"http://localhost:$port/libraries") + ) + .get + } val pkg = PackageManager.Default.loadPackage(libPath.toFile).get pkg.name shouldEqual "Bar" val sources = pkg.listSources sources should have size 1 sources.head.file.getName shouldEqual "Main.enso" - // TODO [RW] check that the license is missing + assert( + Files.notExists(libPath.resolve("LICENSE.md")), + "The license file should not exist as it was not provided " + + "in the repository." + ) + logs should contain( + TestLogMessage( + LogLevel.Warning, + "License file for library [Foo.Bar:1.0.0] was missing." + ) + ) } finally { - server.destroy() - server.waitFor() + server.kill() + server.join() } } } diff --git a/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala b/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala index e05b8ad27dfe..6bb6026575f4 100644 --- a/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala +++ b/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala @@ -21,7 +21,7 @@ object TestLogger { * be ran with `parallelExecution` set to false, as global logger state has * to be modified to gather the logs. */ - def gatherLogs(action: => Unit): Seq[TestLogMessage] = { + def gatherLogs[R](action: => R): (R, Seq[TestLogMessage]) = { LoggingServiceManager.dropPendingLogs() if (LoggingServiceManager.isSetUp()) { throw new IllegalStateException( @@ -35,10 +35,10 @@ object TestLogger { LogLevel.Trace ) Await.ready(future, 1.second) - action + val result = action Thread.sleep(100) LoggingServiceManager.tearDown() - printer.getLoggedMessages + (result, printer.getLoggedMessages) } /** Drops any logs that are pending due to the logging service not being set diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala index e511d9f4ee0c..5588e3054591 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala @@ -1,20 +1,10 @@ package org.enso.runtimeversionmanager.test import org.enso.cli.OS +import org.enso.testkit.process.{RunResult, WrappedProcess} -import java.io.{ - BufferedReader, - IOException, - InputStream, - InputStreamReader, - PrintWriter -} -import java.util.concurrent.{Semaphore, TimeUnit} import java.lang.{ProcessBuilder => JProcessBuilder} -import scala.collection.Factory -import scala.concurrent.TimeoutException import scala.jdk.CollectionConverters._ -import scala.jdk.StreamConverters._ /** A mix-in providing helper functions for running native commands in tests. * @@ -23,14 +13,6 @@ import scala.jdk.StreamConverters._ */ trait NativeTestHelper { - /** A result of running the native launcher binary. - * - * @param exitCode the returned exit code - * @param stdout contents of the standard output stream - * @param stderr contents of the standard error stream - */ - case class RunResult(exitCode: Int, stdout: String, stderr: String) - /** Starts the provided `command`. * * `extraEnv` may be provided to extend the environment. Care must be taken @@ -80,172 +62,6 @@ trait NativeTestHelper { } } - /** Represents a started and possibly running process. */ - class WrappedProcess(command: Seq[String], process: Process) { - - private val outQueue = - new java.util.concurrent.LinkedTransferQueue[String]() - private val errQueue = - new java.util.concurrent.LinkedTransferQueue[String]() - - sealed trait StreamType - case object StdErr extends StreamType - case object StdOut extends StreamType - @volatile private var ioHandlers: Seq[(String, StreamType) => Unit] = Seq() - - private def watchStream( - stream: InputStream, - streamType: StreamType - ): Unit = { - val reader = new BufferedReader(new InputStreamReader(stream)) - var line: String = null - val queue = streamType match { - case StdErr => errQueue - case StdOut => outQueue - } - try { - while ({ line = reader.readLine(); line != null }) { - queue.add(line) - ioHandlers.foreach(f => f(line, streamType)) - } - } catch { - case _: InterruptedException => - case _: IOException => - ioHandlers.foreach(f => f("", streamType)) - } - } - - private val outThread = new Thread(() => - watchStream(process.getInputStream, StdOut) - ) - private val errThread = new Thread(() => - watchStream(process.getErrorStream, StdErr) - ) - outThread.start() - errThread.start() - - /** Waits for a message on the stderr to appear. */ - def waitForMessageOnErrorStream( - message: String, - timeoutSeconds: Long - ): Unit = { - val semaphore = new Semaphore(0) - def handler(line: String, streamType: StreamType): Unit = { - if (streamType == StdErr && line.contains(message)) { - semaphore.release() - } - } - - this.synchronized { - ioHandlers ++= Seq(handler _) - } - - errQueue.asScala.toSeq.foreach(handler(_, StdErr)) - - val acquired = semaphore.tryAcquire(timeoutSeconds, TimeUnit.SECONDS) - if (!acquired) { - throw new TimeoutException(s"Waiting for `$message` timed out.") - } - } - - private lazy val inputWriter = new PrintWriter(process.getOutputStream) - - /** Prints a message to the standard input stream of the process. - * - * Does not append any newlines, so if a newline is expected, it has to be - * contained within the `message`. - */ - def sendToInputStream(message: String): Unit = { - inputWriter.print(message) - inputWriter.flush() - } - - /** Starts printing the stdout and stderr of the started process to the - * stdout with prefixes to indicate that these messages come from another - * process. - * - * It also prints lines that were printed before invoking this method. - * Thus, it is possible that a line may be printed twice (once as - * 'before-printIO' and once normally). - */ - def printIO(): Unit = { - def handler(line: String, streamType: StreamType): Unit = { - val prefix = streamType match { - case StdErr => "stderr> " - case StdOut => "stdout> " - } - println(prefix + line) - } - this.synchronized { - ioHandlers ++= Seq(handler _) - } - outQueue.asScala.toSeq.foreach(line => - println(s"stdout-before-printIO> $line") - ) - errQueue.asScala.toSeq.foreach(line => - println(s"stderr-before-printIO> $line") - ) - } - - /** Checks if the process is still running. */ - def isAlive: Boolean = process.isAlive - - /** Tries to kill the process immediately. */ - def kill(): Unit = { - process.destroyForcibly() - } - - /** Waits for the process to finish and returns its [[RunResult]]. - * - * If `waitForDescendants` is set, tries to wait for descendants of the - * launched process to finish too. Especially important on Windows where - * child processes may run after the launcher parent has been terminated. - * - * It will timeout after `timeoutSeconds` and try to kill the process (or - * its descendants), although it may not always be able to. - */ - def join( - waitForDescendants: Boolean = true, - timeoutSeconds: Long = 15 - ): RunResult = { - var descendants: Seq[ProcessHandle] = Seq() - try { - val exitCode = - if (process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) - process.exitValue() - else throw new TimeoutException("Process timed out") - if (waitForDescendants) { - descendants = - process.descendants().toScala(Factory.arrayFactory).toSeq - descendants.foreach(_.onExit().get(timeoutSeconds, TimeUnit.SECONDS)) - } - errThread.join(1000) - outThread.join(1000) - if (errThread.isAlive) { - errThread.interrupt() - } - if (outThread.isAlive) { - outThread.interrupt() - } - val stdout = outQueue.asScala.toSeq.mkString("\n") - val stderr = errQueue.asScala.toSeq.mkString("\n") - RunResult(exitCode, stdout, stderr) - } catch { - case e @ (_: InterruptedException | _: TimeoutException) => - if (process.isAlive) { - println(s"Killing the timed-out process: ${command.mkString(" ")}") - process.destroyForcibly() - } - for (processHandle <- descendants) { - if (processHandle.isAlive) { - processHandle.destroyForcibly() - } - } - throw e - } - } - } - /** Runs the provided `command`. * * `extraEnv` may be provided to extend the environment. Care must be taken diff --git a/lib/scala/testkit/src/main/scala/org/enso/testkit/process/RunResult.scala b/lib/scala/testkit/src/main/scala/org/enso/testkit/process/RunResult.scala new file mode 100644 index 000000000000..b100246b56ae --- /dev/null +++ b/lib/scala/testkit/src/main/scala/org/enso/testkit/process/RunResult.scala @@ -0,0 +1,9 @@ +package org.enso.testkit.process + +/** A result of running a process. + * + * @param exitCode the returned exit code + * @param stdout contents of the standard output stream + * @param stderr contents of the standard error stream + */ +case class RunResult(exitCode: Int, stdout: String, stderr: String) diff --git a/lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala b/lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala new file mode 100644 index 000000000000..f35d65496ce9 --- /dev/null +++ b/lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala @@ -0,0 +1,185 @@ +package org.enso.testkit.process + +import java.io._ +import java.util.concurrent.{Semaphore, TimeUnit} +import scala.collection.Factory +import scala.concurrent.TimeoutException +import scala.jdk.CollectionConverters._ +import scala.jdk.StreamConverters._ + +/** Represents a started and possibly running process. */ +class WrappedProcess(command: Seq[String], process: Process) { + + private val outQueue = + new java.util.concurrent.LinkedTransferQueue[String]() + private val errQueue = + new java.util.concurrent.LinkedTransferQueue[String]() + + sealed trait StreamType + case object StdErr extends StreamType + case object StdOut extends StreamType + @volatile private var ioHandlers: Seq[(String, StreamType) => Unit] = Seq() + + private def watchStream( + stream: InputStream, + streamType: StreamType + ): Unit = { + val reader = new BufferedReader(new InputStreamReader(stream)) + var line: String = null + val queue = streamType match { + case StdErr => errQueue + case StdOut => outQueue + } + try { + while ({ line = reader.readLine(); line != null }) { + queue.add(line) + ioHandlers.foreach(f => f(line, streamType)) + } + } catch { + case _: InterruptedException => + case _: IOException => + ioHandlers.foreach(f => f("", streamType)) + } + } + + private val outThread = new Thread(() => + watchStream(process.getInputStream, StdOut) + ) + private val errThread = new Thread(() => + watchStream(process.getErrorStream, StdErr) + ) + outThread.start() + errThread.start() + + /** Waits for a message on the stderr to appear. */ + def waitForMessageOnErrorStream( + message: String, + timeoutSeconds: Long + ): Unit = waitForMessage(message, timeoutSeconds, StdErr) + + /** Waits for a message on one of the output streams. */ + def waitForMessage( + message: String, + timeoutSeconds: Long, + stream: StreamType + ): Unit = { + val semaphore = new Semaphore(0) + def handler(line: String, streamType: StreamType): Unit = { + if (streamType == stream && line.contains(message)) { + semaphore.release() + } + } + + this.synchronized { + ioHandlers ++= Seq(handler _) + } + + stream match { + case StdErr => + errQueue.asScala.toSeq.foreach(handler(_, StdErr)) + case StdOut => + outQueue.asScala.toSeq.foreach(handler(_, StdOut)) + } + + val acquired = semaphore.tryAcquire(timeoutSeconds, TimeUnit.SECONDS) + if (!acquired) { + throw new TimeoutException(s"Waiting for `$message` timed out.") + } + } + + private lazy val inputWriter = new PrintWriter(process.getOutputStream) + + /** Prints a message to the standard input stream of the process. + * + * Does not append any newlines, so if a newline is expected, it has to be + * contained within the `message`. + */ + def sendToInputStream(message: String): Unit = { + inputWriter.print(message) + inputWriter.flush() + } + + /** Starts printing the stdout and stderr of the started process to the + * stdout with prefixes to indicate that these messages come from another + * process. + * + * It also prints lines that were printed before invoking this method. + * Thus, it is possible that a line may be printed twice (once as + * 'before-printIO' and once normally). + */ + def printIO(): Unit = { + def handler(line: String, streamType: StreamType): Unit = { + val prefix = streamType match { + case StdErr => "stderr> " + case StdOut => "stdout> " + } + println(prefix + line) + } + this.synchronized { + ioHandlers ++= Seq(handler _) + } + outQueue.asScala.toSeq.foreach(line => + println(s"stdout-before-printIO> $line") + ) + errQueue.asScala.toSeq.foreach(line => + println(s"stderr-before-printIO> $line") + ) + } + + /** Checks if the process is still running. */ + def isAlive: Boolean = process.isAlive + + /** Tries to kill the process immediately. */ + def kill(): Unit = { + process.destroyForcibly() + } + + /** Waits for the process to finish and returns its [[RunResult]]. + * + * If `waitForDescendants` is set, tries to wait for descendants of the + * launched process to finish too. Especially important on Windows where + * child processes may run after the launcher parent has been terminated. + * + * It will timeout after `timeoutSeconds` and try to kill the process (or + * its descendants), although it may not always be able to. + */ + def join( + waitForDescendants: Boolean = true, + timeoutSeconds: Long = 15 + ): RunResult = { + var descendants: Seq[ProcessHandle] = Seq() + try { + val exitCode = + if (process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) + process.exitValue() + else throw new TimeoutException("Process timed out") + if (waitForDescendants) { + descendants = process.descendants().toScala(Factory.arrayFactory).toSeq + descendants.foreach(_.onExit().get(timeoutSeconds, TimeUnit.SECONDS)) + } + errThread.join(1000) + outThread.join(1000) + if (errThread.isAlive) { + errThread.interrupt() + } + if (outThread.isAlive) { + outThread.interrupt() + } + val stdout = outQueue.asScala.toSeq.mkString("\n") + val stderr = errQueue.asScala.toSeq.mkString("\n") + RunResult(exitCode, stdout, stderr) + } catch { + case e @ (_: InterruptedException | _: TimeoutException) => + if (process.isAlive) { + println(s"Killing the timed-out process: ${command.mkString(" ")}") + process.destroyForcibly() + } + for (processHandle <- descendants) { + if (processHandle.isAlive) { + processHandle.destroyForcibly() + } + } + throw e + } + } +} diff --git a/tools/simple-library-server/main.js b/tools/simple-library-server/main.js index 83ead01cca33..d33ce4079d0d 100755 --- a/tools/simple-library-server/main.js +++ b/tools/simple-library-server/main.js @@ -22,13 +22,14 @@ const argv = yargs .help() .alias("help", "h").argv; +const app = express(); +app.use(compression({ filter: shouldCompress })); +app.use(express.static(argv.root)); + console.log( `Serving the repository located under ${argv.root} on port ${argv.port}.` ); -const app = express(); -app.use(compression({ filter: shouldCompress })); -app.use(express.static(argv.root)); app.listen(argv.port); function shouldCompress(req, res) { From 312748b31410893cd481907054b95998a880a860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 11:55:29 +0200 Subject: [PATCH 18/30] CHANGELOG --- RELEASES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 314c67a79d93..136dbe9c632f 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,5 +1,11 @@ # Enso Next +## Tooling + +- Implement a basic library downloader + ([#1885](https://github.com/enso-org/enso/pull/1885)), allowing to download + missing libraries. + ## Libraries - Added support for reading XLS and XLSX spreadsheets From 42f2d40fe3dd358f2d6b9c25b0c65e115a55a663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 12:39:35 +0200 Subject: [PATCH 19/30] docs --- .../org/enso/compiler/PackageRepository.scala | 19 +++++++------- .../instrument/NotificationHandler.scala | 6 +++-- ...ProgressAndLockNotificationForwarder.scala | 7 +++++ .../TemporaryDirectoryManager.scala | 5 ++++ .../enso/downloader/http/APIResponse.scala | 11 ++++++++ .../enso/downloader/http/HTTPDownload.scala | 21 +-------------- .../org/enso/downloader/http/Header.scala | 12 +++++++++ .../editions/updater/EditionUpdater.scala | 2 +- .../scala/org/enso/editions/EditionName.scala | 16 +++++++++++- .../provider/FileSystemEditionProvider.scala | 14 +++------- .../enso/editions/repository/Manifest.scala | 2 +- .../DefaultLibraryProvider.scala | 22 +++++++--------- .../published/cache/LibraryResource.scala | 1 + ...Exception.scala => LibraryException.scala} | 2 ++ .../repository/LibraryManifest.scala | 2 +- .../published/repository/ArchiveWriter.scala | 11 ++++++++ .../repository/DummyRepository.scala | 26 +++++++++++++++++++ .../repository/ExampleRepository.scala | 8 +++++- .../locking/Resources.scala | 6 ++--- 19 files changed, 130 insertions(+), 63 deletions(-) create mode 100644 lib/scala/downloader/src/main/scala/org/enso/downloader/http/APIResponse.scala create mode 100644 lib/scala/downloader/src/main/scala/org/enso/downloader/http/Header.scala rename lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/{LibraryNotFoundException.scala => LibraryException.scala} (75%) diff --git a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala index 1f41e204a26c..d645a88f0c42 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala @@ -199,16 +199,14 @@ object PackageRepository { else { logger.trace(s"Resolving library $libraryName.") val resolvedLibrary = libraryProvider.findLibrary(libraryName) - logger.whenTraceEnabled { - resolvedLibrary match { - case Left(error) => - logger.trace(s"Resolution failed with [$error].") - case Right(resolved) => - logger.trace( - s"Found library ${resolved.name} @ ${resolved.version} " + - s"at [${MaskedPath(resolved.location).applyMasking()}]." - ) - } + resolvedLibrary match { + case Left(error) => + logger.error(s"Resolution failed with [$error].", error) + case Right(resolved) => + logger.trace( + s"Found library ${resolved.name} @ ${resolved.version} " + + s"at [${MaskedPath(resolved.location).applyMasking()}]." + ) } this.synchronized { @@ -319,6 +317,7 @@ object PackageRepository { * @param projectPackage the package of the current project (if ran inside of a project) * @param languageHome the language home (if set) * @param distributionManager the distribution manager + * @param resourceManager the resource manager instance * @param context the context reference, needed to add polyglot libraries to * the classpath * @param builtins the builtins that are always preloaded diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala index 0160a12069df..5217ce9430f5 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala @@ -10,8 +10,8 @@ import org.enso.polyglot.runtime.Runtime.{Api, ApiResponse} import java.nio.file.Path -/** A class that forwards notifications about loaded libraries and long-running - * tasks to the user interface. +/** A class that forwards notifications about loaded libraries, locks and + * long-running tasks to the user interface. */ trait NotificationHandler extends ProgressReporter with LockUserInterface { @@ -56,9 +56,11 @@ object NotificationHandler { } } + /** @inheritdoc */ override def startWaitingForResource(resource: Resource): Unit = logger.warn(resource.waitMessage) + /** @inheritdoc */ override def finishWaitingForResource(resource: Resource): Unit = () } diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala index e91f8fac0feb..6a31a5253792 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala @@ -9,6 +9,13 @@ import org.enso.distribution.locking.{LockUserInterface, Resource} import java.util.UUID +/** A helper class that provides an implementation of + * [[ProgressNotificationForwarder]] and [[LockUserInterface]] which are also + * forwarded as progress notifications with indeterminate progress amounts. + * + * All it needs to function is for the user to define the + * `sendProgressNotification` method. + */ abstract class ProgressAndLockNotificationForwarder extends ProgressNotificationForwarder with LockUserInterface { diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala index 6b18bcac0af2..62b6300a266e 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala @@ -85,6 +85,11 @@ class TemporaryDirectoryManager( } object TemporaryDirectoryManager { + + /** A helper constructor that creates a [[TemporaryDirectoryManager]] using + * the temporary directory inside of the distribution managed by the provided + * [[DistributionManager]]. + */ def apply( distribution: DistributionManager, resourceManager: ResourceManager diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/APIResponse.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/APIResponse.scala new file mode 100644 index 000000000000..851f45f4b3f5 --- /dev/null +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/APIResponse.scala @@ -0,0 +1,11 @@ +package org.enso.downloader.http + +/** Contains the response contents as a string alongside with the headers + * included in the response. + * + * @param content the response decoded as a string + * @param headers sequence of headers included in the response + * @param statusCode the response status code, indicating whether the request + * has succeeded or failed + */ +case class APIResponse(content: String, headers: Seq[Header], statusCode: Int) diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala index 391c5c51240d..497aa0ade71a 100644 --- a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala @@ -20,25 +20,6 @@ import scala.concurrent.Future import scala.jdk.CollectionConverters.IterableHasAsJava import scala.util.{Failure, Success, Try} -/** Represents a HTTP header. */ -case class Header(name: String, value: String) { - - /** Checks if this header instance corresponds to a `headerName`. - * - * The check is case-insensitive. - */ - def is(headerName: String): Boolean = - name.toLowerCase == headerName.toLowerCase -} - -/** Contains the response contents as a string alongside with the headers - * included in the response. - * - * @param content the response decoded as a string - * @param headers sequence of headers included in the response - */ -case class APIResponse(content: String, headers: Seq[Header], statusCode: Int) - /** Contains utility functions for fetching data using the HTTP(S) protocol. */ object HTTPDownload { private val logger = Logger[HTTPDownload.type] @@ -204,7 +185,7 @@ object HTTPDownload { * response or handle any early errors; it is run * before passing the response through the * `sink`; thus it can be used to avoid creating - * downloaded files on failure + * downloaded files if the request fails * @param sink specifies how the response content should be handled, it * receives chunks of [[ByteString]] and should produce a * [[Future]] with some result diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/Header.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/Header.scala new file mode 100644 index 000000000000..bc60d9c1052d --- /dev/null +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/Header.scala @@ -0,0 +1,12 @@ +package org.enso.downloader.http + +/** Represents a HTTP header. */ +case class Header(name: String, value: String) { + + /** Checks if this header instance corresponds to a `headerName`. + * + * The check is case-insensitive. + */ + def is(headerName: String): Boolean = + name.toLowerCase == headerName.toLowerCase +} diff --git a/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala b/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala index 683b267fb7eb..d380895ba64e 100644 --- a/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala +++ b/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala @@ -59,7 +59,7 @@ class EditionUpdater(cachePath: Path, sources: Seq[String]) { repositoryRoot: URIBuilder ): Try[Manifest] = Try { - val uri = repositoryRoot.addPathSegment(Manifest.fileName).build() + val uri = repositoryRoot.addPathSegment(Manifest.filename).build() val request = HTTPRequestBuilder.fromURI(uri).GET val response = HTTPDownload.fetchString(request).force() YamlHelper.parseString[Manifest](response.content).toTry.get diff --git a/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala b/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala index a51dbdf27bcc..757763b6fafa 100644 --- a/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala +++ b/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala @@ -11,7 +11,7 @@ import io.circe.Decoder case class EditionName(name: String) extends AnyVal { /** Returns the name of the file that is associated with the edition name. */ - def toFileName: String = s"$name.yaml" + def toFileName: String = name + EditionName.editionSuffix } object EditionName { @@ -29,4 +29,18 @@ object EditionName { .orElse(json.as[Float].map(_.toString)) .map(EditionName(_)) } + + /** The filename suffix that is used to create a filename corresponding to a + * named edition. + */ + val editionSuffix = ".yaml" + + /** Creates an [[EditionName]] from the corresponding filename. + * + * Returns None if the filename does not correspond to an edition. + */ + def fromFilename(filename: String): Option[EditionName] = + if (filename.endsWith(editionSuffix)) + Some(EditionName(filename.stripSuffix(editionSuffix))) + else None } diff --git a/lib/scala/editions/src/main/scala/org/enso/editions/provider/FileSystemEditionProvider.scala b/lib/scala/editions/src/main/scala/org/enso/editions/provider/FileSystemEditionProvider.scala index bfd96b8a1418..46e7e2c5fcae 100644 --- a/lib/scala/editions/src/main/scala/org/enso/editions/provider/FileSystemEditionProvider.scala +++ b/lib/scala/editions/src/main/scala/org/enso/editions/provider/FileSystemEditionProvider.scala @@ -1,6 +1,6 @@ package org.enso.editions.provider -import org.enso.editions.{EditionSerialization, Editions} +import org.enso.editions.{EditionName, EditionSerialization, Editions} import java.io.FileNotFoundException import java.nio.file.{Files, Path} @@ -26,8 +26,6 @@ class FileSystemEditionProvider(searchPaths: List[Path]) } } - private val editionSuffix = ".yaml" - @tailrec private def findEdition( name: String, @@ -52,7 +50,7 @@ class FileSystemEditionProvider(searchPaths: List[Path]) name: String, path: Path ): Either[EditionLoadingError, Editions.Raw.Edition] = { - val fileName = name + editionSuffix + val fileName = EditionName(name).toFileName val editionPath = path.resolve(fileName) if (Files.exists(editionPath)) { EditionSerialization @@ -67,12 +65,8 @@ class FileSystemEditionProvider(searchPaths: List[Path]) def findAvailableEditions(): Seq[String] = searchPaths.flatMap(findEditionsAt).distinct - private def findEditionName(path: Path): Option[String] = { - val name = path.getFileName.toString - if (name.endsWith(editionSuffix)) { - Some(name.stripSuffix(editionSuffix)) - } else None - } + private def findEditionName(path: Path): Option[String] = + EditionName.fromFilename(path.getFileName.toString).map(_.name) private def findEditionsAt(path: Path): Seq[String] = listDir(path).filter(Files.isRegularFile(_)).flatMap(findEditionName) diff --git a/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala b/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala index 1aa3d42a5e38..5fa69170616c 100644 --- a/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala +++ b/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala @@ -23,5 +23,5 @@ object Manifest { /** The name of the manifest file that should be present at the root of * editions repository. */ - val fileName = "manifest.yaml" + val filename = "manifest.yaml" } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala index 0cc90ff74d17..c2c8184b4d72 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala @@ -24,6 +24,10 @@ import java.nio.file.Path * * @param distributionManager a distribution manager * @param resourceManager a resource manager + * @param lockUserInterface an interface that will handle notifications + * about waiting on locks + * @param progressReporter an interface that will handle progress + * notifications * @param languageHome a language home which may contain bundled libraries * @param edition the edition used in the project * @param preferLocalLibraries project setting whether to use local libraries @@ -110,18 +114,12 @@ class DefaultLibraryProvider( } case Right(version @ LibraryVersion.Published(semver, repository)) => - val res = - publishedLibraryProvider - .findLibrary(libraryName, semver, repository) - .map(ResolvedLibrary(libraryName, version, _)) - .toEither - res match { - case Left(value) => - println(s"Download error: $value") - value.printStackTrace() - case Right(_) => - } - res.left.map(ResolvingLibraryProvider.Error.DownloadFailed) + publishedLibraryProvider + .findLibrary(libraryName, semver, repository) + .map(ResolvedLibrary(libraryName, version, _)) + .toEither + .left + .map(ResolvingLibraryProvider.Error.DownloadFailed) } } } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala index fa23a78eac0d..0a8c2a257c32 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala @@ -4,6 +4,7 @@ import nl.gn0s1s.bump.SemVer import org.enso.distribution.locking.Resource import org.enso.editions.LibraryName +/** A resource that synchronizes installation of a library in the cache. */ case class LibraryResource(libraryName: LibraryName, version: SemVer) extends Resource { override def name: String = diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryNotFoundException.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryException.scala similarity index 75% rename from lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryNotFoundException.scala rename to lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryException.scala index 848ea4ad9deb..e4f8b7cdb8a1 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryNotFoundException.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryException.scala @@ -3,9 +3,11 @@ package org.enso.librarymanager.published.repository import nl.gn0s1s.bump.SemVer import org.enso.editions.LibraryName +/** Indicates that the library could not be downloaded. */ sealed class LibraryDownloadFailure(message: String) extends RuntimeException(message) +/** Indicates that the library was not found in the recommended repository. */ case class LibraryNotFoundException( libraryName: LibraryName, version: SemVer, diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala index 1ff0a74df116..af2cf5229db2 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala @@ -46,5 +46,5 @@ object LibraryManifest { /** The name of the manifest file as included in the directory associated with * a given library in the library repository. */ - val fileName = "manifest.yaml" + val filename = "manifest.yaml" } diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala index 1fde03cde031..023f1b89a8e2 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala @@ -10,13 +10,24 @@ import java.io.{BufferedOutputStream, FileOutputStream} import java.nio.file.Path import scala.util.Using +/** A helper class used for creating TAR-GZ archives in tests. */ object ArchiveWriter { + + /** A file to add to the archive. */ sealed trait FileToWrite { + + /** The path that this file should have within the archive. */ def relativePath: String } + /** Represents a text file to be added to a test archive. + * + * @param relativePath the path in the archive + * @param content the text contents for the file + */ case class TextFile(relativePath: String, content: String) extends FileToWrite + /** Creates a tar archive at the given path, containing the provided files. */ def writeTarArchive(path: Path, files: Seq[FileToWrite]): Unit = { Using(new FileOutputStream(path.toFile)) { outputStream => Using(new BufferedOutputStream(outputStream)) { bufferedStream => diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index 1e0051eda791..c3b38725df5a 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -11,18 +11,33 @@ import java.io.File import java.nio.file.{Files, Path} import scala.util.control.NonFatal +/** A helper class managing a library repository for testing purposes. */ abstract class DummyRepository { + /** A library used for testing. + * + * @param libraryName name of the library + * @param version version of the library + * @param mainContent contents of the `Main.enso` file + */ case class DummyLibrary( libraryName: LibraryName, version: SemVer, mainContent: String ) + /** Name of the repository, as it will be indicated in the generated edition. + */ def repoName: String = "test_repo" + /** Sequence of libraries to create in the repository and include in the + * edition. + */ def libraries: Seq[DummyLibrary] + /** Creates a directory structure for the repository at the given root and + * populates it with [[libraries]]. + */ def createRepository(root: Path): Unit = { for (lib <- libraries) { val libraryRoot = root @@ -63,6 +78,11 @@ abstract class DummyRepository { ) } + /** Creates an edition which contains libraries defined in this repository. + * + * @param repoUrl the URL where the repository is going to be accessible; the + * URL should include the `libraries` prefix + */ def createEdition(repoUrl: String): RawEdition = { Editions.Raw.Edition( parent = Some(buildinfo.Info.currentEdition), @@ -74,6 +94,12 @@ abstract class DummyRepository { ) } + /** Starts a server for the library repository. + * + * @param port port to listen on + * @param root root of the library repository, the same as the argument to + * [[createRepository]] + */ def startServer(port: Int, root: Path): WrappedProcess = { val serverDirectory = Path.of("tools/simple-library-server").toAbsolutePath.normalize diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala index 7fd64d44cddc..dbd2f11b9232 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala @@ -3,8 +3,13 @@ package org.enso.librarymanager.published.repository import nl.gn0s1s.bump.SemVer import org.enso.editions.LibraryName +/** A simple [[DummyRepository]] containing a single library for testing + * downloads. + */ class ExampleRepository extends DummyRepository { - val testLib = DummyLibrary( + + /** The library provided by this repository. */ + val testLib: DummyLibrary = DummyLibrary( LibraryName("Foo", "Bar"), SemVer(1, 0, 0), """baz = 42 @@ -13,5 +18,6 @@ class ExampleRepository extends DummyRepository { |""".stripMargin ) + /** @inheritdoc */ override def libraries: Seq[DummyLibrary] = Seq(testLib) } diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/locking/Resources.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/locking/Resources.scala index 7574f476cd26..f8e2ee977271 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/locking/Resources.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/locking/Resources.scala @@ -6,8 +6,7 @@ import org.enso.runtimeversionmanager.components.GraalVMVersion object Resources { - /** Synchronizes launcher upgrades. - */ + /** Synchronizes launcher upgrades. */ case object LauncherExecutable extends Resource { override def name: String = "launcher-executable" override def waitMessage: String = @@ -15,8 +14,7 @@ object Resources { "the current process must wait until it is completed." } - /** This resource is held when adding or removing any components. - */ + /** This resource is held when adding or removing any components. */ case object AddOrRemoveComponents extends Resource { override def name: String = "add-remove-components" override def waitMessage: String = From 54490424a9d9a5bed2d6ca0127f05fae7b3f6714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 13:15:59 +0200 Subject: [PATCH 20/30] More docs --- .../cache/DownloadingLibraryCache.scala | 63 ++++++++++----- .../repository/RepositoryHelper.scala | 76 +++++++++++++++---- 2 files changed, 105 insertions(+), 34 deletions(-) diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala index 05ef587099cf..9ff2b0a01724 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -1,4 +1,5 @@ package org.enso.librarymanager.published.cache + import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer import org.enso.cli.task.{ProgressReporter, TaskProgress} @@ -26,11 +27,14 @@ import scala.util.{Success, Try} /** A [[LibraryCache]] that will try to download missing libraries. * - * @param cacheRoot - * @param temporaryDirectoryRoot - * @param resourceManager - * @param lockUserInterface - * @param progressReporter + * @param cacheRoot the root of the library cache + * @param temporaryDirectoryManager a local temporary directory used to store + * intermediate files during installation + * @param resourceManager the resource manager instance + * @param lockUserInterface an interface that will handle notifications + * about waiting on locks + * @param progressReporter an interface that will handle progress + * notifications */ class DownloadingLibraryCache( cacheRoot: Path, @@ -129,6 +133,7 @@ class DownloadingLibraryCache( } } + /** Downloads and parses the library manifest. */ private def downloadManifest( libraryName: LibraryName, access: LibraryAccess @@ -150,6 +155,12 @@ class DownloadingLibraryCache( private def verifyPackageIntegrity(packageRoot: Path): Unit = PackageManager.Default.loadPackage(packageRoot.toFile).get + /** Downloads the package config and license file. + * + * If the license file does not exist, a warning is issued, but the + * installation proceeds. However if it fails to download for other reasons, + * the installation fails in the same way as it would for any other file. + */ private def downloadLooseFiles( libraryName: LibraryName, version: SemVer, @@ -181,6 +192,11 @@ class DownloadingLibraryCache( .get } + /** Downloads relevant library sub-archvies and extracts them to the library + * root. + * + * All archives are assumed to be gzipped TAR archives. + */ private def downloadAndExtractArchives( libraryName: LibraryName, access: LibraryAccess, @@ -218,11 +234,17 @@ class DownloadingLibraryCache( } } + /** Checks if a given sub-archive should be downloaded. + * + * Currently all archives, apart from the ones starting with `tests`, are + * downloaded. + */ private def shouldDownloadArchive(archiveName: String): Boolean = { val isTestData = archiveName.startsWith("tests") !isTestData } + /** @inheritdoc */ override def preinstallLibrary( libraryName: LibraryName, version: SemVer, @@ -230,6 +252,9 @@ class DownloadingLibraryCache( dependencyResolver: LibraryName => Option[LibraryVersion] ): Try[Unit] = { logger.warn("Predownloading dependencies is not yet implemented.") + // TODO [RW] until fully fledged dependency preinstall is implemented, it + // just preinstalls the library itself; if the library has any + // dependencies, they will be downloaded by the compiler val _ = dependencyResolver findOrInstallLibrary(libraryName, version, recommendedRepository) .map(_ => ()) @@ -249,17 +274,19 @@ class DownloadingLibraryCache( * * Thanks to the mentioned assumption, once a library is present in the cache, * we can assume that it will not disappear, so we do not need to synchronize - * read access. The only thing that needs to be synchronized is installing - * libraries - to make sure that if two processes try to install the same - * library, only one of them actually performs the action. We also need to be - * sure that when one process checks if the library exists, and if another - * process is in the middle of installing it, it will not yet report it as - * existing (as this could lead to loading an only-partially installed library). - * The primary way of ensuring that will be to install the libraries to a - * temporary cache directory next to the true cache and atomically move it at - * the end of the operation. However as we do not have real guarantees that the - * filesystem move is atomic (although most of the time it should be if it is - * within a single filesystem), we will use locking to ensure consistency. + * read access (after checking that the library does indeed exist). What needs + * to be synchronized is installing libraries - to make sure that if two + * processes try to install the same library, only one of them actually performs + * the action. We also need to be sure that when one process checks if the + * library exists, and if another process is in the middle of installing it, it + * will not yet report it as existing (as this could lead to loading an + * only-partially installed library). The primary way of ensuring that will be + * to install the libraries to a temporary cache directory next to the true + * cache and atomically move it at the end of the operation. However as we do + * not have real guarantees that the filesystem move is atomic (although most of + * the time it should be if it is within a single filesystem), we will use + * locking to ensure consistency. + * * Obviously, every client that tries to install a library will acquire a write * lock for it, so that only one client is actually installing; but also every * client checking for the existence of the library will briefly acquire a read @@ -272,9 +299,7 @@ class DownloadingLibraryCache( * other process installed it in the meantime. This solution is efficient * because every library is locked independently and read locks can be acquired * by multiple clients at the same time, so the synchronization overhead for - * already installed libraries is negligible, and for libraries that need to be - * installed it is too negligible in comparison to the time required to download - * the libraries. + * already installed libraries is negligible. * * A single LockManager (and its locks directory) should be associated with at * most one library cache directory, as it makes sense for the distribution to diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala index 97717a0313b8..146656f35f96 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala @@ -6,17 +6,59 @@ import org.enso.distribution.FileSystem.PathSyntax import org.enso.downloader.http.{HTTPDownload, URIBuilder} import org.enso.editions.Editions.Repository import org.enso.editions.LibraryName +import org.enso.pkg.Package import org.enso.yaml.YamlHelper import java.nio.file.Path import scala.util.Failure +/** A class that manages the HTTP API of the Library Repository. + * + * @see docs/libraries/repositories.md#libraries-repository + */ object RepositoryHelper { + + /** Adds extension methods to the [[Repository]] type. */ + implicit class RepositoryMethods(val repository: Repository) { + + /** Creates a [[LibraryAccess]] instance that aids with downloading data of + * the given library. + */ + def accessLibrary(name: LibraryName, version: SemVer): LibraryAccess = + new LibraryAccess(name, version, resolveLibraryRoot(name, version)) + + /** Creates a [[URIBuilder]] that points to the directory in the repository + * corresponding to the given library. + */ + def resolveLibraryRoot(name: LibraryName, version: SemVer): URIBuilder = + URIBuilder + .fromUri(repository.url) + .addPathSegment(name.namespace) + .addPathSegment(name.name) + .addPathSegment(version.toString) + } + + /** A helper class that allows to access the Library Repository to query it + * for metadata of a specific library or download its packages. + * + * @param libraryName name of the library + * @param version version of the library + * @param libraryRoot a [[URIBuilder]] that points to the directory + * corresponding to the library + */ class LibraryAccess( libraryName: LibraryName, version: SemVer, libraryRoot: URIBuilder ) { + + /** Downloads and parses the manifest file. + * + * If the repository responds with 404 status code, it returns a special + * [[LibraryNotFoundException]] indicating that the repository does not + * provide that library. Any other failures are indicated with the more + * generic [[LibraryDownloadFailure]]. + */ def downloadManifest(): TaskProgress[LibraryManifest] = { val url = (libraryRoot / manifestFilename).build() HTTPDownload.fetchString(url).flatMap { response => @@ -27,14 +69,18 @@ object RepositoryHelper { Failure( LibraryNotFoundException(libraryName, version, url.toString) ) - case _ => + case code => Failure( - new LibraryDownloadFailure("Could not download the manifest") + new LibraryDownloadFailure( + s"Could not download the manifest: The repository responded " + + s"with $code status code." + ) ) } } } + /** A helper that downloads an artifact to a specific location. */ private def downloadArtifact( artifactName: String, destination: Path @@ -43,31 +89,31 @@ object RepositoryHelper { HTTPDownload.download(url, destination).map(_ => ()) } + /** Downloads the license file. + * + * It will fail with `ResourceNotFound` error if the license did not exist + * and with a more generic `HTTPException` if it failed for other reasons. + */ def downloadLicense(destinationDirectory: Path): TaskProgress[Unit] = downloadArtifact(licenseFilename, destinationDirectory / licenseFilename) + /** Downloads the package config file. */ def downloadPackageConfig(destinationDirectory: Path): TaskProgress[Unit] = downloadArtifact(packageFileName, destinationDirectory / packageFileName) + /** Downloads a sub-archive. */ def downloadArchive( archiveName: String, destinationDirectory: Path ): TaskProgress[Unit] = downloadArtifact(archiveName, destinationDirectory) } - implicit class RepositoryMethods(val repository: Repository) { - def resolveLibraryRoot(name: LibraryName, version: SemVer): URIBuilder = - URIBuilder - .fromUri(repository.url) - .addPathSegment(name.namespace) - .addPathSegment(name.name) - .addPathSegment(version.toString) + /** Name of the manifest file. */ + val manifestFilename = "manifest.yaml" - def accessLibrary(name: LibraryName, version: SemVer): LibraryAccess = - new LibraryAccess(name, version, resolveLibraryRoot(name, version)) - } + /** Name of the attached license file. */ + val licenseFilename = "LICENSE.md" - val manifestFilename = "manifest.yaml" - val licenseFilename = "LICENSE.md" - val packageFileName = "package.yaml" + /** Name of the package config file. */ + val packageFileName: String = Package.configFileName } From 2a822e1b017fe18d812b79a87031a76a07e9eb03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 13:26:30 +0200 Subject: [PATCH 21/30] note on cloud --- .../published/cache/DownloadingLibraryCache.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala index 9ff2b0a01724..75c04d088834 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -304,6 +304,13 @@ class DownloadingLibraryCache( * A single LockManager (and its locks directory) should be associated with at * most one library cache directory, as it makes sense for the distribution to * have only one cache, so the lock entries are not disambiguated in any way. + * + * Additional note: currently, in the cloud, the library cache is stored in the + * user's workspace, so only a single Language Server will be running at the + * same time; the locking mechanism is still needed to ensure library + * installations of the compiler and preinstalls by user's request do not + * conflict. The same lock manager can be used in the cloud environment, the + * lock files are stored in a local, transient directory of the server. */ /* Note [Temporary Directories for Installation] From adfec13a74a745414a534a8dc5e0ef77ab4cc60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 13:33:51 +0200 Subject: [PATCH 22/30] Update a debug info --- .../org/enso/librarymanager/DefaultLibraryProvider.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala index c2c8184b4d72..9761eba42827 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala @@ -50,8 +50,9 @@ class DefaultLibraryProvider( private val resolver = LibraryResolver(localLibraryProvider) + private val cacheRoot = distributionManager.paths.cachedLibraries private val primaryCache = new DownloadingLibraryCache( - cacheRoot = distributionManager.paths.cachedLibraries, + cacheRoot, TemporaryDirectoryManager(distributionManager, resourceManager), resourceManager, lockUserInterface, @@ -76,7 +77,7 @@ class DefaultLibraryProvider( s"Local library search paths = ${localLibrarySearchPaths.map(mask)}" ) logger.trace( - s"Primary library cache = Not implemented" + s"Primary library cache = ${mask(cacheRoot)}" ) logger.trace( s"Auxiliary (bundled) library caches = " + From 776296536a3e010a0ad6267df123d678e9021aa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 13:37:29 +0200 Subject: [PATCH 23/30] Pad tmp library prefix --- .../org/enso/distribution/TemporaryDirectoryManager.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala index 62b6300a266e..0eb97f1aa3e9 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala @@ -29,8 +29,9 @@ class TemporaryDirectoryManager( /** Creates a unique temporary subdirectory. */ def temporarySubdirectory(prefix: String = ""): Path = { + val paddedPrefix = if (prefix != "") prefix + "-" else prefix val path = - safeTemporaryDirectory.resolve(prefix + random.nextInt().toString) + safeTemporaryDirectory.resolve(paddedPrefix + random.nextInt().toString) if (Files.exists(path)) temporarySubdirectory(prefix) else { From a3fa8c97ff1535c1dd6d8efcb9e7fde3f687ef19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 13:41:19 +0200 Subject: [PATCH 24/30] Install dependencies of the simple library server on CI --- .github/workflows/scala.yml | 5 +++++ .../repository/DummyRepository.scala | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index b988473a2877..01d7ad50a4c4 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -86,6 +86,11 @@ jobs: uses: actions/setup-node@v1 with: node-version: ${{ env.nodeVersion }} + - name: Install Dependencies of the Simple Library Server + shell: bash + working-directory: tools/simple-library-server + run: | + npm install - name: Set Up SBT shell: bash run: | diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index c3b38725df5a..fb282bf9c831 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -103,6 +103,19 @@ abstract class DummyRepository { def startServer(port: Int, root: Path): WrappedProcess = { val serverDirectory = Path.of("tools/simple-library-server").toAbsolutePath.normalize + + val preinstallExitCode = (new ProcessBuilder()) + .command("npm", "install") + .directory(serverDirectory.toFile) + .inheritIO() + .start() + .waitFor() + + if (preinstallExitCode != 0) + throw new RuntimeException( + "Failed to preinstall the Library Repository Server dependencies." + ) + val command = Seq( "node", "main.js", @@ -118,7 +131,11 @@ abstract class DummyRepository { val process = new WrappedProcess(command, rawProcess) try { process.printIO() - process.waitForMessage("Serving the repository", 5, process.StdOut) + process.waitForMessage( + "Serving the repository", + timeoutSeconds = 15, + process.StdOut + ) } catch { case NonFatal(e) => process.kill() From 72e29723eac791b18837ebe4591f275c5005cab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 13:48:37 +0200 Subject: [PATCH 25/30] Fix the tmp dir name again --- .../org/enso/distribution/TemporaryDirectoryManager.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala index 0eb97f1aa3e9..f2936ccf6151 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala @@ -30,8 +30,9 @@ class TemporaryDirectoryManager( /** Creates a unique temporary subdirectory. */ def temporarySubdirectory(prefix: String = ""): Path = { val paddedPrefix = if (prefix != "") prefix + "-" else prefix + val randomSuffix = random.nextLong().toString.stripPrefix("-") val path = - safeTemporaryDirectory.resolve(paddedPrefix + random.nextInt().toString) + safeTemporaryDirectory.resolve(paddedPrefix + randomSuffix) if (Files.exists(path)) temporarySubdirectory(prefix) else { From 2ce43264a215bea60a023796b712b9a3887da5de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 15:03:29 +0200 Subject: [PATCH 26/30] Try fixing Windows CI --- .../org/enso/distribution/TemporaryDirectoryManager.scala | 2 +- .../published/repository/DummyRepository.scala | 8 ++++++-- .../components/RuntimeVersionManager.scala | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala index f2936ccf6151..bbdb823b415b 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala @@ -30,7 +30,7 @@ class TemporaryDirectoryManager( /** Creates a unique temporary subdirectory. */ def temporarySubdirectory(prefix: String = ""): Path = { val paddedPrefix = if (prefix != "") prefix + "-" else prefix - val randomSuffix = random.nextLong().toString.stripPrefix("-") + val randomSuffix = random.nextInt().toString.stripPrefix("-") val path = safeTemporaryDirectory.resolve(paddedPrefix + randomSuffix) if (Files.exists(path)) diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index fb282bf9c831..7370aaf3261c 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -1,6 +1,7 @@ package org.enso.librarymanager.published.repository import nl.gn0s1s.bump.SemVer +import org.enso.cli.OS import org.enso.distribution.FileSystem import org.enso.editions.Editions.RawEdition import org.enso.editions.{Editions, LibraryName} @@ -94,6 +95,9 @@ abstract class DummyRepository { ) } + private def npmCommand: String = if (OS.isWindows) "npm.cmd" else "npm" + private def nodeCommand: String = if (OS.isWindows) "node.exe" else "node" + /** Starts a server for the library repository. * * @param port port to listen on @@ -105,7 +109,7 @@ abstract class DummyRepository { Path.of("tools/simple-library-server").toAbsolutePath.normalize val preinstallExitCode = (new ProcessBuilder()) - .command("npm", "install") + .command(npmCommand, "install") .directory(serverDirectory.toFile) .inheritIO() .start() @@ -117,7 +121,7 @@ abstract class DummyRepository { ) val command = Seq( - "node", + nodeCommand, "main.js", "--port", port.toString, diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala index b1fbd3613d84..4b8fd6667310 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala @@ -850,7 +850,7 @@ class RuntimeVersionManager( private def safelyRemoveComponent(path: Path): Unit = { val temporaryPath = temporaryDirectoryManager.temporarySubdirectory(path.getFileName.toString) - FileSystem.atomicMove(path, temporaryPath) + FileSystem.atomicMove(path, temporaryPath / "tmp") FileSystem.removeDirectory(temporaryPath) } From c20d2c0747ee0c9fc05a638297dd0b244a18b746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 16:14:06 +0200 Subject: [PATCH 27/30] Try fixing Windows CI again --- .../published/repository/DummyRepository.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index 7370aaf3261c..7c94d75c8a83 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -95,6 +95,9 @@ abstract class DummyRepository { ) } + private def commandPrefix: Seq[String] = + if (OS.isWindows) Seq("cmd.exe", "/c") else Seq.empty + private def npmCommand: String = if (OS.isWindows) "npm.cmd" else "npm" private def nodeCommand: String = if (OS.isWindows) "node.exe" else "node" @@ -108,8 +111,9 @@ abstract class DummyRepository { val serverDirectory = Path.of("tools/simple-library-server").toAbsolutePath.normalize - val preinstallExitCode = (new ProcessBuilder()) - .command(npmCommand, "install") + val preinstallCommand = commandPrefix ++ Seq(npmCommand, "install") + val preinstallExitCode = new ProcessBuilder() + .command(preinstallCommand: _*) .directory(serverDirectory.toFile) .inheritIO() .start() @@ -120,7 +124,7 @@ abstract class DummyRepository { "Failed to preinstall the Library Repository Server dependencies." ) - val command = Seq( + val command = commandPrefix ++ Seq( nodeCommand, "main.js", "--port", From 6b859665434b4d41d4c03b4f14308f11f9648437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 17:16:20 +0200 Subject: [PATCH 28/30] Make test directory removal more robust --- build.sbt | 1 + .../websocket/json/BaseServerTest.scala | 4 +- .../org/enso/launcher/PluginManagerSpec.scala | 4 +- .../launcher/installation/InstallerSpec.scala | 2 +- .../installation/UninstallerSpec.scala | 2 +- .../enso/launcher/upgrade/UpgradeSpec.scala | 3 +- .../repository/LibraryDownloadTest.scala | 126 +++++++++--------- .../TestDistributionConfiguration.scala | 2 +- .../test/FakeEnvironment.scala | 1 + .../test/RuntimeVersionManagerTest.scala | 1 + .../locking/ConcurrencyTest.scala | 2 +- .../ThreadSafeFileLockManagerTest.scala | 7 +- .../GlobalConfigurationManagerSpec.scala | 6 +- .../DistributionManagerSpec.scala | 6 +- .../org/enso/testkit}/HasTestDirectory.scala | 2 +- .../testkit}/WithTemporaryDirectory.scala | 8 +- 16 files changed, 89 insertions(+), 88 deletions(-) rename lib/scala/{runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test => testkit/src/main/scala/org/enso/testkit}/HasTestDirectory.scala (78%) rename lib/scala/{runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test => testkit/src/main/scala/org/enso/testkit}/WithTemporaryDirectory.scala (97%) diff --git a/build.sbt b/build.sbt index 1c65bc6a5051..cd2de8b802e8 100644 --- a/build.sbt +++ b/build.sbt @@ -880,6 +880,7 @@ lazy val testkit = project .settings( libraryDependencies ++= Seq( "org.apache.commons" % "commons-lang3" % commonsLangVersion, + "commons-io" % "commons-io" % commonsIoVersion, "org.scalatest" %% "scalatest" % scalatestVersion ) ) diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala index b8fe0dcd021c..dee894374d1b 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala @@ -39,9 +39,9 @@ import org.enso.languageserver.text.BufferRegistry import org.enso.pkg.PackageManager import org.enso.polyglot.data.TypeGraph import org.enso.polyglot.runtime.Runtime.Api -import org.enso.runtimeversionmanager.test.{FakeEnvironment, HasTestDirectory} +import org.enso.runtimeversionmanager.test.FakeEnvironment import org.enso.searcher.sql.{SqlDatabase, SqlSuggestionsRepo, SqlVersionsRepo} -import org.enso.testkit.EitherValue +import org.enso.testkit.{EitherValue, HasTestDirectory} import org.enso.text.Sha3_224VersionCalculator import org.scalatest.OptionValues diff --git a/engine/launcher/src/test/scala/org/enso/launcher/PluginManagerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/PluginManagerSpec.scala index b7ed0f6bd5df..630fe3a53f04 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/PluginManagerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/PluginManagerSpec.scala @@ -1,8 +1,8 @@ package org.enso.launcher -import java.nio.file.{Files, Path} +import org.enso.testkit.WithTemporaryDirectory -import org.enso.runtimeversionmanager.test.WithTemporaryDirectory +import java.nio.file.{Files, Path} import org.scalatest.OptionValues import scala.jdk.CollectionConverters._ diff --git a/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala index 787bc746d7b2..998d06f0ab1d 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala @@ -5,8 +5,8 @@ import org.enso.distribution.FileSystem import java.nio.file.{Files, Path} import FileSystem.PathSyntax import org.enso.cli.OS -import org.enso.runtimeversionmanager.test.WithTemporaryDirectory import org.enso.launcher._ +import org.enso.testkit.WithTemporaryDirectory class InstallerSpec extends NativeTest with WithTemporaryDirectory { def portableRoot = getTestDirectory / "portable" diff --git a/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala index 1fc09a274485..a52ee905a367 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala @@ -5,8 +5,8 @@ import org.enso.distribution.FileSystem import java.nio.file.{Files, Path} import FileSystem.PathSyntax import org.enso.cli.OS -import org.enso.runtimeversionmanager.test.WithTemporaryDirectory import org.enso.launcher.NativeTest +import org.enso.testkit.WithTemporaryDirectory class UninstallerSpec extends NativeTest with WithTemporaryDirectory { def installedRoot: Path = getTestDirectory / "installed" diff --git a/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala index 34b3baabd3af..49d459ef91c3 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala @@ -8,8 +8,7 @@ import org.enso.distribution.locking.{FileLockManager, LockType} import FileSystem.PathSyntax import org.enso.cli.OS import org.enso.launcher._ -import org.enso.runtimeversionmanager.test.WithTemporaryDirectory -import org.enso.testkit.RetrySpec +import org.enso.testkit.{RetrySpec, WithTemporaryDirectory} import org.enso.testkit.process.{RunResult, WrappedProcess} import org.scalatest.exceptions.TestFailedException import org.scalatest.{BeforeAndAfterAll, OptionValues} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala index 930d1fce47bc..5d66e0126f20 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -1,92 +1,98 @@ package org.enso.librarymanager.published.repository import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.distribution.TemporaryDirectoryManager import org.enso.distribution.locking.{ LockUserInterface, Resource, ResourceManager, ThreadSafeFileLockManager } -import org.enso.distribution.{FileSystem, TemporaryDirectoryManager} import org.enso.editions.Editions import org.enso.librarymanager.published.cache.DownloadingLibraryCache -import org.enso.loggingservice.{LogLevel, TestLogger} import org.enso.loggingservice.TestLogger.TestLogMessage +import org.enso.loggingservice.{LogLevel, TestLogger} import org.enso.pkg.PackageManager +import org.enso.testkit.WithTemporaryDirectory import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import java.nio.file.Files -class LibraryDownloadTest extends AnyWordSpec with Matchers { +class LibraryDownloadTest + extends AnyWordSpec + with Matchers + with WithTemporaryDirectory { val port: Int = 47306 "DownloadingLibraryCache" should { "be able to download and install libraries from a repository" in { val repo = new ExampleRepository - FileSystem.withTemporaryDirectory("enso-test-lib") { tmp => - val repoRoot = tmp.resolve("repo") - repo.createRepository(repoRoot) - val lockManager = new ThreadSafeFileLockManager(tmp.resolve("locks")) - val resourceManager = new ResourceManager(lockManager) - val cache = new DownloadingLibraryCache( - cacheRoot = tmp.resolve("cache"), - temporaryDirectoryManager = - new TemporaryDirectoryManager(tmp.resolve("tmp"), resourceManager), - resourceManager = resourceManager, - lockUserInterface = new LockUserInterface { - override def startWaitingForResource(resource: Resource): Unit = - println(s"Waiting for ${resource.name}") - override def finishWaitingForResource(resource: Resource): Unit = - println(s"${resource.name} is ready") - }, - progressReporter = new ProgressReporter { - override def trackProgress( - message: String, - task: TaskProgress[_] - ): Unit = {} - } - ) + val repoRoot = getTestDirectory.resolve("repo") + repo.createRepository(repoRoot) + val lockManager = + new ThreadSafeFileLockManager(getTestDirectory.resolve("locks")) + val resourceManager = new ResourceManager(lockManager) + val cache = new DownloadingLibraryCache( + cacheRoot = getTestDirectory.resolve("cache"), + temporaryDirectoryManager = new TemporaryDirectoryManager( + getTestDirectory.resolve("tmp"), + resourceManager + ), + resourceManager = resourceManager, + lockUserInterface = new LockUserInterface { + override def startWaitingForResource(resource: Resource): Unit = + println(s"Waiting for ${resource.name}") - val server = repo.startServer(port, repoRoot) - try { - cache.findCachedLibrary( - repo.testLib.libraryName, - repo.testLib.version - ) shouldBe empty + override def finishWaitingForResource(resource: Resource): Unit = + println(s"${resource.name} is ready") + }, + progressReporter = new ProgressReporter { + override def trackProgress( + message: String, + task: TaskProgress[_] + ): Unit = {} + } + ) - val (libPath, logs) = TestLogger.gatherLogs { - cache - .findOrInstallLibrary( - repo.testLib.libraryName, - repo.testLib.version, - Editions - .Repository("test_repo", s"http://localhost:$port/libraries") - ) - .get - } - val pkg = PackageManager.Default.loadPackage(libPath.toFile).get - pkg.name shouldEqual "Bar" - val sources = pkg.listSources - sources should have size 1 - sources.head.file.getName shouldEqual "Main.enso" - assert( - Files.notExists(libPath.resolve("LICENSE.md")), - "The license file should not exist as it was not provided " + - "in the repository." - ) - logs should contain( - TestLogMessage( - LogLevel.Warning, - "License file for library [Foo.Bar:1.0.0] was missing." + val server = repo.startServer(port, repoRoot) + try { + cache.findCachedLibrary( + repo.testLib.libraryName, + repo.testLib.version + ) shouldBe empty + + val (libPath, logs) = TestLogger.gatherLogs { + cache + .findOrInstallLibrary( + repo.testLib.libraryName, + repo.testLib.version, + Editions + .Repository("test_repo", s"http://localhost:$port/libraries") ) - ) - } finally { - server.kill() - server.join() + .get } + val pkg = PackageManager.Default.loadPackage(libPath.toFile).get + pkg.name shouldEqual "Bar" + val sources = pkg.listSources + sources should have size 1 + sources.head.file.getName shouldEqual "Main.enso" + assert( + Files.notExists(libPath.resolve("LICENSE.md")), + "The license file should not exist as it was not provided " + + "in the repository." + ) + logs should contain( + TestLogMessage( + LogLevel.Warning, + "License file for library [Foo.Bar:1.0.0] was missing." + ) + ) + } finally { + server.kill() + server.join() } } } diff --git a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala index da5069f3d275..bbb021c956d0 100644 --- a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala @@ -31,10 +31,10 @@ import org.enso.runtimeversionmanager.releases.{ import org.enso.runtimeversionmanager.runner.{JVMSettings, JavaCommand} import org.enso.runtimeversionmanager.test.{ FakeEnvironment, - HasTestDirectory, NoopComponentUpdaterFactory, TestLocalLockManager } +import org.enso.testkit.HasTestDirectory import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/FakeEnvironment.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/FakeEnvironment.scala index f410ca17ab89..1525e03c3758 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/FakeEnvironment.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/FakeEnvironment.scala @@ -1,6 +1,7 @@ package org.enso.runtimeversionmanager.test import org.enso.distribution.{Environment, FileSystem} +import org.enso.testkit.HasTestDirectory import java.nio.file.{Files, Path} diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala index 8f5efcaa1483..25a8214fa5f6 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala @@ -16,6 +16,7 @@ import org.enso.runtimeversionmanager.components.{ } import org.enso.runtimeversionmanager.releases.engine.EngineReleaseProvider import org.enso.runtimeversionmanager.releases.graalvm.GraalVMRuntimeReleaseProvider +import org.enso.testkit.WithTemporaryDirectory import org.scalatest.OptionValues import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala index b94c2fda0e46..65d19ecf1876 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala @@ -30,7 +30,7 @@ import org.enso.runtimeversionmanager.releases.engine.{ import org.enso.runtimeversionmanager.releases.graalvm.GraalCEReleaseProvider import org.enso.runtimeversionmanager.releases.testing.FakeReleaseProvider import org.enso.runtimeversionmanager.test._ -import org.enso.testkit.{FlakySpec, RetrySpec} +import org.enso.testkit.{FlakySpec, RetrySpec, WithTemporaryDirectory} import org.scalatest.BeforeAndAfterEach import org.scalatest.concurrent.TimeLimitedTests import org.scalatest.matchers.should.Matchers diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ThreadSafeFileLockManagerTest.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ThreadSafeFileLockManagerTest.scala index f525c472a03a..9703ecda60f9 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ThreadSafeFileLockManagerTest.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ThreadSafeFileLockManagerTest.scala @@ -8,11 +8,8 @@ import org.enso.distribution.locking.{ } import java.nio.file.Path -import org.enso.runtimeversionmanager.test.{ - NativeTestHelper, - TestSynchronizer, - WithTemporaryDirectory -} +import org.enso.runtimeversionmanager.test.{NativeTestHelper, TestSynchronizer} +import org.enso.testkit.WithTemporaryDirectory import org.scalatest.OptionValues import org.scalatest.concurrent.TimeLimitedTests import org.scalatest.matchers.should.Matchers diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/config/GlobalConfigurationManagerSpec.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/config/GlobalConfigurationManagerSpec.scala index 6c20131af97d..5389a74e4337 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/config/GlobalConfigurationManagerSpec.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/config/GlobalConfigurationManagerSpec.scala @@ -3,10 +3,8 @@ package org.enso.runtimeversionmanager.config import io.circe.Json import nl.gn0s1s.bump.SemVer import org.enso.distribution.DistributionManager -import org.enso.runtimeversionmanager.test.{ - FakeEnvironment, - WithTemporaryDirectory -} +import org.enso.runtimeversionmanager.test.FakeEnvironment +import org.enso.testkit.WithTemporaryDirectory import org.scalatest.OptionValues import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala index 420bc999ea26..e219d05a27a1 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala @@ -9,10 +9,8 @@ import org.enso.distribution.{ import java.nio.file.{Files, Path} import org.enso.distribution.FileSystem.PathSyntax -import org.enso.runtimeversionmanager.test.{ - FakeEnvironment, - WithTemporaryDirectory -} +import org.enso.runtimeversionmanager.test.FakeEnvironment +import org.enso.testkit.WithTemporaryDirectory import org.scalatest.OptionValues import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/HasTestDirectory.scala b/lib/scala/testkit/src/main/scala/org/enso/testkit/HasTestDirectory.scala similarity index 78% rename from lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/HasTestDirectory.scala rename to lib/scala/testkit/src/main/scala/org/enso/testkit/HasTestDirectory.scala index 44f999d093b7..8d72c7fd7d59 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/HasTestDirectory.scala +++ b/lib/scala/testkit/src/main/scala/org/enso/testkit/HasTestDirectory.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.test +package org.enso.testkit import java.nio.file.Path diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/WithTemporaryDirectory.scala b/lib/scala/testkit/src/main/scala/org/enso/testkit/WithTemporaryDirectory.scala similarity index 97% rename from lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/WithTemporaryDirectory.scala rename to lib/scala/testkit/src/main/scala/org/enso/testkit/WithTemporaryDirectory.scala index c4cae32d3608..0b5406502553 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/WithTemporaryDirectory.scala +++ b/lib/scala/testkit/src/main/scala/org/enso/testkit/WithTemporaryDirectory.scala @@ -1,11 +1,11 @@ -package org.enso.runtimeversionmanager.test - -import java.io.{File, IOException} -import java.nio.file.{Files, Path} +package org.enso.testkit import org.apache.commons.io.FileUtils import org.scalatest.{BeforeAndAfterEach, Suite} +import java.io.{File, IOException} +import java.nio.file.{Files, Path} + /** Creates a separate temporary directory for each test. */ trait WithTemporaryDirectory From d5c891dc76c1c41d714419700bf5a7d70bdccfcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 19:08:41 +0200 Subject: [PATCH 29/30] Make sure to kill server descendants --- .../published/repository/DummyRepository.scala | 3 ++- .../published/repository/LibraryDownloadTest.scala | 4 ++-- .../org/enso/testkit/process/WrappedProcess.scala | 12 ++++++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index 7c94d75c8a83..3c163855ec8a 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -121,7 +121,8 @@ abstract class DummyRepository { if (preinstallExitCode != 0) throw new RuntimeException( - "Failed to preinstall the Library Repository Server dependencies." + s"Failed to preinstall the Library Repository Server dependencies: " + + s"npm exited with code $preinstallCommand." ) val command = commandPrefix ++ Seq( diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala index 5d66e0126f20..b2095a11a24a 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -91,8 +91,8 @@ class LibraryDownloadTest ) ) } finally { - server.kill() - server.join() + server.kill(killDescendants = true) + server.join(waitForDescendants = true) } } } diff --git a/lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala b/lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala index f35d65496ce9..4c83475b54c4 100644 --- a/lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala +++ b/lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala @@ -130,10 +130,18 @@ class WrappedProcess(command: Seq[String], process: Process) { def isAlive: Boolean = process.isAlive /** Tries to kill the process immediately. */ - def kill(): Unit = { + def kill(killDescendants: Boolean = false): Unit = { process.destroyForcibly() + if (killDescendants) { + for (processHandle <- findDescendants()) { + processHandle.destroyForcibly() + } + } } + private def findDescendants(): Seq[ProcessHandle] = + process.descendants().toScala(Factory.arrayFactory).toSeq + /** Waits for the process to finish and returns its [[RunResult]]. * * If `waitForDescendants` is set, tries to wait for descendants of the @@ -154,7 +162,7 @@ class WrappedProcess(command: Seq[String], process: Process) { process.exitValue() else throw new TimeoutException("Process timed out") if (waitForDescendants) { - descendants = process.descendants().toScala(Factory.arrayFactory).toSeq + descendants = findDescendants() descendants.foreach(_.onExit().get(timeoutSeconds, TimeUnit.SECONDS)) } errThread.join(1000) From ce6b12f3aaed1cadc01adc536b101c7b056070ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Jul 2021 20:50:57 +0200 Subject: [PATCH 30/30] Release locks to ensure temporary directory may be cleared --- .../repository/LibraryDownloadTest.scala | 109 +++++++++--------- 1 file changed, 57 insertions(+), 52 deletions(-) diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala index b2095a11a24a..882516116dab 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -35,64 +35,69 @@ class LibraryDownloadTest val lockManager = new ThreadSafeFileLockManager(getTestDirectory.resolve("locks")) val resourceManager = new ResourceManager(lockManager) - val cache = new DownloadingLibraryCache( - cacheRoot = getTestDirectory.resolve("cache"), - temporaryDirectoryManager = new TemporaryDirectoryManager( - getTestDirectory.resolve("tmp"), - resourceManager - ), - resourceManager = resourceManager, - lockUserInterface = new LockUserInterface { - override def startWaitingForResource(resource: Resource): Unit = - println(s"Waiting for ${resource.name}") + try { + val cache = new DownloadingLibraryCache( + cacheRoot = getTestDirectory.resolve("cache"), + temporaryDirectoryManager = new TemporaryDirectoryManager( + getTestDirectory.resolve("tmp"), + resourceManager + ), + resourceManager = resourceManager, + lockUserInterface = new LockUserInterface { + override def startWaitingForResource(resource: Resource): Unit = + println(s"Waiting for ${resource.name}") - override def finishWaitingForResource(resource: Resource): Unit = - println(s"${resource.name} is ready") - }, - progressReporter = new ProgressReporter { - override def trackProgress( - message: String, - task: TaskProgress[_] - ): Unit = {} - } - ) + override def finishWaitingForResource(resource: Resource): Unit = + println(s"${resource.name} is ready") + }, + progressReporter = new ProgressReporter { + override def trackProgress( + message: String, + task: TaskProgress[_] + ): Unit = {} + } + ) - val server = repo.startServer(port, repoRoot) - try { - cache.findCachedLibrary( - repo.testLib.libraryName, - repo.testLib.version - ) shouldBe empty + val server = repo.startServer(port, repoRoot) + try { + cache.findCachedLibrary( + repo.testLib.libraryName, + repo.testLib.version + ) shouldBe empty - val (libPath, logs) = TestLogger.gatherLogs { - cache - .findOrInstallLibrary( - repo.testLib.libraryName, - repo.testLib.version, - Editions - .Repository("test_repo", s"http://localhost:$port/libraries") + val (libPath, logs) = TestLogger.gatherLogs { + cache + .findOrInstallLibrary( + repo.testLib.libraryName, + repo.testLib.version, + Editions + .Repository("test_repo", s"http://localhost:$port/libraries") + ) + .get + } + val pkg = PackageManager.Default.loadPackage(libPath.toFile).get + pkg.name shouldEqual "Bar" + val sources = pkg.listSources + sources should have size 1 + sources.head.file.getName shouldEqual "Main.enso" + assert( + Files.notExists(libPath.resolve("LICENSE.md")), + "The license file should not exist as it was not provided " + + "in the repository." + ) + logs should contain( + TestLogMessage( + LogLevel.Warning, + "License file for library [Foo.Bar:1.0.0] was missing." ) - .get - } - val pkg = PackageManager.Default.loadPackage(libPath.toFile).get - pkg.name shouldEqual "Bar" - val sources = pkg.listSources - sources should have size 1 - sources.head.file.getName shouldEqual "Main.enso" - assert( - Files.notExists(libPath.resolve("LICENSE.md")), - "The license file should not exist as it was not provided " + - "in the repository." - ) - logs should contain( - TestLogMessage( - LogLevel.Warning, - "License file for library [Foo.Bar:1.0.0] was missing." ) - ) + } finally { + server.kill(killDescendants = true) + server.join(waitForDescendants = true) + } } finally { - server.kill(killDescendants = true) - server.join(waitForDescendants = true) + resourceManager.releaseMainLock() + resourceManager.unlockTemporaryDirectory() } } }