From 322a9f611aee3e3172aba05b8c4a0afa74b0aa77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 20 Jul 2021 12:30:41 +0200 Subject: [PATCH] Implement a basic download test --- .../distribution/DefaultManagers.scala | 2 +- .../TemporaryDirectoryManager.scala | 30 +++++--- .../cache/DownloadingLibraryCache.scala | 5 +- .../published/repository/ArchiveWriter.scala | 39 ++++++++++ .../repository/DummyRepository.scala | 23 ++++-- .../repository/LibraryDownloadTest.scala | 72 ++++++++++++++----- .../DefaultDistributionConfiguration.scala | 2 +- .../TestDistributionConfiguration.scala | 2 +- .../test/RuntimeVersionManagerTest.scala | 2 +- .../locking/ConcurrencyTest.scala | 2 +- 10 files changed, 134 insertions(+), 45 deletions(-) create mode 100644 lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala diff --git a/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala b/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala index 0c80405f96c5d..c9516d414bc0a 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala @@ -44,7 +44,7 @@ object DefaultManagers { /** Default [[TemporaryDirectoryManager]]. */ lazy val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, defaultResourceManager) + TemporaryDirectoryManager(distributionManager, defaultResourceManager) /** Default [[RuntimeComponentConfiguration]]. */ lazy val componentConfig: RuntimeComponentConfiguration = diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala index 48ce8612b2628..6b18bcac0af2d 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala @@ -21,7 +21,7 @@ import scala.util.Random * so that they can be moved all at once at the last step of the installation. */ class TemporaryDirectoryManager( - distribution: DistributionManager, + unsafeRoot: Path, resourceManager: ResourceManager ) { private val logger = Logger[TemporaryDirectoryManager] @@ -54,11 +54,10 @@ class TemporaryDirectoryManager( * should ensure that). If that fails, it is also cleaned before any future * accesses. */ - private lazy val safeTemporaryDirectory = { + private lazy val safeTemporaryDirectory: Path = { resourceManager.startUsingTemporaryDirectory() - val path = distribution.paths.unsafeTemporaryDirectory - Files.createDirectories(path) - path + Files.createDirectories(unsafeRoot) + unsafeRoot } /** Tries to clean the temporary files directory. @@ -70,18 +69,27 @@ class TemporaryDirectoryManager( * marked as in-use and will not be cleaned. */ def tryCleaningTemporaryDirectory(): Unit = { - val tmp = distribution.paths.unsafeTemporaryDirectory - if (Files.exists(tmp)) { + if (Files.exists(unsafeRoot)) { resourceManager.tryWithExclusiveTemporaryDirectory { - if (!FileSystem.isDirectoryEmpty(tmp)) { + if (!FileSystem.isDirectoryEmpty(unsafeRoot)) { logger.info( "Cleaning up temporary files from a previous installation." ) } - FileSystem.removeDirectory(tmp) - Files.createDirectories(tmp) - FileSystem.removeEmptyDirectoryOnExit(tmp) + FileSystem.removeDirectory(unsafeRoot) + Files.createDirectories(unsafeRoot) + FileSystem.removeEmptyDirectoryOnExit(unsafeRoot) } } } } + +object TemporaryDirectoryManager { + def apply( + distribution: DistributionManager, + resourceManager: ResourceManager + ): TemporaryDirectoryManager = new TemporaryDirectoryManager( + distribution.paths.unsafeTemporaryDirectory, + resourceManager + ) +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala index 1f460b40c5396..2d15d98781fe4 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -188,10 +188,7 @@ class DownloadingLibraryCache( if (shouldDownloadArchive(archiveName)) { val tmpArchivePath = globalTmpDir / archiveName - val download = access.downloadArchive( - archiveName, - tmpArchivePath / archiveName - ) + val download = access.downloadArchive(archiveName, tmpArchivePath) progressReporter.trackProgress( s"Downloading [$archiveName] of [$libraryName].", download diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala new file mode 100644 index 0000000000000..1fde03cde0314 --- /dev/null +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala @@ -0,0 +1,39 @@ +package org.enso.librarymanager.published.repository + +import org.apache.commons.compress.archivers.tar.{ + TarArchiveEntry, + TarArchiveOutputStream +} +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream + +import java.io.{BufferedOutputStream, FileOutputStream} +import java.nio.file.Path +import scala.util.Using + +object ArchiveWriter { + sealed trait FileToWrite { + def relativePath: String + } + + case class TextFile(relativePath: String, content: String) extends FileToWrite + + def writeTarArchive(path: Path, files: Seq[FileToWrite]): Unit = { + Using(new FileOutputStream(path.toFile)) { outputStream => + Using(new BufferedOutputStream(outputStream)) { bufferedStream => + Using(new GzipCompressorOutputStream(bufferedStream)) { gzipStream => + Using(new TarArchiveOutputStream(gzipStream)) { archive => + for (file <- files) { + file match { + case TextFile(relativePath, content) => + val entry = new TarArchiveEntry(relativePath) + archive.putArchiveEntry(entry) + archive.write(content.getBytes) + archive.closeArchiveEntry() + } + } + } + } + } + } + } +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index fca783321e2ca..f83a292b84e25 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -29,11 +29,12 @@ abstract class DummyRepository { .resolve(lib.libraryName.name) .resolve(lib.version.toString) Files.createDirectories(libraryRoot) - val pkg = createLibraryProject(libraryRoot, lib) - FileSystem.writeTextFile( - pkg.sourceDir.toPath.resolve("Main.enso"), - lib.mainContent + createLibraryProject(libraryRoot, lib) + val files = Seq( + ArchiveWriter.TextFile("src/Main.enso", lib.mainContent) ) + ArchiveWriter.writeTarArchive(libraryRoot.resolve("main.tgz"), files) + createManifest(libraryRoot) } } @@ -51,6 +52,15 @@ abstract class DummyRepository { pkg } + private def createManifest(path: Path): Unit = { + FileSystem.writeTextFile( + path.resolve("manifest.yaml"), + s"""archives: + | - main.tgz + |""".stripMargin + ) + } + def createEdition(repoUrl: String): RawEdition = { Editions.Raw.Edition( parent = Some(buildinfo.Info.currentEdition), @@ -62,8 +72,9 @@ abstract class DummyRepository { ) } - def runServer(port: Int, root: Path): Process = { - val serverDirectory = Path.of("../../../tools/simple-library-server") + def startServer(port: Int, root: Path): Process = { + val serverDirectory = + Path.of("tools/simple-library-server").toAbsolutePath.normalize (new ProcessBuilder) .command( "node", diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala index 0463166acb977..d1200d0cfcd5a 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -1,8 +1,16 @@ package org.enso.librarymanager.published.repository -import org.enso.distribution.FileSystem +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.distribution.locking.{ + LockUserInterface, + Resource, + ResourceManager, + ThreadSafeFileLockManager +} +import org.enso.distribution.{FileSystem, TemporaryDirectoryManager} import org.enso.editions.Editions import org.enso.librarymanager.published.cache.DownloadingLibraryCache +import org.enso.pkg.PackageManager import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -13,29 +21,55 @@ class LibraryDownloadTest extends AnyWordSpec with Matchers { "DownloadingLibraryCache" should { "be able to download and install libraries from a repository" in { val repo = new ExampleRepository - FileSystem.withTemporaryDirectory("enso-test-repo") { repoRoot => + FileSystem.withTemporaryDirectory("enso-test-lib") { tmp => + val repoRoot = tmp.resolve("repo") repo.createRepository(repoRoot) - val server = repo.runServer(port, repoRoot) - - // TODO - val cache = new DownloadingLibraryCache(???, ???, ???, ???, ???) + val lockManager = new ThreadSafeFileLockManager(tmp.resolve("locks")) + val resourceManager = new ResourceManager(lockManager) + val cache = new DownloadingLibraryCache( + cacheRoot = tmp.resolve("cache"), + temporaryDirectoryManager = + new TemporaryDirectoryManager(tmp.resolve("tmp"), resourceManager), + resourceManager = resourceManager, + lockUserInterface = new LockUserInterface { + override def startWaitingForResource(resource: Resource): Unit = + println(s"Waiting for ${resource.name}") - cache.findCachedLibrary( - repo.testLib.libraryName, - repo.testLib.version - ) shouldBe empty + override def finishWaitingForResource(resource: Resource): Unit = + println(s"${resource.name} is ready") + }, + progressReporter = new ProgressReporter { + override def trackProgress( + message: String, + task: TaskProgress[_] + ): Unit = {} + } + ) - cache - .findOrInstallLibrary( + val server = repo.startServer(port, repoRoot) + try { + cache.findCachedLibrary( repo.testLib.libraryName, - repo.testLib.version, - Editions - .Repository("test_repo", s"http://localhost:$port/libraries") - ) - .get + repo.testLib.version + ) shouldBe empty - server.destroy() - server.waitFor() + val libPath = cache + .findOrInstallLibrary( + repo.testLib.libraryName, + repo.testLib.version, + Editions + .Repository("test_repo", s"http://localhost:$port/libraries") + ) + .get + val pkg = PackageManager.Default.loadPackage(libPath.toFile).get + pkg.name shouldEqual "Bar" + val sources = pkg.listSources + sources should have size 1 + sources.head.file.getName shouldEqual "Main.enso" + } finally { + server.destroy() + server.waitFor() + } } } } diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala index abfc3dcaca19b..bc1a52f563913 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala @@ -57,7 +57,7 @@ object DefaultDistributionConfiguration /** @inheritdoc */ lazy val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) lazy val componentConfiguration: RuntimeComponentConfiguration = new GraalVMComponentConfiguration diff --git a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala index 7df57cf139f0b..da5069f3d2750 100644 --- a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala @@ -70,7 +70,7 @@ class TestDistributionConfiguration( lazy val editionManager: EditionManager = EditionManager(distributionManager) lazy val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) lazy val componentConfig = new GraalVMComponentConfiguration diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala index c4162d655a66d..8f5efcaa14837 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala @@ -53,7 +53,7 @@ class RuntimeVersionManagerTest val resourceManager = TestLocalResourceManager.create() val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) val componentConfig = new GraalVMComponentConfiguration val runtimeVersionManager = new RuntimeVersionManager( diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala index 0335f3524c904..b94c2fda0e464 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala @@ -152,7 +152,7 @@ class ConcurrencyTest } val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) val componentConfig = new GraalVMComponentConfiguration val componentsManager = new RuntimeVersionManager( TestRuntimeVersionManagementUserInterface.default,