diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index 5a2cc23a41ff..01d7ad50a4c4 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -15,6 +15,8 @@ env: sbtVersion: 1.5.2 # Please ensure that this is in sync with rustVersion in build.sbt rustToolchain: nightly-2021-05-12 + # Some moderately recent version of Node.JS is needed to run the library download tests. + nodeVersion: 12.18.4 jobs: test_and_publish: @@ -80,6 +82,15 @@ jobs: graalvm-version: ${{ env.graalVersion }} java-version: ${{ env.javaVersion }} native-image: true + - name: Install Node + uses: actions/setup-node@v1 + with: + node-version: ${{ env.nodeVersion }} + - name: Install Dependencies of the Simple Library Server + shell: bash + working-directory: tools/simple-library-server + run: | + npm install - name: Set Up SBT shell: bash run: | diff --git a/RELEASES.md b/RELEASES.md index 314c67a79d93..136dbe9c632f 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,5 +1,11 @@ # Enso Next +## Tooling + +- Implement a basic library downloader + ([#1885](https://github.com/enso-org/enso/pull/1885)), allowing to download + missing libraries. + ## Libraries - Added support for reading XLS and XLSX spreadsheets diff --git a/build.sbt b/build.sbt index 2a1ba226b92d..cd2de8b802e8 100644 --- a/build.sbt +++ b/build.sbt @@ -236,16 +236,20 @@ lazy val enso = (project in file(".")) `task-progress-notifications`, `logging-utils`, `logging-service`, + `logging-truffle-connector`, + `locking-test-helper`, `akka-native`, `version-output`, `engine-runner`, runtime, searcher, launcher, + downloader, `runtime-version-manager`, `runtime-version-manager-test`, editions, `distribution-manager`, + `edition-updater`, `library-manager`, syntax.jvm, testkit @@ -710,9 +714,10 @@ lazy val cli = project .configs(Test) .settings( version := "0.1", - libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % scalatestVersion % Test, - "org.typelevel" %% "cats-core" % catsVersion + libraryDependencies ++= circe ++ Seq( + "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + "org.typelevel" %% "cats-core" % catsVersion ), Test / parallelExecution := false ) @@ -875,6 +880,7 @@ lazy val testkit = project .settings( libraryDependencies ++= Seq( "org.apache.commons" % "commons-lang3" % commonsLangVersion, + "commons-io" % "commons-io" % commonsIoVersion, "org.scalatest" %% "scalatest" % scalatestVersion ) ) @@ -1337,6 +1343,7 @@ lazy val `distribution-manager` = project ) ) .dependsOn(editions) + .dependsOn(cli) .dependsOn(pkg) .dependsOn(`logging-utils`) @@ -1354,11 +1361,38 @@ lazy val editions = project ) .dependsOn(testkit % Test) +lazy val downloader = (project in file("lib/scala/downloader")) + .settings( + version := "0.1", + libraryDependencies ++= circe ++ Seq( + "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, + "commons-io" % "commons-io" % commonsIoVersion, + "org.apache.commons" % "commons-compress" % commonsCompressVersion, + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + akkaActor, + akkaStream, + akkaHttp, + akkaSLF4J + ) + ) + .dependsOn(cli) + +lazy val `edition-updater` = project + .in(file("lib/scala/edition-updater")) + .configs(Test) + .settings( + libraryDependencies ++= Seq( + "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, + "org.scalatest" %% "scalatest" % scalatestVersion % Test + ) + ) + .dependsOn(editions) + .dependsOn(downloader) + lazy val `library-manager` = project .in(file("lib/scala/library-manager")) .configs(Test) .settings( - resolvers += Resolver.bintrayRepo("gn0s1s", "releases"), libraryDependencies ++= Seq( "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion, "org.scalatest" %% "scalatest" % scalatestVersion % Test @@ -1368,7 +1402,9 @@ lazy val `library-manager` = project .dependsOn(editions) .dependsOn(cli) .dependsOn(`distribution-manager`) + .dependsOn(downloader) .dependsOn(testkit % Test) + .dependsOn(`logging-service` % Test) lazy val `runtime-version-manager` = project .in(file("lib/scala/runtime-version-manager")) @@ -1385,6 +1421,7 @@ lazy val `runtime-version-manager` = project ) ) .dependsOn(pkg) + .dependsOn(downloader) .dependsOn(`logging-service`) .dependsOn(cli) .dependsOn(`version-output`) diff --git a/distribution/engine/THIRD-PARTY/NOTICE b/distribution/engine/THIRD-PARTY/NOTICE index 2f5e6caf8ed9..80eafe959ff1 100644 --- a/distribution/engine/THIRD-PARTY/NOTICE +++ b/distribution/engine/THIRD-PARTY/NOTICE @@ -11,6 +11,11 @@ The license information can be found along with the copyright notices. Copyright notices related to this dependency can be found in the directory `com.lihaoyi.sourcecode_2.13-0.2.1`. +'commons-compress', licensed under the Apache License, Version 2.0, is distributed with the engine. +The license file can be found at `licenses/APACHE2.0`. +Copyright notices related to this dependency can be found in the directory `org.apache.commons.commons-compress-1.20`. + + 'checker-qual', licensed under the The MIT License, is distributed with the engine. The license information can be found along with the copyright notices. Copyright notices related to this dependency can be found in the directory `org.checkerframework.checker-qual-2.11.1`. @@ -96,11 +101,6 @@ The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `com.fasterxml.jackson.core.jackson-core-2.11.1`. -'config', licensed under the Apache License, Version 2.0, is distributed with the engine. -The license file can be found at `licenses/APACHE2.0`. -Copyright notices related to this dependency can be found in the directory `com.typesafe.config-1.3.2`. - - 'zio_2.13', licensed under the Apache-2.0, is distributed with the engine. The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `dev.zio.zio_2.13-1.0.1`. @@ -161,11 +161,6 @@ The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `org.scala-lang.scala-compiler-2.13.6`. -'reactive-streams', licensed under the CC0, is distributed with the engine. -The license file can be found at `licenses/CC0`. -Copyright notices related to this dependency can be found in the directory `org.reactivestreams.reactive-streams-1.0.2`. - - 'config', licensed under the Apache-2.0, is distributed with the engine. The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `com.typesafe.config-1.4.1`. @@ -206,6 +201,11 @@ The license information can be found along with the copyright notices. Copyright notices related to this dependency can be found in the directory `org.typelevel.jawn-parser_2.13-1.0.0`. +'config', licensed under the Apache-2.0, is distributed with the engine. +The license file can be found at `licenses/APACHE2.0`. +Copyright notices related to this dependency can be found in the directory `com.typesafe.config-1.4.0`. + + 'circe-literal_2.13', licensed under the Apache 2.0, is distributed with the engine. The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `io.circe.circe-literal_2.13-0.14.0-M1`. diff --git a/distribution/engine/THIRD-PARTY/com.google.auto.service.auto-service-1.0-rc7/NOTICES b/distribution/engine/THIRD-PARTY/com.google.auto.service.auto-service-1.0-rc7/NOTICES index 3b5e809ce4c8..3db01d150f9f 100644 --- a/distribution/engine/THIRD-PARTY/com.google.auto.service.auto-service-1.0-rc7/NOTICES +++ b/distribution/engine/THIRD-PARTY/com.google.auto.service.auto-service-1.0-rc7/NOTICES @@ -1,3 +1,17 @@ +/* + * Copyright 2013 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + /* * Copyright 2008 Google LLC * @@ -14,20 +28,6 @@ * limitations under the License. */ -/* - * Copyright 2013 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - /* * Copyright 2008 Google LLC * diff --git a/distribution/engine/THIRD-PARTY/com.google.errorprone.error_prone_annotations-2.3.4/NOTICES b/distribution/engine/THIRD-PARTY/com.google.errorprone.error_prone_annotations-2.3.4/NOTICES index da6f3cd143cb..d345d4fd67c2 100644 --- a/distribution/engine/THIRD-PARTY/com.google.errorprone.error_prone_annotations-2.3.4/NOTICES +++ b/distribution/engine/THIRD-PARTY/com.google.errorprone.error_prone_annotations-2.3.4/NOTICES @@ -31,7 +31,7 @@ */ /* - * Copyright 2014 The Error Prone Authors. + * Copyright 2017 The Error Prone Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,7 +47,7 @@ */ /* - * Copyright 2017 The Error Prone Authors. + * Copyright 2014 The Error Prone Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/distribution/engine/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES b/distribution/engine/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES index 10e8435459c1..c4c899bb76e1 100644 --- a/distribution/engine/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES +++ b/distribution/engine/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES @@ -1,7 +1,7 @@ /* - * Copyright (C) 2009-2020 Lightbend Inc. + * Copyright (C) 2017-2020 Lightbend Inc. */ /* - * Copyright (C) 2017-2020 Lightbend Inc. + * Copyright (C) 2009-2020 Lightbend Inc. */ diff --git a/distribution/engine/THIRD-PARTY/com.typesafe.config-1.3.2/NOTICES b/distribution/engine/THIRD-PARTY/com.typesafe.config-1.4.0/NOTICES similarity index 100% rename from distribution/engine/THIRD-PARTY/com.typesafe.config-1.3.2/NOTICES rename to distribution/engine/THIRD-PARTY/com.typesafe.config-1.4.0/NOTICES diff --git a/distribution/engine/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES b/distribution/engine/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES index d39289b94fc7..1e3906f673f8 100644 --- a/distribution/engine/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES +++ b/distribution/engine/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES @@ -15,8 +15,6 @@ * limitations under the License. */ -Copyright 2017-2019 John A. De Goes and the ZIO Contributors - /* * Copyright 2017-2020 John A. De Goes and the ZIO Contributors * Copyright 2017-2018 Łukasz Biały, Paul Chiusano, Michael Pilquist, @@ -36,3 +34,5 @@ Copyright 2017-2019 John A. De Goes and the ZIO Contributors * See the License for the specific language governing permissions and * limitations under the License. */ + +Copyright 2017-2019 John A. De Goes and the ZIO Contributors diff --git a/distribution/engine/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES b/distribution/engine/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES index fea49c9eb3b8..ff981cbc6e70 100644 --- a/distribution/engine/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES +++ b/distribution/engine/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011,2012 Mathias Doenitz, Johannes Rudolph + * Copyright (C) 2011 Mathias Doenitz * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ /* - * Copyright (C) 2011 Mathias Doenitz + * Copyright (C) 2011,2012 Mathias Doenitz, Johannes Rudolph * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICE.txt b/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICE.txt new file mode 100644 index 000000000000..132b0897babc --- /dev/null +++ b/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICE.txt @@ -0,0 +1,55 @@ +Apache Commons Compress +Copyright 2002-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +--- + +The files in the package org.apache.commons.compress.archivers.sevenz +were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), +which has been placed in the public domain: + +"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) + +--- + +The test file lbzip2_32767.bz2 has been copied from libbzip2's source +repository: + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2019 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org diff --git a/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICES b/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICES new file mode 100644 index 000000000000..ea8ae0ca227f --- /dev/null +++ b/distribution/engine/THIRD-PARTY/org.apache.commons.commons-compress-1.20/NOTICES @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Some portions of this file Copyright (c) 2004-2006 Intel Corportation + * and licensed under the BSD license. + */ diff --git a/distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.2/NOTICES b/distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.2/NOTICES deleted file mode 100644 index bd0a85c0c769..000000000000 --- a/distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.2/NOTICES +++ /dev/null @@ -1,10 +0,0 @@ -/************************************************************************ - * Licensed under Public Domain (CC0) * - * * - * To the extent possible under law, the person who associated CC0 with * - * this code has waived all copyright and related or neighboring * - * rights to this code. * - * * - * You should have received a copy of the CC0 legalcode along with this * - * work. If not, see .* - ************************************************************************/ diff --git a/distribution/engine/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES b/distribution/engine/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES index 788494853651..c2bba08c1748 100644 --- a/distribution/engine/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES +++ b/distribution/engine/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES @@ -14,7 +14,7 @@ * limitations under the License. */ -/* Copyright (c) 2008 Google Inc. - // Copyright 2003-2010 Christian d'Heureuse, Inventec Informatik AG, Zurich, Switzerland // www.source-code.biz, www.inventec.ch/chdh + +/* Copyright (c) 2008 Google Inc. diff --git a/distribution/launcher/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES b/distribution/launcher/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES index 788494853651..c2bba08c1748 100644 --- a/distribution/launcher/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES +++ b/distribution/launcher/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES @@ -14,7 +14,7 @@ * limitations under the License. */ -/* Copyright (c) 2008 Google Inc. - // Copyright 2003-2010 Christian d'Heureuse, Inventec Informatik AG, Zurich, Switzerland // www.source-code.biz, www.inventec.ch/chdh + +/* Copyright (c) 2008 Google Inc. diff --git a/distribution/lib/Standard/Table/0.1.0/THIRD-PARTY/com.fasterxml.woodstox.woodstox-core-5.2.1/NOTICES b/distribution/lib/Standard/Table/0.1.0/THIRD-PARTY/com.fasterxml.woodstox.woodstox-core-5.2.1/NOTICES index a214ba7bd6e3..8ef17d630a31 100644 --- a/distribution/lib/Standard/Table/0.1.0/THIRD-PARTY/com.fasterxml.woodstox.woodstox-core-5.2.1/NOTICES +++ b/distribution/lib/Standard/Table/0.1.0/THIRD-PARTY/com.fasterxml.woodstox.woodstox-core-5.2.1/NOTICES @@ -1,5 +1,5 @@ Copyright (c) 2005 Tatu Saloranta, tatu.saloranta@iki.fi -Copyright (c) 2004- Tatu Saloranta, tatu.saloranta@iki.fi - Copyright (c) 2004 Tatu Saloranta, tatu.saloranta@iki.fi + +Copyright (c) 2004- Tatu Saloranta, tatu.saloranta@iki.fi diff --git a/distribution/project-manager/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES b/distribution/project-manager/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES index 10e8435459c1..c4c899bb76e1 100644 --- a/distribution/project-manager/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES +++ b/distribution/project-manager/THIRD-PARTY/com.typesafe.akka.akka-http-spray-json_2.13-10.2.0-RC1/NOTICES @@ -1,7 +1,7 @@ /* - * Copyright (C) 2009-2020 Lightbend Inc. + * Copyright (C) 2017-2020 Lightbend Inc. */ /* - * Copyright (C) 2017-2020 Lightbend Inc. + * Copyright (C) 2009-2020 Lightbend Inc. */ diff --git a/distribution/project-manager/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES b/distribution/project-manager/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES index d39289b94fc7..1e3906f673f8 100644 --- a/distribution/project-manager/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES +++ b/distribution/project-manager/THIRD-PARTY/dev.zio.zio_2.13-1.0.1/NOTICES @@ -15,8 +15,6 @@ * limitations under the License. */ -Copyright 2017-2019 John A. De Goes and the ZIO Contributors - /* * Copyright 2017-2020 John A. De Goes and the ZIO Contributors * Copyright 2017-2018 Łukasz Biały, Paul Chiusano, Michael Pilquist, @@ -36,3 +34,5 @@ Copyright 2017-2019 John A. De Goes and the ZIO Contributors * See the License for the specific language governing permissions and * limitations under the License. */ + +Copyright 2017-2019 John A. De Goes and the ZIO Contributors diff --git a/distribution/project-manager/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES b/distribution/project-manager/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES index fea49c9eb3b8..ff981cbc6e70 100644 --- a/distribution/project-manager/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES +++ b/distribution/project-manager/THIRD-PARTY/io.spray.spray-json_2.13-1.3.5/NOTICES @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011,2012 Mathias Doenitz, Johannes Rudolph + * Copyright (C) 2011 Mathias Doenitz * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ /* - * Copyright (C) 2011 Mathias Doenitz + * Copyright (C) 2011,2012 Mathias Doenitz, Johannes Rudolph * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/distribution/project-manager/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES b/distribution/project-manager/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES index 788494853651..c2bba08c1748 100644 --- a/distribution/project-manager/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES +++ b/distribution/project-manager/THIRD-PARTY/org.yaml.snakeyaml-1.26/NOTICES @@ -14,7 +14,7 @@ * limitations under the License. */ -/* Copyright (c) 2008 Google Inc. - // Copyright 2003-2010 Christian d'Heureuse, Inventec Informatik AG, Zurich, Switzerland // www.source-code.biz, www.inventec.ch/chdh + +/* Copyright (c) 2008 Google Inc. diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala index b8fe0dcd021c..dee894374d1b 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala @@ -39,9 +39,9 @@ import org.enso.languageserver.text.BufferRegistry import org.enso.pkg.PackageManager import org.enso.polyglot.data.TypeGraph import org.enso.polyglot.runtime.Runtime.Api -import org.enso.runtimeversionmanager.test.{FakeEnvironment, HasTestDirectory} +import org.enso.runtimeversionmanager.test.FakeEnvironment import org.enso.searcher.sql.{SqlDatabase, SqlSuggestionsRepo, SqlVersionsRepo} -import org.enso.testkit.EitherValue +import org.enso.testkit.{EitherValue, HasTestDirectory} import org.enso.text.Sha3_224VersionCalculator import org.scalatest.OptionValues diff --git a/engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala b/engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala index b101eb3440ad..e249772c05bf 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/cli/InternalOpts.scala @@ -4,9 +4,10 @@ import java.io.IOException import java.nio.file.{Files, NoSuchFileException, Path} import cats.implicits._ import nl.gn0s1s.bump.SemVer +import org.enso.cli.OS import org.enso.cli.arguments.Opts import org.enso.cli.arguments.Opts.implicits._ -import org.enso.distribution.{FileSystem, OS} +import org.enso.distribution.FileSystem import org.enso.runtimeversionmanager.CurrentVersion import org.enso.distribution.FileSystem.PathSyntax import org.enso.runtimeversionmanager.cli.Arguments._ diff --git a/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala b/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala index c8aa1968d40e..c9516d414bc0 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala @@ -1,6 +1,9 @@ package org.enso.launcher.distribution -import org.enso.distribution.PortableDistributionManager +import org.enso.distribution.{ + PortableDistributionManager, + TemporaryDirectoryManager +} import org.enso.distribution.locking.{ ResourceManager, ThreadSafeFileLockManager @@ -16,7 +19,6 @@ import org.enso.runtimeversionmanager.components.{ RuntimeComponentUpdaterFactory, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.engine.EngineRepository import org.enso.runtimeversionmanager.releases.graalvm.GraalCEReleaseProvider @@ -42,7 +44,7 @@ object DefaultManagers { /** Default [[TemporaryDirectoryManager]]. */ lazy val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, defaultResourceManager) + TemporaryDirectoryManager(distributionManager, defaultResourceManager) /** Default [[RuntimeComponentConfiguration]]. */ lazy val componentConfig: RuntimeComponentConfiguration = diff --git a/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionInstaller.scala b/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionInstaller.scala index 2cde4028dae2..bcbdba9678a0 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionInstaller.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionInstaller.scala @@ -2,11 +2,10 @@ package org.enso.launcher.installation import java.nio.file.{Files, Path} import com.typesafe.scalalogging.Logger -import org.enso.cli.CLIOutput +import org.enso.cli.{CLIOutput, OS} import org.enso.distribution.{ DistributionManager, FileSystem, - OS, PortableDistributionManager } import org.enso.distribution.locking.ResourceManager diff --git a/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionUninstaller.scala b/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionUninstaller.scala index 36651980cc9c..20fc0f0b51e3 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionUninstaller.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/installation/DistributionUninstaller.scala @@ -3,11 +3,10 @@ package org.enso.launcher.installation import java.nio.file.{Files, Path} import com.typesafe.scalalogging.Logger import org.apache.commons.io.FileUtils -import org.enso.cli.CLIOutput +import org.enso.cli.{CLIOutput, OS} import org.enso.distribution.{ DistributionManager, FileSystem, - OS, PortableDistributionManager } import org.enso.distribution.locking.ResourceManager diff --git a/engine/launcher/src/main/scala/org/enso/launcher/releases/LauncherRepository.scala b/engine/launcher/src/main/scala/org/enso/launcher/releases/LauncherRepository.scala index 50e96ad0bcb1..e74492f6b968 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/releases/LauncherRepository.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/releases/LauncherRepository.scala @@ -1,8 +1,8 @@ package org.enso.launcher.releases import java.nio.file.Path - import com.typesafe.scalalogging.Logger +import org.enso.downloader.http.URIBuilder import org.enso.launcher.distribution.DefaultManagers import org.enso.launcher.releases.fallback.SimpleReleaseProviderWithFallback import org.enso.launcher.releases.fallback.staticwebsite.StaticWebsiteFallbackReleaseProvider @@ -10,7 +10,6 @@ import org.enso.launcher.releases.launcher.{ LauncherRelease, LauncherReleaseProvider } -import org.enso.runtimeversionmanager.http.URIBuilder import org.enso.runtimeversionmanager.releases.engine.EngineRepository import org.enso.runtimeversionmanager.releases.testing.FakeReleaseProvider import org.enso.runtimeversionmanager.releases.{ diff --git a/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsite.scala b/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsite.scala index 0e8bd12c1e3e..6ef2527f4494 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsite.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsite.scala @@ -1,13 +1,9 @@ package org.enso.launcher.releases.fallback.staticwebsite -import java.nio.file.Path - import org.enso.cli.task.TaskProgress -import org.enso.runtimeversionmanager.http.{ - HTTPDownload, - HTTPRequestBuilder, - URIBuilder -} +import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} + +import java.nio.file.Path /** Provides [[FileStorage]] backed by a static HTTPS website. * diff --git a/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsiteFallbackReleaseProvider.scala b/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsiteFallbackReleaseProvider.scala index 82e6a85c6e35..d30cc7e71ac9 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsiteFallbackReleaseProvider.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/releases/fallback/staticwebsite/StaticWebsiteFallbackReleaseProvider.scala @@ -1,6 +1,6 @@ package org.enso.launcher.releases.fallback.staticwebsite -import org.enso.runtimeversionmanager.http.URIBuilder +import org.enso.downloader.http.URIBuilder import org.enso.runtimeversionmanager.releases.Release import org.enso.launcher.releases.fallback.FallbackReleaseProvider diff --git a/engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala b/engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala index edb5ff488cef..3080e753246d 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/upgrade/LauncherUpgrader.scala @@ -3,8 +3,8 @@ package org.enso.launcher.upgrade import java.nio.file.{Files, Path} import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer -import org.enso.cli.CLIOutput -import org.enso.distribution.{DistributionManager, FileSystem, OS} +import org.enso.cli.{CLIOutput, OS} +import org.enso.distribution.{DistributionManager, FileSystem} import org.enso.distribution.locking.{ LockType, LockUserInterface, @@ -13,7 +13,7 @@ import org.enso.distribution.locking.{ } import org.enso.runtimeversionmanager.CurrentVersion import org.enso.distribution.FileSystem.PathSyntax -import org.enso.runtimeversionmanager.archive.Archive +import org.enso.downloader.archive.Archive import org.enso.runtimeversionmanager.components.UpgradeRequiredError import org.enso.launcher.cli.{ CLIProgressReporter, diff --git a/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala b/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala index 9f8f871eb26b..08a289c53f49 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala @@ -1,9 +1,8 @@ package org.enso.launcher -import org.enso.distribution.OS - -import java.nio.file.{Files, Path} +import org.enso.cli.OS import org.enso.runtimeversionmanager.test.NativeTestHelper +import org.enso.testkit.process.RunResult import org.scalatest.concurrent.{Signaler, TimeLimitedTests} import org.scalatest.matchers.should.Matchers import org.scalatest.matchers.{MatchResult, Matcher} @@ -11,6 +10,8 @@ import org.scalatest.time.Span import org.scalatest.time.SpanSugar._ import org.scalatest.wordspec.AnyWordSpec +import java.nio.file.{Files, Path} + /** Contains helper methods for creating tests that need to run the native * launcher binary. */ diff --git a/engine/launcher/src/test/scala/org/enso/launcher/PluginManagerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/PluginManagerSpec.scala index b7ed0f6bd5df..630fe3a53f04 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/PluginManagerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/PluginManagerSpec.scala @@ -1,8 +1,8 @@ package org.enso.launcher -import java.nio.file.{Files, Path} +import org.enso.testkit.WithTemporaryDirectory -import org.enso.runtimeversionmanager.test.WithTemporaryDirectory +import java.nio.file.{Files, Path} import org.scalatest.OptionValues import scala.jdk.CollectionConverters._ diff --git a/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala index 39c1602175ec..18d0ab35e688 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala @@ -145,7 +145,7 @@ class LauncherRunnerSpec extends RuntimeVersionManagerTest { val runner = makeFakeRunner() val projectPath = getTestDirectory / "project2" val nightlyVersion = SemVer(0, 0, 0, Some("SNAPSHOT.2000-01-01")) - val logs = TestLogger.gatherLogs { + val (_, logs) = TestLogger.gatherLogs { runner .newProject( path = projectPath, diff --git a/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala index d0dce631600d..998d06f0ab1d 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/installation/InstallerSpec.scala @@ -1,11 +1,12 @@ package org.enso.launcher.installation -import org.enso.distribution.{FileSystem, OS} +import org.enso.distribution.FileSystem import java.nio.file.{Files, Path} import FileSystem.PathSyntax -import org.enso.runtimeversionmanager.test.WithTemporaryDirectory +import org.enso.cli.OS import org.enso.launcher._ +import org.enso.testkit.WithTemporaryDirectory class InstallerSpec extends NativeTest with WithTemporaryDirectory { def portableRoot = getTestDirectory / "portable" diff --git a/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala index 9cebe1478317..a52ee905a367 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala @@ -1,11 +1,12 @@ package org.enso.launcher.installation -import org.enso.distribution.{FileSystem, OS} +import org.enso.distribution.FileSystem import java.nio.file.{Files, Path} import FileSystem.PathSyntax -import org.enso.runtimeversionmanager.test.WithTemporaryDirectory +import org.enso.cli.OS import org.enso.launcher.NativeTest +import org.enso.testkit.WithTemporaryDirectory class UninstallerSpec extends NativeTest with WithTemporaryDirectory { def installedRoot: Path = getTestDirectory / "installed" diff --git a/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala index 73175a3e0b57..49d459ef91c3 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala @@ -3,12 +3,13 @@ package org.enso.launcher.upgrade import java.nio.file.{Files, Path, StandardCopyOption} import io.circe.parser import nl.gn0s1s.bump.SemVer -import org.enso.distribution.{FileSystem, OS} +import org.enso.distribution.FileSystem import org.enso.distribution.locking.{FileLockManager, LockType} import FileSystem.PathSyntax +import org.enso.cli.OS import org.enso.launcher._ -import org.enso.runtimeversionmanager.test.WithTemporaryDirectory -import org.enso.testkit.RetrySpec +import org.enso.testkit.{RetrySpec, WithTemporaryDirectory} +import org.enso.testkit.process.{RunResult, WrappedProcess} import org.scalatest.exceptions.TestFailedException import org.scalatest.{BeforeAndAfterAll, OptionValues} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java index 15fa10bcb566..93d02381605e 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java @@ -15,6 +15,7 @@ import org.enso.compiler.Compiler; import org.enso.compiler.PackageRepository; import org.enso.compiler.data.CompilerConfig; +import org.enso.distribution.locking.ThreadSafeFileLockManager; import org.enso.editions.LibraryName; import org.enso.interpreter.Language; import org.enso.interpreter.OptionsHelper; @@ -101,11 +102,19 @@ public void initialize() { var languageHome = OptionsHelper.getLanguageHomeOverride(environment).or(() -> Optional.ofNullable(home)); + var distributionManager = RuntimeDistributionManager$.MODULE$; + + // TODO [RW] Once #1890 is implemented, this will need to connect to the Language Server's + // LockManager. + var lockManager = new ThreadSafeFileLockManager(distributionManager.paths().locks()); + var resourceManager = new org.enso.distribution.locking.ResourceManager(lockManager); + packageRepository = PackageRepository.initializeRepository( OptionConverters.toScala(projectPackage), OptionConverters.toScala(languageHome), - RuntimeDistributionManager$.MODULE$, + distributionManager, + resourceManager, this, builtins, notificationHandler); diff --git a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala index 8a17740b7d85..d645a88f0c42 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala @@ -2,6 +2,7 @@ package org.enso.compiler import com.oracle.truffle.api.TruffleFile import com.typesafe.scalalogging.Logger +import org.enso.distribution.locking.ResourceManager import org.enso.distribution.{DistributionManager, EditionManager, LanguageHome} import org.enso.editions.{DefaultEdition, LibraryName, LibraryVersion} import org.enso.interpreter.instrument.NotificationHandler @@ -198,16 +199,14 @@ object PackageRepository { else { logger.trace(s"Resolving library $libraryName.") val resolvedLibrary = libraryProvider.findLibrary(libraryName) - logger.whenTraceEnabled { - resolvedLibrary match { - case Left(error) => - logger.trace(s"Resolution failed with [$error].") - case Right(resolved) => - logger.trace( - s"Found library ${resolved.name} @ ${resolved.version} " + - s"at [${MaskedPath(resolved.location).applyMasking()}]." - ) - } + resolvedLibrary match { + case Left(error) => + logger.error(s"Resolution failed with [$error].", error) + case Right(resolved) => + logger.trace( + s"Found library ${resolved.name} @ ${resolved.version} " + + s"at [${MaskedPath(resolved.location).applyMasking()}]." + ) } this.synchronized { @@ -318,6 +317,7 @@ object PackageRepository { * @param projectPackage the package of the current project (if ran inside of a project) * @param languageHome the language home (if set) * @param distributionManager the distribution manager + * @param resourceManager the resource manager instance * @param context the context reference, needed to add polyglot libraries to * the classpath * @param builtins the builtins that are always preloaded @@ -329,6 +329,7 @@ object PackageRepository { projectPackage: Option[Package[TruffleFile]], languageHome: Option[String], distributionManager: DistributionManager, + resourceManager: ResourceManager, context: Context, builtins: Builtins, notificationHandler: NotificationHandler @@ -344,6 +345,9 @@ object PackageRepository { val resolvingLibraryProvider = new DefaultLibraryProvider( distributionManager = distributionManager, + resourceManager = resourceManager, + lockUserInterface = notificationHandler, + progressReporter = notificationHandler, languageHome = homeManager, edition = edition, preferLocalLibraries = diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala index ef420792b902..5217ce9430f5 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/NotificationHandler.scala @@ -2,21 +2,18 @@ package org.enso.interpreter.instrument import com.typesafe.scalalogging.Logger import org.enso.cli.ProgressBar -import org.enso.cli.task.{ - ProgressNotification, - ProgressNotificationForwarder, - ProgressReporter, - TaskProgress -} +import org.enso.cli.task.{ProgressNotification, ProgressReporter, TaskProgress} +import org.enso.distribution.ProgressAndLockNotificationForwarder +import org.enso.distribution.locking.{LockUserInterface, Resource} import org.enso.editions.{LibraryName, LibraryVersion} import org.enso.polyglot.runtime.Runtime.{Api, ApiResponse} import java.nio.file.Path -/** A class that forwards notifications about loaded libraries and long-running - * tasks to the user interface. +/** A class that forwards notifications about loaded libraries, locks and + * long-running tasks to the user interface. */ -trait NotificationHandler extends ProgressReporter { +trait NotificationHandler extends ProgressReporter with LockUserInterface { /** Called when a library has been loaded. * @@ -40,6 +37,8 @@ object NotificationHandler { */ object TextMode extends NotificationHandler { + private lazy val logger = Logger[TextMode.type] + /** @inheritdoc */ override def addedLibrary( libraryName: LibraryName, @@ -51,11 +50,18 @@ object NotificationHandler { /** @inheritdoc */ override def trackProgress(message: String, task: TaskProgress[_]): Unit = { - Logger[TextMode.type].info(message) + logger.info(message) if (System.console() != null) { ProgressBar.waitWithProgress(task) } } + + /** @inheritdoc */ + override def startWaitingForResource(resource: Resource): Unit = + logger.warn(resource.waitMessage) + + /** @inheritdoc */ + override def finishWaitingForResource(resource: Resource): Unit = () } /** A [[NotificationHandler]] that forwards messages to other @@ -76,6 +82,14 @@ object NotificationHandler { override def trackProgress(message: String, task: TaskProgress[_]): Unit = for (listener <- listeners) listener.trackProgress(message, task) + /** @inheritdoc */ + override def startWaitingForResource(resource: Resource): Unit = + for (listener <- listeners) listener.startWaitingForResource(resource) + + /** @inheritdoc */ + override def finishWaitingForResource(resource: Resource): Unit = + for (listener <- listeners) listener.finishWaitingForResource(resource) + /** Registers a new listener. */ def addListener(listener: NotificationHandler): Unit = listeners ::= listener @@ -86,8 +100,8 @@ object NotificationHandler { * the IDE. */ class InteractiveMode(endpoint: Endpoint) - extends NotificationHandler - with ProgressNotificationForwarder { + extends ProgressAndLockNotificationForwarder + with NotificationHandler { private val logger = Logger[InteractiveMode] private def sendMessage(message: ApiResponse): Unit = { diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/OS.scala b/lib/scala/cli/src/main/scala/org/enso/cli/OS.scala similarity index 99% rename from lib/scala/distribution-manager/src/main/scala/org/enso/distribution/OS.scala rename to lib/scala/cli/src/main/scala/org/enso/cli/OS.scala index 3288020dc972..a1e537726e32 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/OS.scala +++ b/lib/scala/cli/src/main/scala/org/enso/cli/OS.scala @@ -1,4 +1,4 @@ -package org.enso.distribution +package org.enso.cli import com.typesafe.scalalogging.Logger import io.circe.{Decoder, DecodingFailure} diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala index 056d029ff6db..ce7a31f5aaa5 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala @@ -1,6 +1,7 @@ package org.enso.distribution import com.typesafe.scalalogging.Logger +import org.enso.cli.OS import org.enso.distribution.FileSystem.PathSyntax import org.enso.logger.masking.{MaskedPath, ToLogString} diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/Environment.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/Environment.scala index a7a4f8a9b92e..eced083686e6 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/Environment.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/Environment.scala @@ -1,6 +1,7 @@ package org.enso.distribution import com.typesafe.scalalogging.Logger +import org.enso.cli.OS import org.enso.logger.masking.MaskedString import java.io.File diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/FileSystem.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/FileSystem.scala index a1edcada6618..271ade1cc279 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/FileSystem.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/FileSystem.scala @@ -2,6 +2,7 @@ package org.enso.distribution import com.typesafe.scalalogging.Logger import org.apache.commons.io.FileUtils +import org.enso.cli.OS import java.io.PrintWriter import java.nio.file.attribute.PosixFilePermissions diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala new file mode 100644 index 000000000000..6a31a5253792 --- /dev/null +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/ProgressAndLockNotificationForwarder.scala @@ -0,0 +1,51 @@ +package org.enso.distribution + +import org.enso.cli.task.{ + ProgressNotification, + ProgressNotificationForwarder, + ProgressUnit +} +import org.enso.distribution.locking.{LockUserInterface, Resource} + +import java.util.UUID + +/** A helper class that provides an implementation of + * [[ProgressNotificationForwarder]] and [[LockUserInterface]] which are also + * forwarded as progress notifications with indeterminate progress amounts. + * + * All it needs to function is for the user to define the + * `sendProgressNotification` method. + */ +abstract class ProgressAndLockNotificationForwarder + extends ProgressNotificationForwarder + with LockUserInterface { + private val waitingForResources = + collection.concurrent.TrieMap[String, UUID]() + + /** @inheritdoc */ + override def startWaitingForResource(resource: Resource): Unit = { + val uuid = UUID.randomUUID() + sendProgressNotification( + ProgressNotification.TaskStarted( + uuid, + None, + ProgressUnit.Unspecified + ) + ) + sendProgressNotification( + ProgressNotification.TaskUpdate( + uuid, + Some(resource.waitMessage), + 0 + ) + ) + waitingForResources.put(resource.name, uuid) + } + + /** @inheritdoc */ + override def finishWaitingForResource(resource: Resource): Unit = { + for (uuid <- waitingForResources.remove(resource.name)) { + sendProgressNotification(ProgressNotification.TaskSuccess(uuid)) + } + } +} diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala new file mode 100644 index 000000000000..bbdb823b415b --- /dev/null +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/TemporaryDirectoryManager.scala @@ -0,0 +1,102 @@ +package org.enso.distribution + +import com.typesafe.scalalogging.Logger +import org.enso.distribution.locking.ResourceManager + +import java.nio.file.{FileAlreadyExistsException, Files, Path} +import scala.util.Random + +/** Manages safe access to the local temporary directory. + * + * The temporary directory is created on demand and automatically removed if it + * is empty. Temporary files from previous runs are removed when the temporary + * directory is first accessed. Locking mechanism is used to ensure that the + * old files are no longer used by any other instances running in parallel. + * + * The local temporary directory is located inside of ENSO_DATA_ROOT, which + * means it should be on the same disk partition as directories keeping + * engines, runtimes and cached libraries. + * + * This directory is used as a destination for extracting component packages, + * so that they can be moved all at once at the last step of the installation. + */ +class TemporaryDirectoryManager( + unsafeRoot: Path, + resourceManager: ResourceManager +) { + private val logger = Logger[TemporaryDirectoryManager] + private val random = new Random() + + /** Creates a unique temporary subdirectory. */ + def temporarySubdirectory(prefix: String = ""): Path = { + val paddedPrefix = if (prefix != "") prefix + "-" else prefix + val randomSuffix = random.nextInt().toString.stripPrefix("-") + val path = + safeTemporaryDirectory.resolve(paddedPrefix + randomSuffix) + if (Files.exists(path)) + temporarySubdirectory(prefix) + else { + try { + Files.createDirectory(path) + } catch { + case _: FileAlreadyExistsException => + temporarySubdirectory(prefix) + } + } + } + + /** Returns path to a directory for storing temporary files that is located on + * the same filesystem as `runtimes` and `engines`. + * + * It is used during installation to decrease the possibility of getting a + * broken installation if the installation process has been abruptly + * terminated. The directory is created on demand (when its path is requested + * for the first time) and is removed if the application exits normally (as + * long as it is empty, but normal termination of the installation process + * should ensure that). If that fails, it is also cleaned before any future + * accesses. + */ + private lazy val safeTemporaryDirectory: Path = { + resourceManager.startUsingTemporaryDirectory() + Files.createDirectories(unsafeRoot) + unsafeRoot + } + + /** Tries to clean the temporary files directory. + * + * It should be run at startup whenever the program wants to run clean-up. + * Currently it is run when installation-related operations are taking place. + * It may not proceed if another process is using it. It has to be run before + * the first access to the temporaryDirectory, as after that the directory is + * marked as in-use and will not be cleaned. + */ + def tryCleaningTemporaryDirectory(): Unit = { + if (Files.exists(unsafeRoot)) { + resourceManager.tryWithExclusiveTemporaryDirectory { + if (!FileSystem.isDirectoryEmpty(unsafeRoot)) { + logger.info( + "Cleaning up temporary files from a previous installation." + ) + } + FileSystem.removeDirectory(unsafeRoot) + Files.createDirectories(unsafeRoot) + FileSystem.removeEmptyDirectoryOnExit(unsafeRoot) + } + } + } +} + +object TemporaryDirectoryManager { + + /** A helper constructor that creates a [[TemporaryDirectoryManager]] using + * the temporary directory inside of the distribution managed by the provided + * [[DistributionManager]]. + */ + def apply( + distribution: DistributionManager, + resourceManager: ResourceManager + ): TemporaryDirectoryManager = new TemporaryDirectoryManager( + distribution.paths.unsafeTemporaryDirectory, + resourceManager + ) +} diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/locking/ResourceManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/locking/ResourceManager.scala index 785077da6b5c..7234f7cefc02 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/locking/ResourceManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/locking/ResourceManager.scala @@ -151,10 +151,10 @@ class ResourceManager(lockManager: LockManager) { */ def startUsingTemporaryDirectory(): Unit = { if (temporaryDirectoryLock.isDefined) { - throw new IllegalStateException( - "Temporary directory lock has been acquired twice." - ) + logger.trace("The temporary directory was already in-use.") + return } + val lock = lockManager.acquireLockWithWaitingAction( TemporaryDirectory.name, LockType.Shared, diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/Archive.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/Archive.scala similarity index 98% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/Archive.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/Archive.scala index baadc5044933..8fc9618bb1f5 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/Archive.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/Archive.scala @@ -1,12 +1,6 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive -import java.io.BufferedInputStream -import java.nio.file.{Files, Path} import com.typesafe.scalalogging.Logger -import org.apache.commons.compress.archivers.{ - ArchiveInputStream, - ArchiveEntry => ApacheArchiveEntry -} import org.apache.commons.compress.archivers.tar.{ TarArchiveEntry, TarArchiveInputStream @@ -15,20 +9,26 @@ import org.apache.commons.compress.archivers.zip.{ ZipArchiveEntry, ZipArchiveInputStream } +import org.apache.commons.compress.archivers.{ + ArchiveInputStream, + ArchiveEntry => ApacheArchiveEntry +} import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream import org.apache.commons.io.IOUtils +import org.enso.cli.OS import org.enso.cli.task.{ ProgressUnit, TaskProgress, TaskProgressImplementation } -import org.enso.distribution.OS -import org.enso.runtimeversionmanager.archive.internal.{ +import org.enso.downloader.archive.internal.{ ArchiveIterator, - BaseRenamer + BaseRenamer, + ReadProgress } -import org.enso.runtimeversionmanager.internal.ReadProgress +import java.io.BufferedInputStream +import java.nio.file.{Files, Path} import scala.util.{Try, Using} /** Contains utilities related to the extraction of various archive file diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveEntry.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveEntry.scala similarity index 93% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveEntry.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveEntry.scala index cdd82a5e78b5..e4f7c05e8de7 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveEntry.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveEntry.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive import java.nio.file.Path diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveFormat.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveFormat.scala similarity index 93% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveFormat.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveFormat.scala index cc2a3fc0fc13..dfd3798c5b6a 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/ArchiveFormat.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/ArchiveFormat.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive import java.nio.file.Path diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/FileProgressInputStream.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/FileProgressInputStream.scala similarity index 85% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/FileProgressInputStream.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/FileProgressInputStream.scala index deb8c0470b41..e5dcddb144dd 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/FileProgressInputStream.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/FileProgressInputStream.scala @@ -1,10 +1,10 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive + +import org.enso.downloader.archive.internal.ProgressInputStream import java.io.FileInputStream import java.nio.file.{Files, Path} -import org.enso.runtimeversionmanager.internal.ProgressInputStream - /** A helper that allows to create a [[ProgressInputStream]] for a file located * at the given path. */ diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/POSIXPermissions.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/POSIXPermissions.scala similarity index 96% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/POSIXPermissions.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/POSIXPermissions.scala index 3caee1331183..d3258895c2c4 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/POSIXPermissions.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/POSIXPermissions.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive import java.nio.file.attribute.PosixFilePermission import java.util diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/ArchiveIterator.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ArchiveIterator.scala similarity index 95% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/ArchiveIterator.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ArchiveIterator.scala index 64b25b68e67c..e4d74fdc9e11 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/ArchiveIterator.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ArchiveIterator.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.archive.internal +package org.enso.downloader.archive.internal import org.apache.commons.compress.archivers.{ArchiveEntry, ArchiveInputStream} diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/BaseRenamer.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/BaseRenamer.scala similarity index 96% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/BaseRenamer.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/BaseRenamer.scala index 678ae0670ffd..bc754c8acb16 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/archive/internal/BaseRenamer.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/BaseRenamer.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.archive.internal +package org.enso.downloader.archive.internal import java.nio.file.Path diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/internal/ProgressInputStream.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ProgressInputStream.scala similarity index 97% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/internal/ProgressInputStream.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ProgressInputStream.scala index 10d85b882c05..28c0d1eb1bee 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/internal/ProgressInputStream.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/archive/internal/ProgressInputStream.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.internal +package org.enso.downloader.archive.internal import java.io.InputStream diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/APIResponse.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/APIResponse.scala new file mode 100644 index 000000000000..851f45f4b3f5 --- /dev/null +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/APIResponse.scala @@ -0,0 +1,11 @@ +package org.enso.downloader.http + +/** Contains the response contents as a string alongside with the headers + * included in the response. + * + * @param content the response decoded as a string + * @param headers sequence of headers included in the response + * @param statusCode the response status code, indicating whether the request + * has succeeded or failed + */ +case class APIResponse(content: String, headers: Seq[Header], statusCode: Int) diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPDownload.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala similarity index 67% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPDownload.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala index 9c63d0183ea7..497aa0ade71a 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPDownload.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPDownload.scala @@ -1,12 +1,9 @@ -package org.enso.runtimeversionmanager.http - -import java.nio.charset.{Charset, StandardCharsets} -import java.nio.file.Path +package org.enso.downloader.http import akka.actor.ActorSystem import akka.http.scaladsl.Http import akka.http.scaladsl.model.headers.Location -import akka.http.scaladsl.model.{HttpRequest, HttpResponse} +import akka.http.scaladsl.model.{HttpRequest, HttpResponse, Uri} import akka.stream.scaladsl.{FileIO, Sink} import akka.util.ByteString import com.typesafe.config.{ConfigFactory, ConfigValueFactory} @@ -17,34 +14,17 @@ import org.enso.cli.task.{ TaskProgressImplementation } +import java.nio.charset.{Charset, StandardCharsets} +import java.nio.file.Path import scala.concurrent.Future +import scala.jdk.CollectionConverters.IterableHasAsJava +import scala.util.{Failure, Success, Try} -/** Represents a HTTP header. */ -case class Header(name: String, value: String) { - - /** Checks if this header instance corresponds to a `headerName`. - * - * The check is case-insensitive. - */ - def is(headerName: String): Boolean = - name.toLowerCase == headerName.toLowerCase -} - -/** Contains the response contents as a string alongside with the headers - * included in the response. - * - * @param content the response decoded as a string - * @param headers sequence of headers included in the response - */ -case class APIResponse(content: String, headers: Seq[Header]) - -/** Contains utility functions for fetching data using the HTTP(S) protocol. - */ +/** Contains utility functions for fetching data using the HTTP(S) protocol. */ object HTTPDownload { private val logger = Logger[HTTPDownload.type] - /** Determines how many redirects are taken until an error is thrown. - */ + /** Determines how many redirects are taken until an error is thrown. */ val maximumRedirects: Int = 20 /** Fetches the `request` and tries to decode is as a [[String]]. @@ -74,17 +54,35 @@ object HTTPDownload { def combineChunks(chunks: Seq[ByteString]): String = chunks.reduceOption(_ ++ _).map(_.decodeString(encoding)).getOrElse("") runRequest( - request.requestImpl, - sizeHint, - Sink.seq, - (response, chunks: Seq[ByteString]) => - APIResponse( - combineChunks(chunks), - response.headers.map(header => Header(header.name, header.value)) + request = request.requestImpl, + sizeHint = sizeHint, + earlyResponseMapping = response => Success(response), + sink = Sink.seq, + resultMapping = (response, chunks: Seq[ByteString]) => + Success( + APIResponse( + combineChunks(chunks), + response.headers.map(header => Header(header.name, header.value)), + response.status.intValue() + ) ) ) } + /** Fetches the `uri` and tries to decode is as a [[String]]. + * + * It is a shorthand for the other variant of [[fetchString]] that creates a + * simple GET request from the URI. + * + * @param uri the URI to query + * @return a [[TaskProgress]] that tracks progress of the download and can + * be used to get the final result + */ + def fetchString(uri: Uri): TaskProgress[APIResponse] = { + val request = HTTPRequestBuilder.fromURI(uri).GET + fetchString(request) + } + /** Downloads the `request` and saves the response in the file pointed by the * `destination`. * @@ -114,20 +112,62 @@ object HTTPDownload { destination ) runRequest( - request.requestImpl, - sizeHint, - FileIO.toPath(destination), - (_, _: Any) => destination + request = request.requestImpl, + sizeHint = sizeHint, + earlyResponseMapping = { response => + if (response.status.isSuccess) + Success(response) + else if (response.status.intValue == 404) + Failure(ResourceNotFound()) + else + Failure( + HTTPException(s"Server responded with: [${response.status.value}].") + ) + }, + sink = FileIO.toPath(destination), + resultMapping = (_, _: Any) => Success(destination) ) } + /** Downloads the `uri` and saves the response in the file pointed by the + * `destination`. + * + * It is a shorthand for the other variant of [[download]] that creates a + * simple GET request from the URI. + * + * @param uri the uri to download + * @return a [[TaskProgress]] that tracks progress of the download and can + * be used to wait for the completion of the download. + */ + def download(uri: Uri, destination: Path): TaskProgress[Path] = { + val request = HTTPRequestBuilder.fromURI(uri).GET + download(request, destination) + } + implicit private lazy val actorSystem: ActorSystem = { + val loggers: java.lang.Iterable[String] = + Seq("akka.event.slf4j.Slf4jLogger").asJava val config = ConfigFactory .load() + .withValue( + "akka.extensions", + ConfigValueFactory.fromAnyRef(Seq.empty.asJava) + ) + .withValue( + "akka.library-extensions", + ConfigValueFactory.fromAnyRef(Seq.empty.asJava) + ) + .withValue("akka.loggers", ConfigValueFactory.fromAnyRef(loggers)) + .withValue( + "akka.logging-filter", + ConfigValueFactory.fromAnyRef("akka.event.DefaultLoggingFilter") + ) .withValue("akka.loglevel", ConfigValueFactory.fromAnyRef("WARNING")) + ActorSystem( "http-requests-actor-system", - config + config, + classLoader = getClass.getClassLoader // Note [Actor System Class Loader] ) } @@ -141,6 +181,11 @@ object HTTPDownload { * @param sizeHint an optional hint indicating the expected size of the * response. It is used if the response does not include * explicit Content-Length header. + * @param earlyResponseMapping a mapping that can be used to alter the + * response or handle any early errors; it is run + * before passing the response through the + * `sink`; thus it can be used to avoid creating + * downloaded files if the request fails * @param sink specifies how the response content should be handled, it * receives chunks of [[ByteString]] and should produce a * [[Future]] with some result @@ -154,8 +199,9 @@ object HTTPDownload { private def runRequest[A, B]( request: HttpRequest, sizeHint: Option[Long], + earlyResponseMapping: HttpResponse => Try[HttpResponse], sink: Sink[ByteString, Future[A]], - resultMapping: (HttpResponse, A) => B + resultMapping: (HttpResponse, A) => Try[B] ): TaskProgress[B] = { // TODO [RW] Add optional stream encoding allowing for compression - // add headers and decode the stream if necessary (#1805). @@ -220,8 +266,9 @@ object HTTPDownload { http .singleRequest(request) .flatMap(handleRedirects(maximumRedirects)) + .flatMap(earlyResponseMapping andThen Future.fromTry) .flatMap(handleFinalResponse) - .map(resultMapping.tupled) + .flatMap(resultMapping.tupled andThen Future.fromTry) .onComplete(taskProgress.setComplete) taskProgress } diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPException.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPException.scala new file mode 100644 index 000000000000..42cc86da6d78 --- /dev/null +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPException.scala @@ -0,0 +1,14 @@ +package org.enso.downloader.http + +/** Indicates an error when processing a HTTP request. */ +class HTTPException(message: String) extends RuntimeException(message) + +object HTTPException { + + /** A helper constructor for [[HTTPException]]. */ + def apply(message: String): HTTPException = new HTTPException(message) +} + +/** Indicates that the HTTP request failed with 404 status. */ +case class ResourceNotFound() + extends HTTPException("The server has responded with 404 status.") diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequest.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequest.scala similarity index 83% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequest.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequest.scala index 6578bc3d34d9..134d5e32318e 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequest.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequest.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.http +package org.enso.downloader.http import akka.http.scaladsl.model.HttpRequest diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequestBuilder.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala similarity index 88% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequestBuilder.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala index 5f1b730683d3..f525d20da772 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPRequestBuilder.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/HTTPRequestBuilder.scala @@ -1,7 +1,8 @@ -package org.enso.runtimeversionmanager.http +package org.enso.downloader.http import akka.http.scaladsl.model.HttpHeader.ParsingResult import akka.http.scaladsl.model._ +import org.enso.downloader.http /** A simple immutable builder for HTTP requests. * @@ -39,7 +40,9 @@ case class HTTPRequestBuilder private ( ) } } - HTTPRequest(HttpRequest(method = method, uri = uri, headers = httpHeaders)) + http.HTTPRequest( + HttpRequest(method = method, uri = uri, headers = httpHeaders) + ) } } @@ -54,5 +57,5 @@ object HTTPRequestBuilder { * builder that will send the request to the given `uri`. */ def fromURIString(uri: String): HTTPRequestBuilder = - fromURI(Uri.parseAbsolute(uri)) + fromURI(URIBuilder.fromUri(uri).build()) } diff --git a/lib/scala/downloader/src/main/scala/org/enso/downloader/http/Header.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/Header.scala new file mode 100644 index 000000000000..bc60d9c1052d --- /dev/null +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/Header.scala @@ -0,0 +1,12 @@ +package org.enso.downloader.http + +/** Represents a HTTP header. */ +case class Header(name: String, value: String) { + + /** Checks if this header instance corresponds to a `headerName`. + * + * The check is case-insensitive. + */ + def is(headerName: String): Boolean = + name.toLowerCase == headerName.toLowerCase +} diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/URIBuilder.scala b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/URIBuilder.scala similarity index 78% rename from lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/URIBuilder.scala rename to lib/scala/downloader/src/main/scala/org/enso/downloader/http/URIBuilder.scala index 1f2f3ab81c57..82f7d1a95a22 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/URIBuilder.scala +++ b/lib/scala/downloader/src/main/scala/org/enso/downloader/http/URIBuilder.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.http +package org.enso.downloader.http import akka.http.scaladsl.model.Uri @@ -6,9 +6,6 @@ import akka.http.scaladsl.model.Uri * * It contains very limited functionality that is needed by the APIs used in * the launcher. It can be easily extended if necessary. - * - * As all APIs we use support HTTPS, it does not allow to create a non-HTTPS - * URL. */ case class URIBuilder private (uri: Uri) { @@ -43,6 +40,18 @@ object URIBuilder { def fromHost(host: String): URIBuilder = new URIBuilder(Uri.from(scheme = "https", host = host)) + /** Creates a builder from an arbitrary [[Uri]] instance. */ + def fromUri(uri: Uri): URIBuilder = + new URIBuilder(uri) + + /** Creates a builder from an arbitrary URI represented as string. + * + * If the string is invalid, it throws + * [[akka.http.scaladsl.model.IllegalUriException]]. + */ + def fromUri(uri: String): URIBuilder = + new URIBuilder(Uri.parseAbsolute(uri)) + /** A simple DSL for the URIBuilder. */ implicit class URIBuilderSyntax(builder: URIBuilder) { diff --git a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/archive/POSIXPermissionsSpec.scala b/lib/scala/downloader/src/test/scala/org/enso/downloader/archive/POSIXPermissionsSpec.scala similarity index 91% rename from lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/archive/POSIXPermissionsSpec.scala rename to lib/scala/downloader/src/test/scala/org/enso/downloader/archive/POSIXPermissionsSpec.scala index ea2daa488786..8ba829587dd4 100644 --- a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/archive/POSIXPermissionsSpec.scala +++ b/lib/scala/downloader/src/test/scala/org/enso/downloader/archive/POSIXPermissionsSpec.scala @@ -1,7 +1,8 @@ -package org.enso.runtimeversionmanager.archive +package org.enso.downloader.archive -import java.nio.file.attribute.PosixFilePermissions +import org.enso.downloader.archive.POSIXPermissions +import java.nio.file.attribute.PosixFilePermissions import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/http/HTTPDownloadSpec.scala b/lib/scala/downloader/src/test/scala/org/enso/downloader/http/HTTPDownloadSpec.scala similarity index 92% rename from lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/http/HTTPDownloadSpec.scala rename to lib/scala/downloader/src/test/scala/org/enso/downloader/http/HTTPDownloadSpec.scala index 3cbea72525f8..d88c19c37d78 100644 --- a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/http/HTTPDownloadSpec.scala +++ b/lib/scala/downloader/src/test/scala/org/enso/downloader/http/HTTPDownloadSpec.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.http +package org.enso.downloader.http import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala b/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala new file mode 100644 index 000000000000..d380895ba64e --- /dev/null +++ b/lib/scala/edition-updater/src/main/scala/org/enso/editions/updater/EditionUpdater.scala @@ -0,0 +1,82 @@ +package org.enso.editions.updater + +import com.typesafe.scalalogging.Logger +import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} +import org.enso.editions.EditionName +import org.enso.editions.repository.Manifest +import org.enso.yaml.YamlHelper + +import java.nio.file.{Files, Path} +import scala.util.{Failure, Try} + +/** A helper class that handles updating available editions. + * + * It downloads lists of available editions from all sources and downloads any + * missing (thus new) editions. Any editions that are already cached are not + * re-downloaded, as we assume that, once published, the edition is immutable. + * + * If two sources provide an edition with the same name, the one that is first + * on the sources list will take precedence. + * + * @param cachePath the path to the directory that contains the cached editions + * and where the new editions will be downloaded to + * @param sources the list of URLs indicating roots of edition repositories + * that should be queried for new editions + */ +class EditionUpdater(cachePath: Path, sources: Seq[String]) { + private lazy val logger = Logger[EditionUpdater] + + /** Downloads edition lists from the [[sources]] and downloads any missing + * editions to the [[cachePath]]. + * + * If there are errors when processing one of the edition sources or + * downloading the editions, the errors are logged as warnings, but other + * sources proceed as normal. + */ + def updateEditions(): Try[Unit] = Try { + for { + source <- sources + repositoryRoot <- Try { URIBuilder.fromUri(source) } + .recoverWith { error => + logger.warn(s"Failed to parse the source URI [$source]: $error") + Failure(error) + } + manifest <- downloadEditionRepositoryManifest(repositoryRoot) + .recoverWith { error => + logger.warn(s"Failed to fetch editions from [$source]: $error") + Failure(error) + } + edition <- manifest.editions + if !isEditionAlreadyCached(edition) + } { + downloadEdition(repositoryRoot, edition).getOrElse { + logger.warn(s"Failed to download edition [$edition] from [$source].") + } + } + } + + private def downloadEditionRepositoryManifest( + repositoryRoot: URIBuilder + ): Try[Manifest] = + Try { + val uri = repositoryRoot.addPathSegment(Manifest.filename).build() + val request = HTTPRequestBuilder.fromURI(uri).GET + val response = HTTPDownload.fetchString(request).force() + YamlHelper.parseString[Manifest](response.content).toTry.get + } + + private def downloadEdition( + repositoryRoot: URIBuilder, + editionName: EditionName + ): Try[Unit] = Try { + val destinationPath = cachePath.resolve(editionName.toFileName) + + val uri = repositoryRoot.addPathSegment(editionName.toFileName).build() + val request = HTTPRequestBuilder.fromURI(uri).GET + + HTTPDownload.download(request, destinationPath).force() + } + + private def isEditionAlreadyCached(editionName: EditionName): Boolean = + Files.exists(cachePath.resolve(editionName.toFileName)) +} diff --git a/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala b/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala index 6e50577f5a30..757763b6fafa 100644 --- a/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala +++ b/lib/scala/editions/src/main/scala/org/enso/editions/EditionName.scala @@ -8,7 +8,11 @@ import io.circe.Decoder * unquoted inside of a YAML file, that is treated as a floating point * number, so special care must be taken to correctly parse it. */ -case class EditionName(name: String) extends AnyVal +case class EditionName(name: String) extends AnyVal { + + /** Returns the name of the file that is associated with the edition name. */ + def toFileName: String = name + EditionName.editionSuffix +} object EditionName { @@ -25,4 +29,18 @@ object EditionName { .orElse(json.as[Float].map(_.toString)) .map(EditionName(_)) } + + /** The filename suffix that is used to create a filename corresponding to a + * named edition. + */ + val editionSuffix = ".yaml" + + /** Creates an [[EditionName]] from the corresponding filename. + * + * Returns None if the filename does not correspond to an edition. + */ + def fromFilename(filename: String): Option[EditionName] = + if (filename.endsWith(editionSuffix)) + Some(EditionName(filename.stripSuffix(editionSuffix))) + else None } diff --git a/lib/scala/editions/src/main/scala/org/enso/editions/provider/FileSystemEditionProvider.scala b/lib/scala/editions/src/main/scala/org/enso/editions/provider/FileSystemEditionProvider.scala index bfd96b8a1418..46e7e2c5fcae 100644 --- a/lib/scala/editions/src/main/scala/org/enso/editions/provider/FileSystemEditionProvider.scala +++ b/lib/scala/editions/src/main/scala/org/enso/editions/provider/FileSystemEditionProvider.scala @@ -1,6 +1,6 @@ package org.enso.editions.provider -import org.enso.editions.{EditionSerialization, Editions} +import org.enso.editions.{EditionName, EditionSerialization, Editions} import java.io.FileNotFoundException import java.nio.file.{Files, Path} @@ -26,8 +26,6 @@ class FileSystemEditionProvider(searchPaths: List[Path]) } } - private val editionSuffix = ".yaml" - @tailrec private def findEdition( name: String, @@ -52,7 +50,7 @@ class FileSystemEditionProvider(searchPaths: List[Path]) name: String, path: Path ): Either[EditionLoadingError, Editions.Raw.Edition] = { - val fileName = name + editionSuffix + val fileName = EditionName(name).toFileName val editionPath = path.resolve(fileName) if (Files.exists(editionPath)) { EditionSerialization @@ -67,12 +65,8 @@ class FileSystemEditionProvider(searchPaths: List[Path]) def findAvailableEditions(): Seq[String] = searchPaths.flatMap(findEditionsAt).distinct - private def findEditionName(path: Path): Option[String] = { - val name = path.getFileName.toString - if (name.endsWith(editionSuffix)) { - Some(name.stripSuffix(editionSuffix)) - } else None - } + private def findEditionName(path: Path): Option[String] = + EditionName.fromFilename(path.getFileName.toString).map(_.name) private def findEditionsAt(path: Path): Seq[String] = listDir(path).filter(Files.isRegularFile(_)).flatMap(findEditionName) diff --git a/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala b/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala index b154be295d38..5fa69170616c 100644 --- a/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala +++ b/lib/scala/editions/src/main/scala/org/enso/editions/repository/Manifest.scala @@ -19,4 +19,9 @@ object Manifest { editions <- json.get[Seq[EditionName]](Fields.editions) } yield Manifest(editions) } + + /** The name of the manifest file that should be present at the root of + * editions repository. + */ + val filename = "manifest.yaml" } diff --git a/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala b/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala index 7c1e1fa52875..7d17994ccbcc 100644 --- a/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala +++ b/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala @@ -16,7 +16,7 @@ class EditionResolverSpec with Inside with OptionValues { object FakeEditionProvider extends EditionProvider { - val mainRepo = Repository.make("main", "https://example.com/main").get + val mainRepo = Repository("main", "https://example.com/main") val editions: Map[String, Editions.RawEdition] = Map( "2021.0" -> Editions.Raw.Edition( parent = None, @@ -60,7 +60,7 @@ class EditionResolverSpec "EditionResolver" should { "resolve a simple edition" in { - val repo = Repository.make("foo", "http://example.com").get + val repo = Repository("foo", "http://example.com") val edition = Editions.Raw.Edition( parent = None, engineVersion = Some(SemVer(1, 2, 3)), @@ -127,7 +127,7 @@ class EditionResolverSpec } "correctly handle repository shadowing when resolving libraries" in { - val localRepo = Repository.make("main", "http://example.com/local").get + val localRepo = Repository("main", "http://example.com/local") localRepo should not equal FakeEditionProvider.mainRepo diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala index fbb0fb2b3dbc..9761eba42827 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala @@ -1,11 +1,17 @@ package org.enso.librarymanager import com.typesafe.scalalogging.Logger -import org.enso.distribution.{DistributionManager, LanguageHome} +import org.enso.cli.task.ProgressReporter +import org.enso.distribution.locking.{LockUserInterface, ResourceManager} +import org.enso.distribution.{ + DistributionManager, + LanguageHome, + TemporaryDirectoryManager +} import org.enso.editions.{Editions, LibraryName, LibraryVersion} import org.enso.librarymanager.local.DefaultLocalLibraryProvider import org.enso.librarymanager.published.bundles.LocalReadOnlyRepository -import org.enso.librarymanager.published.cache.NoOpCache +import org.enso.librarymanager.published.cache.DownloadingLibraryCache import org.enso.librarymanager.published.{ DefaultPublishedLibraryProvider, PublishedLibraryProvider @@ -17,12 +23,20 @@ import java.nio.file.Path /** A helper class for loading libraries. * * @param distributionManager a distribution manager + * @param resourceManager a resource manager + * @param lockUserInterface an interface that will handle notifications + * about waiting on locks + * @param progressReporter an interface that will handle progress + * notifications * @param languageHome a language home which may contain bundled libraries * @param edition the edition used in the project * @param preferLocalLibraries project setting whether to use local libraries */ class DefaultLibraryProvider( distributionManager: DistributionManager, + resourceManager: ResourceManager, + lockUserInterface: LockUserInterface, + progressReporter: ProgressReporter, languageHome: Option[LanguageHome], edition: Editions.ResolvedEdition, preferLocalLibraries: Boolean @@ -36,8 +50,14 @@ class DefaultLibraryProvider( private val resolver = LibraryResolver(localLibraryProvider) - // TODO [RW] actual cache that can download libraries will be implemented in #1772 - private val primaryCache = new NoOpCache + private val cacheRoot = distributionManager.paths.cachedLibraries + private val primaryCache = new DownloadingLibraryCache( + cacheRoot, + TemporaryDirectoryManager(distributionManager, resourceManager), + resourceManager, + lockUserInterface, + progressReporter + ) private val additionalCacheLocations = { val engineBundleRoot = languageHome.map(_.libraries) val locations = @@ -57,7 +77,7 @@ class DefaultLibraryProvider( s"Local library search paths = ${localLibrarySearchPaths.map(mask)}" ) logger.trace( - s"Primary library cache = Not implemented" + s"Primary library cache = ${mask(cacheRoot)}" ) logger.trace( s"Auxiliary (bundled) library caches = " + @@ -95,19 +115,8 @@ class DefaultLibraryProvider( } case Right(version @ LibraryVersion.Published(semver, repository)) => - val dependencyResolver = { name: LibraryName => - resolver - .resolveLibraryVersion(name, edition, preferLocalLibraries) - .toOption - } - publishedLibraryProvider - .findLibrary( - libraryName, - semver, - repository, - dependencyResolver - ) + .findLibrary(libraryName, semver, repository) .map(ResolvedLibrary(libraryName, version, _)) .toEither .left diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala index ba8bd0ab3b78..2d5a603c7d2a 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala @@ -2,7 +2,7 @@ package org.enso.librarymanager.published import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer -import org.enso.editions.{Editions, LibraryName, LibraryVersion} +import org.enso.editions.{Editions, LibraryName} import org.enso.librarymanager.published.cache.{ LibraryCache, ReadOnlyLibraryCache @@ -42,8 +42,7 @@ class DefaultPublishedLibraryProvider( override def findLibrary( libraryName: LibraryName, version: SemVer, - recommendedRepository: Editions.Repository, - dependencyResolver: LibraryName => Option[LibraryVersion] + recommendedRepository: Editions.Repository ): Try[Path] = { val cached = findCached(libraryName, version, caches) cached.map(Success(_)).getOrElse { @@ -51,12 +50,8 @@ class DefaultPublishedLibraryProvider( s"$libraryName was not found in any caches, it will need to be " + s"downloaded." ) - primaryCache.findOrInstallLibrary( - libraryName, - version, - recommendedRepository, - dependencyResolver - ) + primaryCache + .findOrInstallLibrary(libraryName, version, recommendedRepository) } } } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala index 3b730c6ff057..c224116f111e 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryProvider.scala @@ -2,7 +2,7 @@ package org.enso.librarymanager.published import nl.gn0s1s.bump.SemVer import org.enso.editions.Editions.Repository -import org.enso.editions.{LibraryName, LibraryVersion} +import org.enso.editions.LibraryName import java.nio.file.Path import scala.util.Try @@ -23,7 +23,6 @@ trait PublishedLibraryProvider { def findLibrary( libraryName: LibraryName, version: SemVer, - recommendedRepository: Repository, - dependencyResolver: LibraryName => Option[LibraryVersion] + recommendedRepository: Repository ): Try[Path] } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala new file mode 100644 index 000000000000..75c04d088834 --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -0,0 +1,337 @@ +package org.enso.librarymanager.published.cache + +import com.typesafe.scalalogging.Logger +import nl.gn0s1s.bump.SemVer +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.distribution.FileSystem.PathSyntax +import org.enso.distribution.locking.{ + LockType, + LockUserInterface, + ResourceManager +} +import org.enso.distribution.{FileSystem, TemporaryDirectoryManager} +import org.enso.downloader.archive.Archive +import org.enso.downloader.http.ResourceNotFound +import org.enso.editions.{Editions, LibraryName, LibraryVersion} +import org.enso.librarymanager.published.repository.LibraryManifest +import org.enso.librarymanager.published.repository.RepositoryHelper.{ + LibraryAccess, + RepositoryMethods +} +import org.enso.logger.masking.MaskedPath +import org.enso.pkg.PackageManager + +import java.nio.file.{Files, Path} +import scala.util.control.NonFatal +import scala.util.{Success, Try} + +/** A [[LibraryCache]] that will try to download missing libraries. + * + * @param cacheRoot the root of the library cache + * @param temporaryDirectoryManager a local temporary directory used to store + * intermediate files during installation + * @param resourceManager the resource manager instance + * @param lockUserInterface an interface that will handle notifications + * about waiting on locks + * @param progressReporter an interface that will handle progress + * notifications + */ +class DownloadingLibraryCache( + cacheRoot: Path, + temporaryDirectoryManager: TemporaryDirectoryManager, + resourceManager: ResourceManager, + lockUserInterface: LockUserInterface, + progressReporter: ProgressReporter +) extends LibraryCache { + private val logger = Logger[DownloadingLibraryCache] + + /** @inheritdoc */ + override def findCachedLibrary( + libraryName: LibraryName, + version: SemVer + ): Option[Path] = { + val path = LibraryCache.resolvePath(cacheRoot, libraryName, version) + resourceManager.withResource( + lockUserInterface, + LibraryResource(libraryName, version), + LockType.Shared + ) { + if (Files.isDirectory(path)) { + logger.trace( + s"Library [$libraryName:$version] found cached at " + + s"[${MaskedPath(path).applyMasking()}]." + ) + Some(path) + } else None + } + } + + /** @inheritdoc */ + override def findOrInstallLibrary( + libraryName: LibraryName, + version: SemVer, + recommendedRepository: Editions.Repository + ): Try[Path] = { + val _ = progressReporter // TODO + val cached = findCachedLibrary(libraryName, version) + cached match { + case Some(result) => Success(result) + case None => + installLibrary(libraryName, version, recommendedRepository) + } + } + + private def installLibrary( + libraryName: LibraryName, + version: SemVer, + recommendedRepository: Editions.Repository + ): Try[Path] = Try { + logger.trace(s"Trying to install [$libraryName:$version].") + resourceManager.withResource( + lockUserInterface, + LibraryResource(libraryName, version), + LockType.Shared + ) { + val cachedLibraryPath = + LibraryCache.resolvePath(cacheRoot, libraryName, version) + if (Files.exists(cachedLibraryPath)) { + logger.info( + s"Another process has just installed [$libraryName:$version]." + ) + cachedLibraryPath + } else { + val access = recommendedRepository.accessLibrary(libraryName, version) + val manifest = downloadManifest(libraryName, access) + + // See [Temporary Directories for Installation] + val localTmpDir = temporaryDirectoryManager.temporarySubdirectory( + s"$libraryName-$version" + ) + + try { + downloadLooseFiles(libraryName, version, access, localTmpDir) + downloadAndExtractArchives(libraryName, access, manifest, localTmpDir) + verifyPackageIntegrity(localTmpDir) + + FileSystem.atomicMove( + source = localTmpDir, + destination = cachedLibraryPath + ) + + cachedLibraryPath + } catch { + case NonFatal(exception) => + logger.error( + s"Installation of library [$libraryName:$version] failed with " + + s"error: [$exception].", + exception + ) + FileSystem.removeDirectoryIfExists(localTmpDir) + throw exception + } + } + } + } + + /** Downloads and parses the library manifest. */ + private def downloadManifest( + libraryName: LibraryName, + access: LibraryAccess + ): LibraryManifest = { + val manifestDownload = access.downloadManifest() + progressReporter.trackProgress( + s"Downloading library manifest of [$libraryName].", + manifestDownload + ) + manifestDownload.force() + } + + /** Verifies that the downloaded package can even be loaded. + * + * For now it only checks if the `package.yaml` file is not corrupted. + * + * In the future, additional checks, like checksums, could be added. + */ + private def verifyPackageIntegrity(packageRoot: Path): Unit = + PackageManager.Default.loadPackage(packageRoot.toFile).get + + /** Downloads the package config and license file. + * + * If the license file does not exist, a warning is issued, but the + * installation proceeds. However if it fails to download for other reasons, + * the installation fails in the same way as it would for any other file. + */ + private def downloadLooseFiles( + libraryName: LibraryName, + version: SemVer, + access: LibraryAccess, + localTmpDir: Path + ): Unit = { + val pkgDownload = access.downloadPackageConfig(localTmpDir) + progressReporter.trackProgress( + s"Downloading package file of [$libraryName].", + pkgDownload + ) + pkgDownload.force() + + val licenseDownload = access.downloadLicense(localTmpDir) + progressReporter.trackProgress( + s"Downloading license of [$libraryName].", + licenseDownload + ) + TaskProgress + .waitForTask(licenseDownload) + .recoverWith { case ResourceNotFound() => + // TODO [RW] Once warnings are reported to the IDE (#1860), + // inform that the license file is missing. + logger.warn( + s"License file for library [$libraryName:$version] was missing." + ) + Success(()) + } + .get + } + + /** Downloads relevant library sub-archvies and extracts them to the library + * root. + * + * All archives are assumed to be gzipped TAR archives. + */ + private def downloadAndExtractArchives( + libraryName: LibraryName, + access: LibraryAccess, + manifest: LibraryManifest, + destinationDirectory: Path + ): Unit = FileSystem.withTemporaryDirectory(s"enso-$libraryName") { + // See [Temporary Directories for Installation] + globalTmpDir => + for (archiveName <- manifest.archives) { + if (shouldDownloadArchive(archiveName)) { + val tmpArchivePath = globalTmpDir / archiveName + + val download = access.downloadArchive(archiveName, tmpArchivePath) + progressReporter.trackProgress( + s"Downloading [$archiveName] of [$libraryName].", + download + ) + download.force() + + val extraction = + Archive.extractArchive(tmpArchivePath, destinationDirectory, None) + progressReporter.trackProgress( + s"Extracting [$archiveName] of [$libraryName].", + extraction + ) + extraction.force() + + Files.delete(tmpArchivePath) + } else { + logger.info( + s"Sub-package [$archiveName] of [$libraryName] is " + + s"skipped, as it is optional." + ) + } + } + } + + /** Checks if a given sub-archive should be downloaded. + * + * Currently all archives, apart from the ones starting with `tests`, are + * downloaded. + */ + private def shouldDownloadArchive(archiveName: String): Boolean = { + val isTestData = archiveName.startsWith("tests") + !isTestData + } + + /** @inheritdoc */ + override def preinstallLibrary( + libraryName: LibraryName, + version: SemVer, + recommendedRepository: Editions.Repository, + dependencyResolver: LibraryName => Option[LibraryVersion] + ): Try[Unit] = { + logger.warn("Predownloading dependencies is not yet implemented.") + // TODO [RW] until fully fledged dependency preinstall is implemented, it + // just preinstalls the library itself; if the library has any + // dependencies, they will be downloaded by the compiler + val _ = dependencyResolver + findOrInstallLibrary(libraryName, version, recommendedRepository) + .map(_ => ()) + } +} + +/* Note [Library Cache Concurrency Model] + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * The library cache may be used by multiple instances of the engine and other + * tools running concurrently and so it needs to handle concurrent access + * scenarios. + * + * Currently our tools do not provide a way to uninstall libraries from the + * cache, which will simplify the logic significantly, in the future it may be + * extended to allow for clearing the cache. (Currently the user can just + * manually clean the cache directory when no instances are running.) + * + * Thanks to the mentioned assumption, once a library is present in the cache, + * we can assume that it will not disappear, so we do not need to synchronize + * read access (after checking that the library does indeed exist). What needs + * to be synchronized is installing libraries - to make sure that if two + * processes try to install the same library, only one of them actually performs + * the action. We also need to be sure that when one process checks if the + * library exists, and if another process is in the middle of installing it, it + * will not yet report it as existing (as this could lead to loading an + * only-partially installed library). The primary way of ensuring that will be + * to install the libraries to a temporary cache directory next to the true + * cache and atomically move it at the end of the operation. However as we do + * not have real guarantees that the filesystem move is atomic (although most of + * the time it should be if it is within a single filesystem), we will use + * locking to ensure consistency. + * + * Obviously, every client that tries to install a library will acquire a write + * lock for it, so that only one client is actually installing; but also every + * client checking for the existence of the library will briefly acquire a read + * lock for that library - thus if the library is currently being installed, the + * client will need to wait for this read lock until the library installation is + * finished and so will never encounter it in a 'partially installed' state. If + * the library is not installed, the client can release the read lock and + * re-acquire the write lock to try to install it. After acquiring the write + * lock, it should check again if the library is available, to make sure that no + * other process installed it in the meantime. This solution is efficient + * because every library is locked independently and read locks can be acquired + * by multiple clients at the same time, so the synchronization overhead for + * already installed libraries is negligible. + * + * A single LockManager (and its locks directory) should be associated with at + * most one library cache directory, as it makes sense for the distribution to + * have only one cache, so the lock entries are not disambiguated in any way. + * + * Additional note: currently, in the cloud, the library cache is stored in the + * user's workspace, so only a single Language Server will be running at the + * same time; the locking mechanism is still needed to ensure library + * installations of the compiler and preinstalls by user's request do not + * conflict. The same lock manager can be used in the cloud environment, the + * lock files are stored in a local, transient directory of the server. + */ + +/* Note [Temporary Directories for Installation] + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * When installing libraries (however the system is very similar for engines and + * runtimes too), we extract the files to a temporary directory, to minimize the + * risk of another process seeing a library in an invalid semi-installed state. + * To achieve that, we extract the archives into a temporary directory that + * resides next to the actual libraries directory, to ensure that they are on + * the same disk partition - this way, after the extraction is complete, the + * move from the temporary location to the final location is likely to be + * atomic. (However even if the move is not atomic, everything should be + * correct, as we also use file locks.) + * + * The temporary directory that is next to the destination directory is called + * local temporary directory. + * + * However we also need a place to download the archives too, and as this place + * does not necessarily need to be on the same partition as the destination, we + * can use the default system-provided temporary directory. This one is called + * the global temporary directory. The benefit of using it is that it is usually + * automatically cleaned by the OS, so we do not need to be as careful about + * cleanup in failure scenarios as with the local temporary directory. + */ diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala index a993e7fb18a5..ef28655277f8 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryCache.scala @@ -11,7 +11,7 @@ import scala.util.Try */ trait LibraryCache extends ReadOnlyLibraryCache { - /** Returns the path to the library it is already cached. + /** Returns the path to the library if it is already cached. * * This method should not attempt to download the library if it is missing, * because other providers may have it. @@ -31,23 +31,39 @@ trait LibraryCache extends ReadOnlyLibraryCache { /** If the cache contains the library, it is returned immediately, otherwise, * it tries to download the missing library. * + * It does not need to install the library's dependencies. However once the + * library is being compiled, installation of its dependencies will be + * triggered automatically by the compiler. + * * @param libraryName the name of the library to search for * @param version the library version * @param recommendedRepository the repository that should be used to * download the library from, if it is missing - * @param dependencyResolver a function that will specify what versions of - * dependencies should be also downloaded when - * installing the missing library (if any) - * TODO [RW] the design of this function should be refined in #1772 * @return the path to the library or a failure if the library could not be * installed */ def findOrInstallLibrary( + libraryName: LibraryName, + version: SemVer, + recommendedRepository: Editions.Repository + ): Try[Path] + + /** Ensures that the given library and all of its dependencies are installed. + * + * @param libraryName the name of the library to search for + * @param version the library version + * @param recommendedRepository the repository that should be used to + * download the library from, if it is missing + * @param dependencyResolver a function that will specify what versions of + * dependencies should be also downloaded when + * installing the missing library (if any) + */ + def preinstallLibrary( libraryName: LibraryName, version: SemVer, recommendedRepository: Editions.Repository, dependencyResolver: LibraryName => Option[LibraryVersion] - ): Try[Path] + ): Try[Unit] } object LibraryCache { diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala new file mode 100644 index 000000000000..0a8c2a257c32 --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/LibraryResource.scala @@ -0,0 +1,15 @@ +package org.enso.librarymanager.published.cache + +import nl.gn0s1s.bump.SemVer +import org.enso.distribution.locking.Resource +import org.enso.editions.LibraryName + +/** A resource that synchronizes installation of a library in the cache. */ +case class LibraryResource(libraryName: LibraryName, version: SemVer) + extends Resource { + override def name: String = + s"cached-library-${libraryName.qualifiedName}-$version" + override def waitMessage: String = + s"Another Enso instance is currently installing $libraryName ($version), " + + s"so this action must wait until the installation is complete." +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala deleted file mode 100644 index c7625f6d1f0e..000000000000 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/NoOpCache.scala +++ /dev/null @@ -1,30 +0,0 @@ -package org.enso.librarymanager.published.cache -import nl.gn0s1s.bump.SemVer -import org.enso.editions.{Editions, LibraryName, LibraryVersion} - -import java.nio.file.Path -import scala.util.{Failure, Try} - -/** A temporary cache that provides no libraries. - * - * This is a temporary poly-fill which will later be replaced when the - * downloading mechanism is implemented. - */ -class NoOpCache extends LibraryCache { - - /** @inheritdoc */ - override def findCachedLibrary( - libraryName: LibraryName, - version: SemVer - ): Option[Path] = None - - /** @inheritdoc */ - override def findOrInstallLibrary( - libraryName: LibraryName, - version: SemVer, - recommendedRepository: Editions.Repository, - dependencyResolver: LibraryName => Option[LibraryVersion] - ): Try[Path] = Failure( - new NotImplementedError("Downloading libraries is not yet implemented.") - ) -} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryException.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryException.scala new file mode 100644 index 000000000000..e4f8b7cdb8a1 --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryException.scala @@ -0,0 +1,17 @@ +package org.enso.librarymanager.published.repository + +import nl.gn0s1s.bump.SemVer +import org.enso.editions.LibraryName + +/** Indicates that the library could not be downloaded. */ +sealed class LibraryDownloadFailure(message: String) + extends RuntimeException(message) + +/** Indicates that the library was not found in the recommended repository. */ +case class LibraryNotFoundException( + libraryName: LibraryName, + version: SemVer, + uri: String +) extends LibraryDownloadFailure( + s"Library [$libraryName:$version] was not found at [$uri]." + ) diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala index a711fdb4826a..af2cf5229db2 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/LibraryManifest.scala @@ -42,4 +42,9 @@ object LibraryManifest { description = description ) } + + /** The name of the manifest file as included in the directory associated with + * a given library in the library repository. + */ + val filename = "manifest.yaml" } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala new file mode 100644 index 000000000000..146656f35f96 --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/repository/RepositoryHelper.scala @@ -0,0 +1,119 @@ +package org.enso.librarymanager.published.repository + +import nl.gn0s1s.bump.SemVer +import org.enso.cli.task.TaskProgress +import org.enso.distribution.FileSystem.PathSyntax +import org.enso.downloader.http.{HTTPDownload, URIBuilder} +import org.enso.editions.Editions.Repository +import org.enso.editions.LibraryName +import org.enso.pkg.Package +import org.enso.yaml.YamlHelper + +import java.nio.file.Path +import scala.util.Failure + +/** A class that manages the HTTP API of the Library Repository. + * + * @see docs/libraries/repositories.md#libraries-repository + */ +object RepositoryHelper { + + /** Adds extension methods to the [[Repository]] type. */ + implicit class RepositoryMethods(val repository: Repository) { + + /** Creates a [[LibraryAccess]] instance that aids with downloading data of + * the given library. + */ + def accessLibrary(name: LibraryName, version: SemVer): LibraryAccess = + new LibraryAccess(name, version, resolveLibraryRoot(name, version)) + + /** Creates a [[URIBuilder]] that points to the directory in the repository + * corresponding to the given library. + */ + def resolveLibraryRoot(name: LibraryName, version: SemVer): URIBuilder = + URIBuilder + .fromUri(repository.url) + .addPathSegment(name.namespace) + .addPathSegment(name.name) + .addPathSegment(version.toString) + } + + /** A helper class that allows to access the Library Repository to query it + * for metadata of a specific library or download its packages. + * + * @param libraryName name of the library + * @param version version of the library + * @param libraryRoot a [[URIBuilder]] that points to the directory + * corresponding to the library + */ + class LibraryAccess( + libraryName: LibraryName, + version: SemVer, + libraryRoot: URIBuilder + ) { + + /** Downloads and parses the manifest file. + * + * If the repository responds with 404 status code, it returns a special + * [[LibraryNotFoundException]] indicating that the repository does not + * provide that library. Any other failures are indicated with the more + * generic [[LibraryDownloadFailure]]. + */ + def downloadManifest(): TaskProgress[LibraryManifest] = { + val url = (libraryRoot / manifestFilename).build() + HTTPDownload.fetchString(url).flatMap { response => + response.statusCode match { + case 200 => + YamlHelper.parseString[LibraryManifest](response.content).toTry + case 404 => + Failure( + LibraryNotFoundException(libraryName, version, url.toString) + ) + case code => + Failure( + new LibraryDownloadFailure( + s"Could not download the manifest: The repository responded " + + s"with $code status code." + ) + ) + } + } + } + + /** A helper that downloads an artifact to a specific location. */ + private def downloadArtifact( + artifactName: String, + destination: Path + ): TaskProgress[Unit] = { + val url = (libraryRoot / artifactName).build() + HTTPDownload.download(url, destination).map(_ => ()) + } + + /** Downloads the license file. + * + * It will fail with `ResourceNotFound` error if the license did not exist + * and with a more generic `HTTPException` if it failed for other reasons. + */ + def downloadLicense(destinationDirectory: Path): TaskProgress[Unit] = + downloadArtifact(licenseFilename, destinationDirectory / licenseFilename) + + /** Downloads the package config file. */ + def downloadPackageConfig(destinationDirectory: Path): TaskProgress[Unit] = + downloadArtifact(packageFileName, destinationDirectory / packageFileName) + + /** Downloads a sub-archive. */ + def downloadArchive( + archiveName: String, + destinationDirectory: Path + ): TaskProgress[Unit] = downloadArtifact(archiveName, destinationDirectory) + } + + /** Name of the manifest file. */ + val manifestFilename = "manifest.yaml" + + /** Name of the attached license file. */ + val licenseFilename = "LICENSE.md" + + /** Name of the package config file. */ + val packageFileName: String = Package.configFileName +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/LibraryResolverSpec.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/LibraryResolverSpec.scala index 2cdc407f1d83..567f1711de1d 100644 --- a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/LibraryResolverSpec.scala +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/LibraryResolverSpec.scala @@ -17,7 +17,7 @@ class LibraryResolverSpec with EitherValue with Inside { "LibraryResolver" should { - val mainRepo = Repository.make("main", "https://example.com/main").get + val mainRepo = Repository("main", "https://example.com/main") val parentEdition = Editions.Resolved.Edition( parent = None, engineVersion = Some(SemVer(0, 0, 0)), @@ -31,7 +31,7 @@ class LibraryResolverSpec ) ) ) - val customRepo = Repository.make("custom", "https://example.com/custom").get + val customRepo = Repository("custom", "https://example.com/custom") val currentEdition = Editions.Resolved.Edition( parent = Some(parentEdition), engineVersion = None, diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala new file mode 100644 index 000000000000..023f1b89a8e2 --- /dev/null +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ArchiveWriter.scala @@ -0,0 +1,50 @@ +package org.enso.librarymanager.published.repository + +import org.apache.commons.compress.archivers.tar.{ + TarArchiveEntry, + TarArchiveOutputStream +} +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream + +import java.io.{BufferedOutputStream, FileOutputStream} +import java.nio.file.Path +import scala.util.Using + +/** A helper class used for creating TAR-GZ archives in tests. */ +object ArchiveWriter { + + /** A file to add to the archive. */ + sealed trait FileToWrite { + + /** The path that this file should have within the archive. */ + def relativePath: String + } + + /** Represents a text file to be added to a test archive. + * + * @param relativePath the path in the archive + * @param content the text contents for the file + */ + case class TextFile(relativePath: String, content: String) extends FileToWrite + + /** Creates a tar archive at the given path, containing the provided files. */ + def writeTarArchive(path: Path, files: Seq[FileToWrite]): Unit = { + Using(new FileOutputStream(path.toFile)) { outputStream => + Using(new BufferedOutputStream(outputStream)) { bufferedStream => + Using(new GzipCompressorOutputStream(bufferedStream)) { gzipStream => + Using(new TarArchiveOutputStream(gzipStream)) { archive => + for (file <- files) { + file match { + case TextFile(relativePath, content) => + val entry = new TarArchiveEntry(relativePath) + archive.putArchiveEntry(entry) + archive.write(content.getBytes) + archive.closeArchiveEntry() + } + } + } + } + } + } + } +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala new file mode 100644 index 000000000000..3c163855ec8a --- /dev/null +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -0,0 +1,155 @@ +package org.enso.librarymanager.published.repository + +import nl.gn0s1s.bump.SemVer +import org.enso.cli.OS +import org.enso.distribution.FileSystem +import org.enso.editions.Editions.RawEdition +import org.enso.editions.{Editions, LibraryName} +import org.enso.pkg.{Package, PackageManager} +import org.enso.testkit.process.WrappedProcess + +import java.io.File +import java.nio.file.{Files, Path} +import scala.util.control.NonFatal + +/** A helper class managing a library repository for testing purposes. */ +abstract class DummyRepository { + + /** A library used for testing. + * + * @param libraryName name of the library + * @param version version of the library + * @param mainContent contents of the `Main.enso` file + */ + case class DummyLibrary( + libraryName: LibraryName, + version: SemVer, + mainContent: String + ) + + /** Name of the repository, as it will be indicated in the generated edition. + */ + def repoName: String = "test_repo" + + /** Sequence of libraries to create in the repository and include in the + * edition. + */ + def libraries: Seq[DummyLibrary] + + /** Creates a directory structure for the repository at the given root and + * populates it with [[libraries]]. + */ + def createRepository(root: Path): Unit = { + for (lib <- libraries) { + val libraryRoot = root + .resolve("libraries") + .resolve(lib.libraryName.namespace) + .resolve(lib.libraryName.name) + .resolve(lib.version.toString) + Files.createDirectories(libraryRoot) + createLibraryProject(libraryRoot, lib) + val files = Seq( + ArchiveWriter.TextFile("src/Main.enso", lib.mainContent) + ) + ArchiveWriter.writeTarArchive(libraryRoot.resolve("main.tgz"), files) + createManifest(libraryRoot) + } + } + + private def createLibraryProject( + path: Path, + lib: DummyLibrary + ): Package[File] = { + val pkg = PackageManager.Default.create( + path.toFile, + name = lib.libraryName.name, + namespace = lib.libraryName.namespace, + version = lib.version.toString() + ) + pkg.save().get + pkg + } + + private def createManifest(path: Path): Unit = { + FileSystem.writeTextFile( + path.resolve("manifest.yaml"), + s"""archives: + | - main.tgz + |""".stripMargin + ) + } + + /** Creates an edition which contains libraries defined in this repository. + * + * @param repoUrl the URL where the repository is going to be accessible; the + * URL should include the `libraries` prefix + */ + def createEdition(repoUrl: String): RawEdition = { + Editions.Raw.Edition( + parent = Some(buildinfo.Info.currentEdition), + repositories = Map(repoName -> Editions.Repository(repoName, repoUrl)), + libraries = Map.from(libraries.map { lib => + lib.libraryName -> Editions.Raw + .PublishedLibrary(lib.libraryName, lib.version, repoName) + }) + ) + } + + private def commandPrefix: Seq[String] = + if (OS.isWindows) Seq("cmd.exe", "/c") else Seq.empty + + private def npmCommand: String = if (OS.isWindows) "npm.cmd" else "npm" + private def nodeCommand: String = if (OS.isWindows) "node.exe" else "node" + + /** Starts a server for the library repository. + * + * @param port port to listen on + * @param root root of the library repository, the same as the argument to + * [[createRepository]] + */ + def startServer(port: Int, root: Path): WrappedProcess = { + val serverDirectory = + Path.of("tools/simple-library-server").toAbsolutePath.normalize + + val preinstallCommand = commandPrefix ++ Seq(npmCommand, "install") + val preinstallExitCode = new ProcessBuilder() + .command(preinstallCommand: _*) + .directory(serverDirectory.toFile) + .inheritIO() + .start() + .waitFor() + + if (preinstallExitCode != 0) + throw new RuntimeException( + s"Failed to preinstall the Library Repository Server dependencies: " + + s"npm exited with code $preinstallCommand." + ) + + val command = commandPrefix ++ Seq( + nodeCommand, + "main.js", + "--port", + port.toString, + "--root", + root.toAbsolutePath.normalize.toString + ) + val rawProcess = (new ProcessBuilder) + .command(command: _*) + .directory(serverDirectory.toFile) + .start() + val process = new WrappedProcess(command, rawProcess) + try { + process.printIO() + process.waitForMessage( + "Serving the repository", + timeoutSeconds = 15, + process.StdOut + ) + } catch { + case NonFatal(e) => + process.kill() + throw e + } + process + } +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala new file mode 100644 index 000000000000..dbd2f11b9232 --- /dev/null +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/ExampleRepository.scala @@ -0,0 +1,23 @@ +package org.enso.librarymanager.published.repository + +import nl.gn0s1s.bump.SemVer +import org.enso.editions.LibraryName + +/** A simple [[DummyRepository]] containing a single library for testing + * downloads. + */ +class ExampleRepository extends DummyRepository { + + /** The library provided by this repository. */ + val testLib: DummyLibrary = DummyLibrary( + LibraryName("Foo", "Bar"), + SemVer(1, 0, 0), + """baz = 42 + | + |quux = "foobar" + |""".stripMargin + ) + + /** @inheritdoc */ + override def libraries: Seq[DummyLibrary] = Seq(testLib) +} diff --git a/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala new file mode 100644 index 000000000000..882516116dab --- /dev/null +++ b/lib/scala/library-manager/src/test/scala/org/enso/librarymanager/published/repository/LibraryDownloadTest.scala @@ -0,0 +1,104 @@ +package org.enso.librarymanager.published.repository + +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.distribution.TemporaryDirectoryManager +import org.enso.distribution.locking.{ + LockUserInterface, + Resource, + ResourceManager, + ThreadSafeFileLockManager +} +import org.enso.editions.Editions +import org.enso.librarymanager.published.cache.DownloadingLibraryCache +import org.enso.loggingservice.TestLogger.TestLogMessage +import org.enso.loggingservice.{LogLevel, TestLogger} +import org.enso.pkg.PackageManager +import org.enso.testkit.WithTemporaryDirectory +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import java.nio.file.Files + +class LibraryDownloadTest + extends AnyWordSpec + with Matchers + with WithTemporaryDirectory { + + val port: Int = 47306 + + "DownloadingLibraryCache" should { + "be able to download and install libraries from a repository" in { + val repo = new ExampleRepository + + val repoRoot = getTestDirectory.resolve("repo") + repo.createRepository(repoRoot) + val lockManager = + new ThreadSafeFileLockManager(getTestDirectory.resolve("locks")) + val resourceManager = new ResourceManager(lockManager) + try { + val cache = new DownloadingLibraryCache( + cacheRoot = getTestDirectory.resolve("cache"), + temporaryDirectoryManager = new TemporaryDirectoryManager( + getTestDirectory.resolve("tmp"), + resourceManager + ), + resourceManager = resourceManager, + lockUserInterface = new LockUserInterface { + override def startWaitingForResource(resource: Resource): Unit = + println(s"Waiting for ${resource.name}") + + override def finishWaitingForResource(resource: Resource): Unit = + println(s"${resource.name} is ready") + }, + progressReporter = new ProgressReporter { + override def trackProgress( + message: String, + task: TaskProgress[_] + ): Unit = {} + } + ) + + val server = repo.startServer(port, repoRoot) + try { + cache.findCachedLibrary( + repo.testLib.libraryName, + repo.testLib.version + ) shouldBe empty + + val (libPath, logs) = TestLogger.gatherLogs { + cache + .findOrInstallLibrary( + repo.testLib.libraryName, + repo.testLib.version, + Editions + .Repository("test_repo", s"http://localhost:$port/libraries") + ) + .get + } + val pkg = PackageManager.Default.loadPackage(libPath.toFile).get + pkg.name shouldEqual "Bar" + val sources = pkg.listSources + sources should have size 1 + sources.head.file.getName shouldEqual "Main.enso" + assert( + Files.notExists(libPath.resolve("LICENSE.md")), + "The license file should not exist as it was not provided " + + "in the repository." + ) + logs should contain( + TestLogMessage( + LogLevel.Warning, + "License file for library [Foo.Bar:1.0.0] was missing." + ) + ) + } finally { + server.kill(killDescendants = true) + server.join(waitForDescendants = true) + } + } finally { + resourceManager.releaseMainLock() + resourceManager.unlockTemporaryDirectory() + } + } + } +} diff --git a/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala b/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala index e05b8ad27dfe..6bb6026575f4 100644 --- a/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala +++ b/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala @@ -21,7 +21,7 @@ object TestLogger { * be ran with `parallelExecution` set to false, as global logger state has * to be modified to gather the logs. */ - def gatherLogs(action: => Unit): Seq[TestLogMessage] = { + def gatherLogs[R](action: => R): (R, Seq[TestLogMessage]) = { LoggingServiceManager.dropPendingLogs() if (LoggingServiceManager.isSetUp()) { throw new IllegalStateException( @@ -35,10 +35,10 @@ object TestLogger { LogLevel.Trace ) Await.ready(future, 1.second) - action + val result = action Thread.sleep(100) LoggingServiceManager.tearDown() - printer.getLoggedMessages + (result, printer.getLoggedMessages) } /** Drops any logs that are pending due to the logging service not being set diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/ControllerInterface.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/ControllerInterface.scala index 808c4c27d97f..09467e5285b4 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/ControllerInterface.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/ControllerInterface.scala @@ -3,34 +3,28 @@ package org.enso.projectmanager.service.versionmanagement import akka.actor.ActorRef import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer -import org.enso.cli.task.{ - ProgressNotification, - ProgressNotificationForwarder, - ProgressUnit -} -import org.enso.distribution.locking.Resource +import org.enso.cli.task.ProgressNotification +import org.enso.distribution.ProgressAndLockNotificationForwarder import org.enso.runtimeversionmanager.components.{ GraalVMVersion, RuntimeVersionManagementUserInterface } -import java.util.UUID - /** A [[RuntimeVersionManagementUserInterface]] that sends * [[ProgressNotification]] to the specified actors (both for usual tasks and * indeterminate progress when waiting on locks). * - * @param progressTracker the actor to send progress updates to + * @param progressTracker the actor to send progress updates to * @param allowMissingComponents specifies if missing components should be * automatically installed - * @param allowBrokenComponents specifies if broken components can be installed + * @param allowBrokenComponents specifies if broken components can be installed */ class ControllerInterface( progressTracker: ActorRef, allowMissingComponents: Boolean, allowBrokenComponents: Boolean -) extends RuntimeVersionManagementUserInterface - with ProgressNotificationForwarder { +) extends ProgressAndLockNotificationForwarder + with RuntimeVersionManagementUserInterface { /** @inheritdoc */ override def shouldInstallMissingEngine(version: SemVer): Boolean = @@ -48,32 +42,6 @@ class ControllerInterface( override def logInfo(message: => String): Unit = Logger[ControllerInterface].info(message) - private val waitingForResources = - collection.concurrent.TrieMap[String, UUID]() - - /** @inheritdoc */ - override def startWaitingForResource(resource: Resource): Unit = { - val uuid = UUID.randomUUID() - progressTracker ! ProgressNotification.TaskStarted( - uuid, - None, - ProgressUnit.Unspecified - ) - progressTracker ! ProgressNotification.TaskUpdate( - uuid, - Some(resource.waitMessage), - 0 - ) - waitingForResources.put(resource.name, uuid) - } - - /** @inheritdoc */ - override def finishWaitingForResource(resource: Resource): Unit = { - for (uuid <- waitingForResources.remove(resource.name)) { - progressTracker ! ProgressNotification.TaskSuccess(uuid) - } - } - /** @inheritdoc */ override def sendProgressNotification( notification: ProgressNotification diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala index 832ce6b7e4c9..bc1a52f56391 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala @@ -1,7 +1,12 @@ package org.enso.projectmanager.versionmanagement import com.typesafe.scalalogging.LazyLogging -import org.enso.distribution.{DistributionManager, EditionManager, Environment} +import org.enso.distribution.{ + DistributionManager, + EditionManager, + Environment, + TemporaryDirectoryManager +} import org.enso.distribution.locking.{ ResourceManager, ThreadSafeFileLockManager @@ -14,7 +19,6 @@ import org.enso.runtimeversionmanager.components.{ RuntimeVersionManagementUserInterface, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.ReleaseProvider import org.enso.runtimeversionmanager.releases.engine.{ EngineRelease, @@ -53,7 +57,7 @@ object DefaultDistributionConfiguration /** @inheritdoc */ lazy val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) lazy val componentConfiguration: RuntimeComponentConfiguration = new GraalVMComponentConfiguration diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DistributionConfiguration.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DistributionConfiguration.scala index 970b21f236a3..cb8dac791d80 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DistributionConfiguration.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DistributionConfiguration.scala @@ -1,12 +1,16 @@ package org.enso.projectmanager.versionmanagement import org.enso.distribution.locking.ResourceManager -import org.enso.distribution.{DistributionManager, EditionManager, Environment} +import org.enso.distribution.{ + DistributionManager, + EditionManager, + Environment, + TemporaryDirectoryManager +} import org.enso.runtimeversionmanager.components.{ RuntimeVersionManagementUserInterface, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.ReleaseProvider import org.enso.runtimeversionmanager.releases.engine.EngineRelease import org.enso.runtimeversionmanager.runner.JVMSettings diff --git a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/BaseServerSpec.scala b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/BaseServerSpec.scala index 4dd99519e59a..c769ea0531ad 100644 --- a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/BaseServerSpec.scala +++ b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/BaseServerSpec.scala @@ -11,8 +11,9 @@ import io.circe.parser.parse import nl.gn0s1s.bump.SemVer import org.apache.commons.io.FileUtils import org.enso.distribution.FileSystem.PathSyntax -import org.enso.distribution.{FileSystem, OS} +import org.enso.distribution.FileSystem import org.enso.editions.Editions +import org.enso.cli.OS import org.enso.jsonrpc.test.JsonRpcServerTestKit import org.enso.jsonrpc.{ClientControllerFactory, Protocol} import org.enso.loggingservice.printers.StderrPrinterWithColors diff --git a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala index 24564a78d081..bbb021c956d0 100644 --- a/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/test/scala/org/enso/projectmanager/TestDistributionConfiguration.scala @@ -1,6 +1,10 @@ package org.enso.projectmanager -import org.enso.distribution.{DistributionManager, EditionManager} +import org.enso.distribution.{ + DistributionManager, + EditionManager, + TemporaryDirectoryManager +} import org.enso.distribution.locking.ResourceManager import java.nio.file.Path @@ -11,7 +15,6 @@ import org.enso.runtimeversionmanager.components.{ RuntimeVersionManagementUserInterface, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.engine.{ EngineRelease, EngineReleaseProvider @@ -28,10 +31,10 @@ import org.enso.runtimeversionmanager.releases.{ import org.enso.runtimeversionmanager.runner.{JVMSettings, JavaCommand} import org.enso.runtimeversionmanager.test.{ FakeEnvironment, - HasTestDirectory, NoopComponentUpdaterFactory, TestLocalLockManager } +import org.enso.testkit.HasTestDirectory import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} @@ -67,7 +70,7 @@ class TestDistributionConfiguration( lazy val editionManager: EditionManager = EditionManager(distributionManager) lazy val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) lazy val componentConfig = new GraalVMComponentConfiguration diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/FakeEnvironment.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/FakeEnvironment.scala index f410ca17ab89..1525e03c3758 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/FakeEnvironment.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/FakeEnvironment.scala @@ -1,6 +1,7 @@ package org.enso.runtimeversionmanager.test import org.enso.distribution.{Environment, FileSystem} +import org.enso.testkit.HasTestDirectory import java.nio.file.{Files, Path} diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala index 242aea79c10d..5588e3054591 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/NativeTestHelper.scala @@ -1,20 +1,10 @@ package org.enso.runtimeversionmanager.test -import org.enso.distribution.OS +import org.enso.cli.OS +import org.enso.testkit.process.{RunResult, WrappedProcess} -import java.io.{ - BufferedReader, - IOException, - InputStream, - InputStreamReader, - PrintWriter -} -import java.util.concurrent.{Semaphore, TimeUnit} import java.lang.{ProcessBuilder => JProcessBuilder} -import scala.collection.Factory -import scala.concurrent.TimeoutException import scala.jdk.CollectionConverters._ -import scala.jdk.StreamConverters._ /** A mix-in providing helper functions for running native commands in tests. * @@ -23,14 +13,6 @@ import scala.jdk.StreamConverters._ */ trait NativeTestHelper { - /** A result of running the native launcher binary. - * - * @param exitCode the returned exit code - * @param stdout contents of the standard output stream - * @param stderr contents of the standard error stream - */ - case class RunResult(exitCode: Int, stdout: String, stderr: String) - /** Starts the provided `command`. * * `extraEnv` may be provided to extend the environment. Care must be taken @@ -80,172 +62,6 @@ trait NativeTestHelper { } } - /** Represents a started and possibly running process. */ - class WrappedProcess(command: Seq[String], process: Process) { - - private val outQueue = - new java.util.concurrent.LinkedTransferQueue[String]() - private val errQueue = - new java.util.concurrent.LinkedTransferQueue[String]() - - sealed trait StreamType - case object StdErr extends StreamType - case object StdOut extends StreamType - @volatile private var ioHandlers: Seq[(String, StreamType) => Unit] = Seq() - - private def watchStream( - stream: InputStream, - streamType: StreamType - ): Unit = { - val reader = new BufferedReader(new InputStreamReader(stream)) - var line: String = null - val queue = streamType match { - case StdErr => errQueue - case StdOut => outQueue - } - try { - while ({ line = reader.readLine(); line != null }) { - queue.add(line) - ioHandlers.foreach(f => f(line, streamType)) - } - } catch { - case _: InterruptedException => - case _: IOException => - ioHandlers.foreach(f => f("", streamType)) - } - } - - private val outThread = new Thread(() => - watchStream(process.getInputStream, StdOut) - ) - private val errThread = new Thread(() => - watchStream(process.getErrorStream, StdErr) - ) - outThread.start() - errThread.start() - - /** Waits for a message on the stderr to appear. */ - def waitForMessageOnErrorStream( - message: String, - timeoutSeconds: Long - ): Unit = { - val semaphore = new Semaphore(0) - def handler(line: String, streamType: StreamType): Unit = { - if (streamType == StdErr && line.contains(message)) { - semaphore.release() - } - } - - this.synchronized { - ioHandlers ++= Seq(handler _) - } - - errQueue.asScala.toSeq.foreach(handler(_, StdErr)) - - val acquired = semaphore.tryAcquire(timeoutSeconds, TimeUnit.SECONDS) - if (!acquired) { - throw new TimeoutException(s"Waiting for `$message` timed out.") - } - } - - private lazy val inputWriter = new PrintWriter(process.getOutputStream) - - /** Prints a message to the standard input stream of the process. - * - * Does not append any newlines, so if a newline is expected, it has to be - * contained within the `message`. - */ - def sendToInputStream(message: String): Unit = { - inputWriter.print(message) - inputWriter.flush() - } - - /** Starts printing the stdout and stderr of the started process to the - * stdout with prefixes to indicate that these messages come from another - * process. - * - * It also prints lines that were printed before invoking this method. - * Thus, it is possible that a line may be printed twice (once as - * 'before-printIO' and once normally). - */ - def printIO(): Unit = { - def handler(line: String, streamType: StreamType): Unit = { - val prefix = streamType match { - case StdErr => "stderr> " - case StdOut => "stdout> " - } - println(prefix + line) - } - this.synchronized { - ioHandlers ++= Seq(handler _) - } - outQueue.asScala.toSeq.foreach(line => - println(s"stdout-before-printIO> $line") - ) - errQueue.asScala.toSeq.foreach(line => - println(s"stderr-before-printIO> $line") - ) - } - - /** Checks if the process is still running. */ - def isAlive: Boolean = process.isAlive - - /** Tries to kill the process immediately. */ - def kill(): Unit = { - process.destroyForcibly() - } - - /** Waits for the process to finish and returns its [[RunResult]]. - * - * If `waitForDescendants` is set, tries to wait for descendants of the - * launched process to finish too. Especially important on Windows where - * child processes may run after the launcher parent has been terminated. - * - * It will timeout after `timeoutSeconds` and try to kill the process (or - * its descendants), although it may not always be able to. - */ - def join( - waitForDescendants: Boolean = true, - timeoutSeconds: Long = 15 - ): RunResult = { - var descendants: Seq[ProcessHandle] = Seq() - try { - val exitCode = - if (process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) - process.exitValue() - else throw new TimeoutException("Process timed out") - if (waitForDescendants) { - descendants = - process.descendants().toScala(Factory.arrayFactory).toSeq - descendants.foreach(_.onExit().get(timeoutSeconds, TimeUnit.SECONDS)) - } - errThread.join(1000) - outThread.join(1000) - if (errThread.isAlive) { - errThread.interrupt() - } - if (outThread.isAlive) { - outThread.interrupt() - } - val stdout = outQueue.asScala.toSeq.mkString("\n") - val stderr = errQueue.asScala.toSeq.mkString("\n") - RunResult(exitCode, stdout, stderr) - } catch { - case e @ (_: InterruptedException | _: TimeoutException) => - if (process.isAlive) { - println(s"Killing the timed-out process: ${command.mkString(" ")}") - process.destroyForcibly() - } - for (processHandle <- descendants) { - if (processHandle.isAlive) { - processHandle.destroyForcibly() - } - } - throw e - } - } - } - /** Runs the provided `command`. * * `extraEnv` may be provided to extend the environment. Care must be taken diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala index f22e491f17d3..25a8214fa5f6 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/RuntimeVersionManagerTest.scala @@ -4,7 +4,8 @@ import nl.gn0s1s.bump.SemVer import org.enso.distribution.{ DistributionManager, Environment, - PortableDistributionManager + PortableDistributionManager, + TemporaryDirectoryManager } import org.enso.pkg.{Config, PackageManager} import org.enso.runtimeversionmanager.components.{ @@ -13,9 +14,9 @@ import org.enso.runtimeversionmanager.components.{ RuntimeVersionManagementUserInterface, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.engine.EngineReleaseProvider import org.enso.runtimeversionmanager.releases.graalvm.GraalVMRuntimeReleaseProvider +import org.enso.testkit.WithTemporaryDirectory import org.scalatest.OptionValues import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -53,7 +54,7 @@ class RuntimeVersionManagerTest val resourceManager = TestLocalResourceManager.create() val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) val componentConfig = new GraalVMComponentConfiguration val runtimeVersionManager = new RuntimeVersionManager( diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala index 2bb651b48f65..65d19ecf1876 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ConcurrencyTest.scala @@ -3,7 +3,11 @@ package org.enso.distribution.locking import java.nio.file.{Files, Path} import nl.gn0s1s.bump.SemVer import org.enso.cli.task.TaskProgress -import org.enso.distribution.{DistributionManager, FileSystem} +import org.enso.distribution.{ + DistributionManager, + FileSystem, + TemporaryDirectoryManager +} import org.enso.distribution.locking.{ LockManager, LockType, @@ -19,7 +23,6 @@ import org.enso.runtimeversionmanager.components.{ Manifest, RuntimeVersionManager } -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager import org.enso.runtimeversionmanager.releases.engine.{ EngineRelease, EngineReleaseProvider @@ -27,7 +30,7 @@ import org.enso.runtimeversionmanager.releases.engine.{ import org.enso.runtimeversionmanager.releases.graalvm.GraalCEReleaseProvider import org.enso.runtimeversionmanager.releases.testing.FakeReleaseProvider import org.enso.runtimeversionmanager.test._ -import org.enso.testkit.{FlakySpec, RetrySpec} +import org.enso.testkit.{FlakySpec, RetrySpec, WithTemporaryDirectory} import org.scalatest.BeforeAndAfterEach import org.scalatest.concurrent.TimeLimitedTests import org.scalatest.matchers.should.Matchers @@ -149,7 +152,7 @@ class ConcurrencyTest } val temporaryDirectoryManager = - new TemporaryDirectoryManager(distributionManager, resourceManager) + TemporaryDirectoryManager(distributionManager, resourceManager) val componentConfig = new GraalVMComponentConfiguration val componentsManager = new RuntimeVersionManager( TestRuntimeVersionManagementUserInterface.default, diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ThreadSafeFileLockManagerTest.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ThreadSafeFileLockManagerTest.scala index f525c472a03a..9703ecda60f9 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ThreadSafeFileLockManagerTest.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/distribution/locking/ThreadSafeFileLockManagerTest.scala @@ -8,11 +8,8 @@ import org.enso.distribution.locking.{ } import java.nio.file.Path -import org.enso.runtimeversionmanager.test.{ - NativeTestHelper, - TestSynchronizer, - WithTemporaryDirectory -} +import org.enso.runtimeversionmanager.test.{NativeTestHelper, TestSynchronizer} +import org.enso.testkit.WithTemporaryDirectory import org.scalatest.OptionValues import org.scalatest.concurrent.TimeLimitedTests import org.scalatest.matchers.should.Matchers diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala index 1ca9a80801bd..4fb6561663bd 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala @@ -2,7 +2,8 @@ package org.enso.runtimeversionmanager.components import java.nio.file.{Files, Path} import nl.gn0s1s.bump.SemVer -import org.enso.distribution.{FileSystem, OS} +import org.enso.cli.OS +import org.enso.distribution.FileSystem import org.enso.distribution.FileSystem.PathSyntax import org.enso.runtimeversionmanager.config.GlobalConfigurationManager import org.enso.runtimeversionmanager.releases.ReleaseNotFound diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/config/GlobalConfigurationManagerSpec.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/config/GlobalConfigurationManagerSpec.scala index 6c20131af97d..5389a74e4337 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/config/GlobalConfigurationManagerSpec.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/config/GlobalConfigurationManagerSpec.scala @@ -3,10 +3,8 @@ package org.enso.runtimeversionmanager.config import io.circe.Json import nl.gn0s1s.bump.SemVer import org.enso.distribution.DistributionManager -import org.enso.runtimeversionmanager.test.{ - FakeEnvironment, - WithTemporaryDirectory -} +import org.enso.runtimeversionmanager.test.FakeEnvironment +import org.enso.testkit.WithTemporaryDirectory import org.scalatest.OptionValues import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala index 420bc999ea26..e219d05a27a1 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala @@ -9,10 +9,8 @@ import org.enso.distribution.{ import java.nio.file.{Files, Path} import org.enso.distribution.FileSystem.PathSyntax -import org.enso.runtimeversionmanager.test.{ - FakeEnvironment, - WithTemporaryDirectory -} +import org.enso.runtimeversionmanager.test.FakeEnvironment +import org.enso.testkit.WithTemporaryDirectory import org.scalatest.OptionValues import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala index a9a40ed67e37..77ba54d4c82c 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.components -import org.enso.distribution.OS +import org.enso.cli.OS import java.nio.file.{Files, Path} import org.enso.distribution.FileSystem.PathSyntax diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfiguration.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfiguration.scala index d656f34730ce..808e012cde16 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfiguration.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfiguration.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.components -import org.enso.distribution.OS +import org.enso.cli.OS /** Component configuration of the GraalVM distribution. */ class GraalVMComponentConfiguration extends RuntimeComponentConfiguration { diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Manifest.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Manifest.scala index 171e7171fa6a..478a9803a553 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Manifest.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Manifest.scala @@ -5,7 +5,7 @@ import java.nio.file.Path import cats.Show import io.circe.{yaml, Decoder} import nl.gn0s1s.bump.SemVer -import org.enso.distribution.OS +import org.enso.cli.OS import org.enso.editions.SemVerJson._ import org.enso.runtimeversionmanager.components.Manifest.{ JVMOption, diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeComponentConfiguration.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeComponentConfiguration.scala index a7533d0f2d43..f57cd56edb5a 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeComponentConfiguration.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeComponentConfiguration.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.components -import org.enso.distribution.OS +import org.enso.cli.OS /** Provides configuration of the runtime components. */ trait RuntimeComponentConfiguration { diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala index b3a8d8404928..4b8fd6667310 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala @@ -3,13 +3,17 @@ package org.enso.runtimeversionmanager.components import java.nio.file.{Files, Path, StandardOpenOption} import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer -import org.enso.distribution.{DistributionManager, FileSystem, OS} +import org.enso.cli.OS +import org.enso.distribution.{ + DistributionManager, + FileSystem, + TemporaryDirectoryManager +} import org.enso.distribution.locking.{LockType, ResourceManager} import org.enso.runtimeversionmanager.CurrentVersion import org.enso.distribution.FileSystem.PathSyntax import org.enso.logger.masking.MaskedPath -import org.enso.runtimeversionmanager.archive.Archive -import org.enso.runtimeversionmanager.distribution.TemporaryDirectoryManager +import org.enso.downloader.archive.Archive import org.enso.runtimeversionmanager.locking.Resources import org.enso.runtimeversionmanager.releases.ReleaseProvider import org.enso.runtimeversionmanager.releases.engine.EngineRelease @@ -429,9 +433,9 @@ class RuntimeVersionManager( ) } } - FileSystem.withTemporaryDirectory("enso-install") { directory => - logger.debug("Downloading packages to [{}].", directory) - val enginePackage = directory / engineRelease.packageFileName + FileSystem.withTemporaryDirectory("enso-install") { globalTmpDirectory => + logger.debug("Downloading packages to [{}].", globalTmpDirectory) + val enginePackage = globalTmpDirectory / engineRelease.packageFileName val downloadTask = engineRelease.downloadPackage(enginePackage) userInterface.trackProgress( s"Downloading ${enginePackage.getFileName}.", @@ -442,18 +446,19 @@ class RuntimeVersionManager( val engineDirectoryName = engineDirectoryNameForVersion(engineRelease.version) + val localTmpDirectory = + temporaryDirectoryManager.temporarySubdirectory(s"engine-$version") + val extractionTask = Archive .extractArchive( enginePackage, - temporaryDirectoryManager.accessTemporaryDirectory(), + localTmpDirectory, Some(engineDirectoryName) ) userInterface.trackProgress("Extracting the engine.", extractionTask) extractionTask.force() - val engineTemporaryPath = - temporaryDirectoryManager - .accessTemporaryDirectory() / engineDirectoryName + val engineTemporaryPath = localTmpDirectory / engineDirectoryName def undoTemporaryEngine(): Unit = { if (Files.exists(engineTemporaryPath)) { FileSystem.removeDirectory(engineTemporaryPath) @@ -694,19 +699,21 @@ class RuntimeVersionManager( downloadTask.force() val runtimeDirectoryName = graalDirectoryForVersion(runtimeVersion) + val localTmpDirectory = + temporaryDirectoryManager.temporarySubdirectory( + s"runtime-${runtimeVersion.graalVersion}-java${runtimeVersion.java}" + ) val extractionTask = Archive.extractArchive( runtimePackage, - temporaryDirectoryManager.accessTemporaryDirectory(), + localTmpDirectory, Some(runtimeDirectoryName) ) logger.debug("Extracting [{}].", runtimePackage) userInterface.trackProgress("Extracting the runtime.", extractionTask) extractionTask.force() - val runtimeTemporaryPath = - temporaryDirectoryManager - .accessTemporaryDirectory() / runtimeDirectoryName + val runtimeTemporaryPath = localTmpDirectory / runtimeDirectoryName def undoTemporaryRuntime(): Unit = { if (Files.exists(runtimeTemporaryPath)) { @@ -842,8 +849,8 @@ class RuntimeVersionManager( */ private def safelyRemoveComponent(path: Path): Unit = { val temporaryPath = - temporaryDirectoryManager.accessTemporaryDirectory() / path.getFileName - FileSystem.atomicMove(path, temporaryPath) + temporaryDirectoryManager.temporarySubdirectory(path.getFileName.toString) + FileSystem.atomicMove(path, temporaryPath / "tmp") FileSystem.removeDirectory(temporaryPath) } diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/TemporaryDirectoryManager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/TemporaryDirectoryManager.scala deleted file mode 100644 index f399e2e3260e..000000000000 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/TemporaryDirectoryManager.scala +++ /dev/null @@ -1,62 +0,0 @@ -package org.enso.runtimeversionmanager.distribution - -import java.nio.file.{Files, Path} -import com.typesafe.scalalogging.Logger -import org.enso.distribution.{DistributionManager, FileSystem} -import org.enso.distribution.locking.ResourceManager - -/** Manages safe access to the temporary directory. - * - * The temporary directory is created on demand and automatically removed if it - * is empty. Temporary files from previous runs are removed when the temporary - * directory is first accessed. Locking mechanism is used to ensure that the - * old files are no longer used by any other instances running in parallel. - */ -class TemporaryDirectoryManager( - distribution: DistributionManager, - resourceManager: ResourceManager -) { - private val logger = Logger[TemporaryDirectoryManager] - - /** Returns path to a directory for storing temporary files that is located on - * the same filesystem as `runtimes` and `engines`. - * - * It is used during installation to decrease the possibility of getting a - * broken installation if the installation process has been abruptly - * terminated. The directory is created on demand (when its path is requested - * for the first time) and is removed if the application exits normally (as - * long as it is empty, but normal termination of the installation process - * should ensure that). If that fails, it is also cleaned before any future - * accesses. - */ - def accessTemporaryDirectory(): Path = safeTemporaryDirectory - - private lazy val safeTemporaryDirectory = { - resourceManager.startUsingTemporaryDirectory() - distribution.paths.unsafeTemporaryDirectory - } - - /** Tries to clean the temporary files directory. - * - * It should be run at startup whenever the program wants to run clean-up. - * Currently it is run when installation-related operations are taking place. - * It may not proceed if another process is using it. It has to be run before - * the first access to the temporaryDirectory, as after that the directory is - * marked as in-use and will not be cleaned. - */ - def tryCleaningTemporaryDirectory(): Unit = { - val tmp = distribution.paths.unsafeTemporaryDirectory - if (Files.exists(tmp)) { - resourceManager.tryWithExclusiveTemporaryDirectory { - if (!FileSystem.isDirectoryEmpty(tmp)) { - logger.info( - "Cleaning up temporary files from a previous installation." - ) - } - FileSystem.removeDirectory(tmp) - Files.createDirectories(tmp) - FileSystem.removeEmptyDirectoryOnExit(tmp) - } - } - } -} diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPException.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPException.scala deleted file mode 100644 index a66cf24c05af..000000000000 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/http/HTTPException.scala +++ /dev/null @@ -1,5 +0,0 @@ -package org.enso.runtimeversionmanager.http - -/** Indicates an error when processing a HTTP request. - */ -case class HTTPException(message: String) extends RuntimeException(message) diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/locking/Resources.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/locking/Resources.scala index 7574f476cd26..f8e2ee977271 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/locking/Resources.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/locking/Resources.scala @@ -6,8 +6,7 @@ import org.enso.runtimeversionmanager.components.GraalVMVersion object Resources { - /** Synchronizes launcher upgrades. - */ + /** Synchronizes launcher upgrades. */ case object LauncherExecutable extends Resource { override def name: String = "launcher-executable" override def waitMessage: String = @@ -15,8 +14,7 @@ object Resources { "the current process must wait until it is completed." } - /** This resource is held when adding or removing any components. - */ + /** This resource is held when adding or removing any components. */ case object AddOrRemoveComponents extends Resource { override def name: String = "add-remove-components" override def waitMessage: String = diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/EnsoReleaseProvider.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/EnsoReleaseProvider.scala index e9c18dc4b43b..94e8c71cf148 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/EnsoReleaseProvider.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/EnsoReleaseProvider.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.releases import nl.gn0s1s.bump.SemVer -import org.enso.distribution.OS +import org.enso.cli.OS import scala.util.{Failure, Success, Try} diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala index 94ed977a83bf..2d4babd9dfff 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala @@ -4,7 +4,7 @@ import java.nio.file.Path import io.circe._ import io.circe.parser._ import org.enso.cli.task.TaskProgress -import org.enso.runtimeversionmanager.http.{ +import org.enso.downloader.http.{ APIResponse, HTTPDownload, HTTPRequestBuilder, diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala index 15a5d149e8ed..5dc403bc2175 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala @@ -1,8 +1,9 @@ package org.enso.runtimeversionmanager.releases.graalvm +import org.enso.cli.OS + import java.nio.file.Path import org.enso.cli.task.TaskProgress -import org.enso.distribution.OS import org.enso.runtimeversionmanager.components.GraalVMVersion import org.enso.runtimeversionmanager.releases.github.GithubReleaseProvider import org.enso.runtimeversionmanager.releases.{ diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/TestArchivePackager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/TestArchivePackager.scala index c873c868f1df..d9678421c179 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/TestArchivePackager.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/TestArchivePackager.scala @@ -1,6 +1,7 @@ package org.enso.runtimeversionmanager.releases.testing -import org.enso.distribution.{FileSystem, OS} +import org.enso.cli.OS +import org.enso.distribution.FileSystem import java.nio.file.Path import scala.sys.process.Process diff --git a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfigurationSpec.scala b/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfigurationSpec.scala index a835db55d170..1fca23b0cf45 100644 --- a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfigurationSpec.scala +++ b/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentConfigurationSpec.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.components -import org.enso.distribution.OS +import org.enso.cli.OS import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala b/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala index 9bd888568492..2b54d5109be8 100644 --- a/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala +++ b/lib/scala/runtime-version-manager/src/test/scala/org/enso/runtimeversionmanager/components/GraalVMComponentUpdaterSpec.scala @@ -1,6 +1,6 @@ package org.enso.runtimeversionmanager.components -import org.enso.distribution.OS +import org.enso.cli.OS import java.nio.file.Path import org.scalatest.matchers.should.Matchers diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/HasTestDirectory.scala b/lib/scala/testkit/src/main/scala/org/enso/testkit/HasTestDirectory.scala similarity index 78% rename from lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/HasTestDirectory.scala rename to lib/scala/testkit/src/main/scala/org/enso/testkit/HasTestDirectory.scala index 44f999d093b7..8d72c7fd7d59 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/HasTestDirectory.scala +++ b/lib/scala/testkit/src/main/scala/org/enso/testkit/HasTestDirectory.scala @@ -1,4 +1,4 @@ -package org.enso.runtimeversionmanager.test +package org.enso.testkit import java.nio.file.Path diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/WithTemporaryDirectory.scala b/lib/scala/testkit/src/main/scala/org/enso/testkit/WithTemporaryDirectory.scala similarity index 97% rename from lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/WithTemporaryDirectory.scala rename to lib/scala/testkit/src/main/scala/org/enso/testkit/WithTemporaryDirectory.scala index c4cae32d3608..0b5406502553 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/WithTemporaryDirectory.scala +++ b/lib/scala/testkit/src/main/scala/org/enso/testkit/WithTemporaryDirectory.scala @@ -1,11 +1,11 @@ -package org.enso.runtimeversionmanager.test - -import java.io.{File, IOException} -import java.nio.file.{Files, Path} +package org.enso.testkit import org.apache.commons.io.FileUtils import org.scalatest.{BeforeAndAfterEach, Suite} +import java.io.{File, IOException} +import java.nio.file.{Files, Path} + /** Creates a separate temporary directory for each test. */ trait WithTemporaryDirectory diff --git a/lib/scala/testkit/src/main/scala/org/enso/testkit/process/RunResult.scala b/lib/scala/testkit/src/main/scala/org/enso/testkit/process/RunResult.scala new file mode 100644 index 000000000000..b100246b56ae --- /dev/null +++ b/lib/scala/testkit/src/main/scala/org/enso/testkit/process/RunResult.scala @@ -0,0 +1,9 @@ +package org.enso.testkit.process + +/** A result of running a process. + * + * @param exitCode the returned exit code + * @param stdout contents of the standard output stream + * @param stderr contents of the standard error stream + */ +case class RunResult(exitCode: Int, stdout: String, stderr: String) diff --git a/lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala b/lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala new file mode 100644 index 000000000000..4c83475b54c4 --- /dev/null +++ b/lib/scala/testkit/src/main/scala/org/enso/testkit/process/WrappedProcess.scala @@ -0,0 +1,193 @@ +package org.enso.testkit.process + +import java.io._ +import java.util.concurrent.{Semaphore, TimeUnit} +import scala.collection.Factory +import scala.concurrent.TimeoutException +import scala.jdk.CollectionConverters._ +import scala.jdk.StreamConverters._ + +/** Represents a started and possibly running process. */ +class WrappedProcess(command: Seq[String], process: Process) { + + private val outQueue = + new java.util.concurrent.LinkedTransferQueue[String]() + private val errQueue = + new java.util.concurrent.LinkedTransferQueue[String]() + + sealed trait StreamType + case object StdErr extends StreamType + case object StdOut extends StreamType + @volatile private var ioHandlers: Seq[(String, StreamType) => Unit] = Seq() + + private def watchStream( + stream: InputStream, + streamType: StreamType + ): Unit = { + val reader = new BufferedReader(new InputStreamReader(stream)) + var line: String = null + val queue = streamType match { + case StdErr => errQueue + case StdOut => outQueue + } + try { + while ({ line = reader.readLine(); line != null }) { + queue.add(line) + ioHandlers.foreach(f => f(line, streamType)) + } + } catch { + case _: InterruptedException => + case _: IOException => + ioHandlers.foreach(f => f("", streamType)) + } + } + + private val outThread = new Thread(() => + watchStream(process.getInputStream, StdOut) + ) + private val errThread = new Thread(() => + watchStream(process.getErrorStream, StdErr) + ) + outThread.start() + errThread.start() + + /** Waits for a message on the stderr to appear. */ + def waitForMessageOnErrorStream( + message: String, + timeoutSeconds: Long + ): Unit = waitForMessage(message, timeoutSeconds, StdErr) + + /** Waits for a message on one of the output streams. */ + def waitForMessage( + message: String, + timeoutSeconds: Long, + stream: StreamType + ): Unit = { + val semaphore = new Semaphore(0) + def handler(line: String, streamType: StreamType): Unit = { + if (streamType == stream && line.contains(message)) { + semaphore.release() + } + } + + this.synchronized { + ioHandlers ++= Seq(handler _) + } + + stream match { + case StdErr => + errQueue.asScala.toSeq.foreach(handler(_, StdErr)) + case StdOut => + outQueue.asScala.toSeq.foreach(handler(_, StdOut)) + } + + val acquired = semaphore.tryAcquire(timeoutSeconds, TimeUnit.SECONDS) + if (!acquired) { + throw new TimeoutException(s"Waiting for `$message` timed out.") + } + } + + private lazy val inputWriter = new PrintWriter(process.getOutputStream) + + /** Prints a message to the standard input stream of the process. + * + * Does not append any newlines, so if a newline is expected, it has to be + * contained within the `message`. + */ + def sendToInputStream(message: String): Unit = { + inputWriter.print(message) + inputWriter.flush() + } + + /** Starts printing the stdout and stderr of the started process to the + * stdout with prefixes to indicate that these messages come from another + * process. + * + * It also prints lines that were printed before invoking this method. + * Thus, it is possible that a line may be printed twice (once as + * 'before-printIO' and once normally). + */ + def printIO(): Unit = { + def handler(line: String, streamType: StreamType): Unit = { + val prefix = streamType match { + case StdErr => "stderr> " + case StdOut => "stdout> " + } + println(prefix + line) + } + this.synchronized { + ioHandlers ++= Seq(handler _) + } + outQueue.asScala.toSeq.foreach(line => + println(s"stdout-before-printIO> $line") + ) + errQueue.asScala.toSeq.foreach(line => + println(s"stderr-before-printIO> $line") + ) + } + + /** Checks if the process is still running. */ + def isAlive: Boolean = process.isAlive + + /** Tries to kill the process immediately. */ + def kill(killDescendants: Boolean = false): Unit = { + process.destroyForcibly() + if (killDescendants) { + for (processHandle <- findDescendants()) { + processHandle.destroyForcibly() + } + } + } + + private def findDescendants(): Seq[ProcessHandle] = + process.descendants().toScala(Factory.arrayFactory).toSeq + + /** Waits for the process to finish and returns its [[RunResult]]. + * + * If `waitForDescendants` is set, tries to wait for descendants of the + * launched process to finish too. Especially important on Windows where + * child processes may run after the launcher parent has been terminated. + * + * It will timeout after `timeoutSeconds` and try to kill the process (or + * its descendants), although it may not always be able to. + */ + def join( + waitForDescendants: Boolean = true, + timeoutSeconds: Long = 15 + ): RunResult = { + var descendants: Seq[ProcessHandle] = Seq() + try { + val exitCode = + if (process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) + process.exitValue() + else throw new TimeoutException("Process timed out") + if (waitForDescendants) { + descendants = findDescendants() + descendants.foreach(_.onExit().get(timeoutSeconds, TimeUnit.SECONDS)) + } + errThread.join(1000) + outThread.join(1000) + if (errThread.isAlive) { + errThread.interrupt() + } + if (outThread.isAlive) { + outThread.interrupt() + } + val stdout = outQueue.asScala.toSeq.mkString("\n") + val stderr = errQueue.asScala.toSeq.mkString("\n") + RunResult(exitCode, stdout, stderr) + } catch { + case e @ (_: InterruptedException | _: TimeoutException) => + if (process.isAlive) { + println(s"Killing the timed-out process: ${command.mkString(" ")}") + process.destroyForcibly() + } + for (processHandle <- descendants) { + if (processHandle.isAlive) { + processHandle.destroyForcibly() + } + } + throw e + } + } +} diff --git a/tools/legal-review/Table/report-state b/tools/legal-review/Table/report-state index a2c9c1222358..944ed7826a3f 100644 --- a/tools/legal-review/Table/report-state +++ b/tools/legal-review/Table/report-state @@ -1,3 +1,3 @@ B39E86F4C9F95AE99476021C49ED7121CA82A4F734D1D1E493F2A00773D753C2 -440AAA28949F11A1839DEBDB7B4BEEDC6D95E50287A2C852D68427195C1277BE +E19E5AB2117CED2FE672C2C43F47181E21DB10D59506779505DE197A85E2D6F7 0 diff --git a/tools/legal-review/engine/com.typesafe.config-1.3.2/copyright-keep-context b/tools/legal-review/engine/com.typesafe.config-1.4.0/copyright-keep-context similarity index 100% rename from tools/legal-review/engine/com.typesafe.config-1.3.2/copyright-keep-context rename to tools/legal-review/engine/com.typesafe.config-1.4.0/copyright-keep-context index 77674fa31387..6d007f558b96 100644 --- a/tools/legal-review/engine/com.typesafe.config-1.3.2/copyright-keep-context +++ b/tools/legal-review/engine/com.typesafe.config-1.4.0/copyright-keep-context @@ -1,3 +1,3 @@ Copyright (C) 2011-2012 Typesafe Inc. -Copyright (C) 2014 Typesafe Inc. Copyright (C) 2015 Typesafe Inc. +Copyright (C) 2014 Typesafe Inc. diff --git a/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-ignore b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-ignore new file mode 100644 index 000000000000..47f54b72cf57 --- /dev/null +++ b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-ignore @@ -0,0 +1,2 @@ +regarding copyright ownership. The ASF licenses this file +this work for additional information regarding copyright ownership. diff --git a/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-keep-context b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-keep-context new file mode 100644 index 000000000000..6ab681a5b3bd --- /dev/null +++ b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/copyright-keep-context @@ -0,0 +1 @@ +Some portions of this file Copyright (c) 2004-2006 Intel Corportation diff --git a/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-ignore b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-ignore new file mode 100644 index 000000000000..0256724c8d06 --- /dev/null +++ b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-ignore @@ -0,0 +1 @@ +META-INF/LICENSE.txt diff --git a/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-keep b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-keep new file mode 100644 index 000000000000..f9a3ec844f02 --- /dev/null +++ b/tools/legal-review/engine/org.apache.commons.commons-compress-1.20/files-keep @@ -0,0 +1 @@ +META-INF/NOTICE.txt diff --git a/tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.2/copyright-keep-context b/tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.2/copyright-keep-context deleted file mode 100644 index 2d1291cd5315..000000000000 --- a/tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.2/copyright-keep-context +++ /dev/null @@ -1 +0,0 @@ -this code has waived all copyright and related or neighboring * diff --git a/tools/legal-review/engine/report-state b/tools/legal-review/engine/report-state index 56d111c2ff2b..de78f167d4c4 100644 --- a/tools/legal-review/engine/report-state +++ b/tools/legal-review/engine/report-state @@ -1,3 +1,3 @@ -61713B297871FFB55B9BDF492049F90A7756BBEDF91DA31B72F0EFB98221754D -C309ED3582802FAA0EA238BB40A24F34CACC4FC4A36334C260232FF49DCD9C72 +038976FC3077212393B2513AB30E376A4E496A9A23EAF71F2CA8B4721ED512E9 +31BAA7838578F7775841BE7BDA52C0BDFDB4F59766C5FA57110D312AE6138651 0 diff --git a/tools/legal-review/launcher/report-state b/tools/legal-review/launcher/report-state index 1a22d8ada645..383a001dff56 100644 --- a/tools/legal-review/launcher/report-state +++ b/tools/legal-review/launcher/report-state @@ -1,3 +1,3 @@ -2D9E29668392299BAE4C1D0D0D7562556712E78CF8FA1022853E42C9E4C05FA1 -0F4AF9C887470D371F850DF94EF5155B3872A8537D0B753FE5ADD94171DFCC35 +56351584030E6947A34374E8672DB955C5A7077228182A1053BE489E9B076315 +28E1F445EA0515C2F3E73ABD9DDB73395B5E0F7A84D07038A1117B782E32A1C6 0 diff --git a/tools/legal-review/project-manager/report-state b/tools/legal-review/project-manager/report-state index df9d55110fee..213a01e574f1 100644 --- a/tools/legal-review/project-manager/report-state +++ b/tools/legal-review/project-manager/report-state @@ -1,3 +1,3 @@ -0AED341E22D16C5722BF722FD5F92039464E9FE3CCD3288D3643F05E09AF62FB -E4B0E3E0602F9227B8D9334913B2D903C652DADA95732D0F55E26B469E82B89C +4A89A53D6497DFF6A8B647FAD7B63844C15C22020B1AC1CC9B260C9BA0819F1F +B7948AB0E996317E7F073260BA28A63D14215436079C09E793F803CF999D607C 0 diff --git a/tools/simple-library-server/main.js b/tools/simple-library-server/main.js index 83ead01cca33..d33ce4079d0d 100755 --- a/tools/simple-library-server/main.js +++ b/tools/simple-library-server/main.js @@ -22,13 +22,14 @@ const argv = yargs .help() .alias("help", "h").argv; +const app = express(); +app.use(compression({ filter: shouldCompress })); +app.use(express.static(argv.root)); + console.log( `Serving the repository located under ${argv.root} on port ${argv.port}.` ); -const app = express(); -app.use(compression({ filter: shouldCompress })); -app.use(express.static(argv.root)); app.listen(argv.port); function shouldCompress(req, res) {