From 35e60ed402d82867c7e56c0b07bc9971a7c80856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Olczak?= Date: Tue, 19 Jan 2021 10:07:17 +0100 Subject: [PATCH] Fix for SQLite DB busy error on Azure (#1395) depending on an environmental variable is used either default locking mode or a mode that uses flock syscall --- .../languageserver/boot/DeploymentType.scala | 40 +++++++++++++++++++ .../enso/languageserver/boot/MainModule.scala | 16 +++++++- .../search/SuggestionsHandlerSpec.scala | 14 ++++--- .../websocket/json/BaseServerTest.scala | 12 +++--- .../src/main/resources/reference.conf | 1 + .../org/enso/searcher/sql/SqlDatabase.scala | 34 +++++++++------- .../enso/searcher/sqlite/LockingMode.scala | 24 +++++++++++ tools/ci/docker/Dockerfile | 11 +++-- 8 files changed, 119 insertions(+), 33 deletions(-) create mode 100644 engine/language-server/src/main/scala/org/enso/languageserver/boot/DeploymentType.scala create mode 100644 lib/scala/searcher/src/main/scala/org/enso/searcher/sqlite/LockingMode.scala diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/boot/DeploymentType.scala b/engine/language-server/src/main/scala/org/enso/languageserver/boot/DeploymentType.scala new file mode 100644 index 000000000000..d9e5541efdec --- /dev/null +++ b/engine/language-server/src/main/scala/org/enso/languageserver/boot/DeploymentType.scala @@ -0,0 +1,40 @@ +package org.enso.languageserver.boot + +/** Signal where the lang. server is deployed. + */ +sealed trait DeploymentType + +object DeploymentType { + + /** Desktop deployment. + */ + case object Desktop extends DeploymentType + + /** Azure deployment. + */ + case object Azure extends DeploymentType + + /** Determines the current deployment type from environment variables. + * @return the current deployment type + */ + def fromEnvironment(): DeploymentType = { + if (sys.env.contains(DeploymentTypeVariableName)) { + val value = sys.env(DeploymentTypeVariableName) + fromString(value) + } else { + Desktop + } + } + + /** Determines a current deployment type from a string value. + * @return a deployment type + */ + def fromString(value: String): DeploymentType = + value.toLowerCase.trim match { + case "desktop" | "" => Desktop + case "azure" => Azure + } + + private lazy val DeploymentTypeVariableName = "DEPLOYMENT_TYPE" + +} diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala b/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala index 6cb1757dcbad..b19cf5fa8199 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala @@ -5,6 +5,7 @@ import java.net.URI import akka.actor.ActorSystem import org.enso.jsonrpc.JsonRpcServer +import org.enso.languageserver.boot.DeploymentType.{Azure, Desktop} import org.enso.languageserver.capability.CapabilityRouter import org.enso.languageserver.data._ import org.enso.languageserver.effect.ZioExec @@ -34,6 +35,7 @@ import org.enso.languageserver.util.binary.BinaryEncoder import org.enso.loggingservice.{JavaLoggingLogHandler, LogLevel} import org.enso.polyglot.{LanguageInfo, RuntimeOptions, RuntimeServerInfo} import org.enso.searcher.sql.{SqlDatabase, SqlSuggestionsRepo, SqlVersionsRepo} +import org.enso.searcher.sqlite.LockingMode import org.enso.text.{ContentBasedVersioning, Sha3_224VersionCalculator} import org.graalvm.polyglot.Context import org.graalvm.polyglot.io.MessageEndpoint @@ -84,7 +86,19 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: LogLevel) { log.trace("Created ActorSystem") val sqlDatabase = - SqlDatabase(languageServerConfig.directories.suggestionsDatabaseFile) + DeploymentType.fromEnvironment() match { + case Desktop => + SqlDatabase( + languageServerConfig.directories.suggestionsDatabaseFile.toString + ) + + case Azure => + SqlDatabase( + languageServerConfig.directories.suggestionsDatabaseFile.toString, + Some(LockingMode.UnixFlock) + ) + } + val suggestionsRepo = new SqlSuggestionsRepo(sqlDatabase)(system.dispatcher) val versionsRepo = new SqlVersionsRepo(sqlDatabase)(system.dispatcher) log.trace("Created SQL Repos") diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala b/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala index 7bbe431dd26b..9cbd0a9304cf 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala @@ -19,11 +19,11 @@ import org.enso.languageserver.session.JsonSession import org.enso.languageserver.session.SessionRouter.DeliverToJsonController import org.enso.polyglot.data.Tree import org.enso.polyglot.runtime.Runtime.Api -import org.enso.searcher.{FileVersionsRepo, SuggestionsRepo} import org.enso.searcher.sql.{SqlDatabase, SqlSuggestionsRepo, SqlVersionsRepo} +import org.enso.searcher.{FileVersionsRepo, SuggestionsRepo} import org.enso.testkit.RetrySpec -import org.enso.text.{ContentVersion, Sha3_224VersionCalculator} import org.enso.text.editing.model.Position +import org.enso.text.{ContentVersion, Sha3_224VersionCalculator} import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike @@ -594,10 +594,12 @@ class SuggestionsHandlerSpec ): Unit = { val testContentRoot = Files.createTempDirectory(null).toRealPath() sys.addShutdownHook(FileUtils.deleteQuietly(testContentRoot.toFile)) - val config = newConfig(testContentRoot.toFile) - val router = TestProbe("session-router") - val connector = TestProbe("runtime-connector") - val sqlDatabase = SqlDatabase(config.directories.suggestionsDatabaseFile) + val config = newConfig(testContentRoot.toFile) + val router = TestProbe("session-router") + val connector = TestProbe("runtime-connector") + val sqlDatabase = SqlDatabase( + config.directories.suggestionsDatabaseFile.toString + ) val suggestionsRepo = new SqlSuggestionsRepo(sqlDatabase) val versionsRepo = new SqlVersionsRepo(sqlDatabase) val handler = newSuggestionsHandler( diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala index 3d8fd05bbeb8..b4cd919304a8 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala @@ -81,13 +81,13 @@ class BaseServerTest extends JsonRpcServerTestKit { InputRedirectionController.props(stdIn, stdInSink, sessionRouter) ) - override def clientControllerFactory: ClientControllerFactory = { - - val zioExec = ZioExec(zio.Runtime.default) - val sqlDatabase = SqlDatabase(config.directories.suggestionsDatabaseFile) - val suggestionsRepo = new SqlSuggestionsRepo(sqlDatabase)(system.dispatcher) - val versionsRepo = new SqlVersionsRepo(sqlDatabase)(system.dispatcher) + val zioExec = ZioExec(zio.Runtime.default) + val sqlDatabase = + SqlDatabase(config.directories.suggestionsDatabaseFile.toString) + val suggestionsRepo = new SqlSuggestionsRepo(sqlDatabase)(system.dispatcher) + val versionsRepo = new SqlVersionsRepo(sqlDatabase)(system.dispatcher) + override def clientControllerFactory: ClientControllerFactory = { val fileManager = system.actorOf(FileManager.props(config, new FileSystem, zioExec)) val bufferRegistry = diff --git a/lib/scala/searcher/src/main/resources/reference.conf b/lib/scala/searcher/src/main/resources/reference.conf index 21a3d1ec3c68..f2520be795eb 100644 --- a/lib/scala/searcher/src/main/resources/reference.conf +++ b/lib/scala/searcher/src/main/resources/reference.conf @@ -4,6 +4,7 @@ searcher { driver = "org.sqlite.JDBC" connectionPool = disabled properties.journal_mode = "wal" + properties.locking_mode = "EXCLUSIVE" numThreads = 1 } } diff --git a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlDatabase.scala b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlDatabase.scala index 3e4f08663f43..f9dc4d481459 100644 --- a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlDatabase.scala +++ b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlDatabase.scala @@ -1,9 +1,8 @@ package org.enso.searcher.sql -import java.io.File - import com.typesafe.config.{Config, ConfigFactory} import org.enso.searcher.Database +import org.enso.searcher.sqlite.LockingMode import slick.dbio.DBIO import slick.jdbc.SQLiteProfile import slick.jdbc.SQLiteProfile.api._ @@ -43,24 +42,31 @@ object SqlDatabase { * @param filename the database file path * @return new sql database instance */ - def apply(filename: File): SqlDatabase = - apply(filename.toString) - - /** Create [[SqlDatabase]] instance. - * - * @param filename the database file path - * @return new sql database instance - */ - def apply(filename: String): SqlDatabase = { + def apply( + filename: String, + maybeLockingMode: Option[LockingMode] = None + ): SqlDatabase = { val config = ConfigFactory - .parseString(s"""$configPath.url = "${jdbcUrl(filename)}"""") + .parseString( + s"""$configPath.url = "${jdbcUrl(filename, maybeLockingMode)}"""" + ) .withFallback(ConfigFactory.load()) new SqlDatabase(Some(config)) } /** Create JDBC URL from the file path. */ - private def jdbcUrl(filename: String): String = - s"jdbc:sqlite:${escapePath(filename)}" + private def jdbcUrl( + filename: String, + maybeLockingMode: Option[LockingMode] + ): String = { + maybeLockingMode match { + case None => + s"jdbc:sqlite:${escapePath(filename)}" + + case Some(lockingMode) => + s"jdbc:sqlite:file:${escapePath(filename)}?vfs=${lockingMode.name}" + } + } /** Escape Windows path. */ private def escapePath(path: String): String = diff --git a/lib/scala/searcher/src/main/scala/org/enso/searcher/sqlite/LockingMode.scala b/lib/scala/searcher/src/main/scala/org/enso/searcher/sqlite/LockingMode.scala new file mode 100644 index 000000000000..c7fbd01a9bac --- /dev/null +++ b/lib/scala/searcher/src/main/scala/org/enso/searcher/sqlite/LockingMode.scala @@ -0,0 +1,24 @@ +package org.enso.searcher.sqlite + +/** A mode used to handle file locking in SQLite */ +case class LockingMode private (name: String) + +object LockingMode { + + /** Default mode that uses POSIX advisory locks. + */ + val UnixPosix = LockingMode("unix") + + /** It obtains and holds an exclusive lock on database files, + * preventing other processes from accessing the database. + * It uses the `flock` system call. + */ + val UnixFlock = LockingMode("unix-excl") + + /** It uses dot-file locking rather than POSIX advisory locks. */ + val UnixDotFile = LockingMode("unix-dotfile") + + /** All file locking operations are no-ops. */ + val UnixNone = LockingMode("unix-none") + +} diff --git a/tools/ci/docker/Dockerfile b/tools/ci/docker/Dockerfile index 625bb786f8d7..745ba99169fa 100644 --- a/tools/ci/docker/Dockerfile +++ b/tools/ci/docker/Dockerfile @@ -11,9 +11,9 @@ ADD runtime.jar /opt/enso/runtime.jar RUN chown -hR enso:enso /opt/enso RUN chmod -R u=rX,g=rX /opt/enso -RUN mkdir -p /home/enso/workspace -RUN chown -hR enso:enso /home/enso/workspace -RUN chmod -R u=rwX,g=rwX /home/enso/workspace +RUN mkdir -p /volumes +RUN chown -hR enso:enso /volumes +RUN chmod -R u=rwX,g=rwX /volumes USER enso:enso @@ -24,7 +24,6 @@ ENTRYPOINT ["java", "-jar", "-Dtruffle.class.path.append=runtime.jar", "-Dpolyg EXPOSE 30001 EXPOSE 30002 -VOLUME /home/enso/workspace - -CMD ["--server", "--rpc-port", "30001", "--data-port", "30002", "--root-id", "00000000-0000-0000-0000-000000000001", "--path", "/home/enso/workspace", "--interface", "0.0.0.0"] +VOLUME /volumes/workspace +CMD ["--server", "--daemon", "--rpc-port", "30001", "--data-port", "30002", "--root-id", "00000000-0000-0000-0000-000000000001", "--path", "/volumes/workspace", "--interface", "0.0.0.0", "--log-level", "INFO"]