From 8cf4136625a98900d2bd303c20355c9f7d9341c7 Mon Sep 17 00:00:00 2001 From: Jan Chyb Date: Fri, 12 May 2023 10:49:35 +0200 Subject: [PATCH 1/3] Support Scala 3's Best Effort compilation This commit allows bloop to support best effort compilation for Metals, with it being enabled with the other Metals options. Best Effort is meant to be a set of Scala 3 compiler options that allow the errored, but typed programs, to be able to be serialized into a TASTy aligned format (Best Effort TASTy), and later to reuse those typed program trees in both the dependent projects, and in the presentation compiler. Those best effort tasty files are serialized to a META-INF/best-effort in classesDir. Best effort compilation may fail, similarly to the regular compilation. In that case we stop compiling the downstream dependencies. Currently best-effort compilation is unable t assist in producing the zinc analysis files, which lead to the projects being recompiled every time. This is solved with a custom hashing solution. --- .../scala/bloop/BloopClassFileManager.scala | 41 +- backend/src/main/scala/bloop/Compiler.scala | 351 +++++++++++++++--- .../scala/bloop/util/BestEffortUtils.scala | 67 ++++ .../src/it/scala/bloop/CommunityBuild.scala | 1 + .../scala/bloop/bsp/BloopBspDefinitions.scala | 6 +- .../scala/bloop/bsp/BloopBspServices.scala | 8 +- .../src/main/scala/bloop/data/Project.scala | 67 +++- .../scala/bloop/data/WorkspaceSettings.scala | 6 +- .../main/scala/bloop/engine/BuildLoader.scala | 20 +- .../main/scala/bloop/engine/Interpreter.scala | 1 + .../bloop/engine/tasks/CompileTask.scala | 41 +- .../compilation/CompileDependenciesData.scala | 13 +- .../tasks/compilation/CompileGraph.scala | 74 ++-- .../tasks/compilation/CompileResult.scala | 12 +- .../test/scala/bloop/BuildLoaderSpec.scala | 3 +- frontend/src/test/scala/bloop/DagSpec.scala | 2 +- .../test/scala/bloop/bsp/BspBaseSuite.scala | 14 +- .../bloop/bsp/BspIntellijClientSpec.scala | 3 +- .../scala/bloop/bsp/BspMetalsClientSpec.scala | 161 +++++++- .../scala/bloop/bsp/BspProtocolSpec.scala | 3 +- .../test/scala/bloop/testing/BaseSuite.scala | 2 +- .../src/test/scala/bloop/util/TestUtil.scala | 1 + 22 files changed, 739 insertions(+), 158 deletions(-) create mode 100644 backend/src/main/scala/bloop/util/BestEffortUtils.scala diff --git a/backend/src/main/scala/bloop/BloopClassFileManager.scala b/backend/src/main/scala/bloop/BloopClassFileManager.scala index 8c82c6566e..02ed3d639f 100644 --- a/backend/src/main/scala/bloop/BloopClassFileManager.scala +++ b/backend/src/main/scala/bloop/BloopClassFileManager.scala @@ -216,19 +216,36 @@ final class BloopClassFileManager( ) => { clientTracer.traceTaskVerbose("copy new products to external classes dir") { _ => val config = ParallelOps.CopyConfiguration(5, CopyMode.ReplaceExisting, Set.empty) - ParallelOps - .copyDirectories(config)( - newClassesDir, - clientExternalClassesDir.underlying, - inputs.ioScheduler, - enableCancellation = false, - inputs.logger - ) - .map { walked => - readOnlyCopyDenylist.++=(walked.target) + val clientExternalBestEffortDir = + clientExternalClassesDir.underlying.resolve("META-INF/best-effort") + + // Deletes all previous best-effort artifacts to get rid of all of the outdated ones. + // Since best effort compilation is not affected by incremental compilation, + // all relevant files are always produced by the compiler. Because of this, + // we can always delete all previous files and copy newly created ones + // without losing anything in the process. + val deleteClientExternalBestEffortDir = + Task { + if (Files.exists(clientExternalBestEffortDir)) { + BloopPaths.delete(AbsolutePath(clientExternalBestEffortDir)) + } () - } - .flatMap(_ => deleteAfterCompilation) + }.memoize + + deleteClientExternalBestEffortDir *> + ParallelOps + .copyDirectories(config)( + newClassesDir, + clientExternalClassesDir.underlying, + inputs.ioScheduler, + enableCancellation = false, + inputs.logger + ) + .map { walked => + readOnlyCopyDenylist.++=(walked.target) + () + } + .flatMap(_ => deleteAfterCompilation) } } ) diff --git a/backend/src/main/scala/bloop/Compiler.scala b/backend/src/main/scala/bloop/Compiler.scala index 0903368e5b..f804eccbc2 100644 --- a/backend/src/main/scala/bloop/Compiler.scala +++ b/backend/src/main/scala/bloop/Compiler.scala @@ -41,6 +41,10 @@ import xsbti.T2 import xsbti.VirtualFileRef import xsbti.compile._ +import bloop.Compiler.Result.Failed +import bloop.util.BestEffortUtils +import bloop.util.BestEffortUtils.BestEffortProducts + case class CompileInputs( scalaInstance: ScalaInstance, compilerCache: CompilerCache, @@ -213,7 +217,8 @@ object Compiler { problems: List[ProblemPerPhase], t: Option[Throwable], elapsed: Long, - backgroundTasks: CompileBackgroundTasks + backgroundTasks: CompileBackgroundTasks, + bestEffortProducts: Option[BestEffortProducts] ) extends Result with CacheHashCode @@ -233,14 +238,18 @@ object Compiler { object NotOk { def unapply(result: Result): Option[Result] = result match { - case f @ (Failed(_, _, _, _) | Cancelled(_, _, _) | Blocked(_) | GlobalError(_, _)) => + case f @ (_: Failed | _: Cancelled | _: Blocked | _: GlobalError) => Some(f) case _ => None } } } - def compile(compileInputs: CompileInputs): Task[Result] = { + def compile( + compileInputs: CompileInputs, + isBestEffortMode: Boolean, + isBestEffortDep: Boolean + ): Task[Result] = { val logger = compileInputs.logger val tracer = compileInputs.tracer val compileOut = compileInputs.out @@ -357,13 +366,74 @@ object Compiler { val uniqueInputs = compileInputs.uniqueInputs reporter.reportStartCompilation(previousProblems) + val fileManager = newFileManager + + // Manually skip redundant best-effort compilations. This is necessary because compiler + // phases supplying the data needed to skip compilations in zinc remain unimplemented for now. + val noopBestEffortResult = compileInputs.previousCompilerResult match { + case Failed( + problems, + t, + elapsed, + _, + bestEffortProducts @ Some(BestEffortProducts(compileProducts, previousHash)) + ) if isBestEffortMode => + val newHash = BestEffortUtils.hashResult( + compileProducts.newClassesDir, + compileInputs.sources, + compileInputs.classpath + ) + + if (newHash == previousHash) { + reporter.processEndCompilation( + problems, + bsp.StatusCode.Error, + None, + None + ) + reporter.reportEndCompilation() + val backgroundTasks = new CompileBackgroundTasks { + def trigger( + clientClassesObserver: ClientClassesObserver, + clientReporter: Reporter, + clientTracer: BraveTracer, + clientLogger: Logger + ): Task[Unit] = Task.defer { + val clientClassesDir = clientClassesObserver.classesDir + clientLogger.debug(s"Triggering background tasks for $clientClassesDir") + val firstTask = Task { BloopPaths.delete(AbsolutePath(newClassesDir)) } + val secondTask = updateExternalClassesDirWithReadOnly( + clientClassesDir, + clientTracer, + clientLogger, + compileInputs, + compileProducts.newClassesDir, + readOnlyCopyDenylist = mutable.HashSet.empty, + allInvalidatedClassFilesForProject, + allInvalidatedExtraCompileProducts + ) + Task + .gatherUnordered(List(firstTask, secondTask)) + .map(_ => ()) + } + } + Some(Failed(problems, t, elapsed, backgroundTasks, bestEffortProducts)) + } else None + case _ => None + } + + if (noopBestEffortResult.isDefined) { + logger.debug("Skipping redundant best-effort compilation") + return Task { noopBestEffortResult.get } + } + BloopZincCompiler .compile( inputs, reporter, logger, uniqueInputs, - newFileManager, + fileManager, cancelPromise, tracer, classpathOptions @@ -372,6 +442,18 @@ object Compiler { .doOnCancel(Task(cancel())) .map { case Success(_) if cancelPromise.isCompleted => handleCancellation + case Success(_) if isBestEffortMode && isBestEffortDep => + handleBestEffortSuccess( + compileInputs, + compileOut, + () => elapsed, + reporter, + backgroundTasksWhenNewSuccessfulAnalysis, + allInvalidatedClassFilesForProject, + allInvalidatedExtraCompileProducts, + previousSuccessfulProblems, + None + ) case Success(result) => // Report end of compilation only after we have reported all warnings from previous runs val sourcesWithFatal = reporter.getSourceFilesWithFatalWarnings @@ -391,41 +473,6 @@ object Compiler { PreviousResult.of(Optional.of(result.analysis()), Optional.of(result.setup())) val analysis = result.analysis() - def updateExternalClassesDirWithReadOnly( - clientClassesDir: AbsolutePath, - clientTracer: BraveTracer, - clientLogger: Logger - ): Task[Unit] = Task.defer { - val descriptionMsg = s"Updating external classes dir with read only $clientClassesDir" - clientTracer.traceTaskVerbose(descriptionMsg) { _ => - Task.defer { - clientLogger.debug(descriptionMsg) - val invalidatedClassFiles = - allInvalidatedClassFilesForProject.iterator.map(_.toPath).toSet - val invalidatedExtraProducts = - allInvalidatedExtraCompileProducts.iterator.map(_.toPath).toSet - val invalidatedInThisProject = invalidatedClassFiles ++ invalidatedExtraProducts - val denyList = invalidatedInThisProject ++ readOnlyCopyDenylist.iterator - val config = - ParallelOps.CopyConfiguration(5, CopyMode.ReplaceIfMetadataMismatch, denyList) - val lastCopy = ParallelOps.copyDirectories(config)( - readOnlyClassesDir, - clientClassesDir.underlying, - compileInputs.ioScheduler, - enableCancellation = false, - compileInputs.logger - ) - - lastCopy.map { _ => - clientLogger.debug( - s"Finished copying classes from $readOnlyClassesDir to $clientClassesDir" - ) - () - } - } - } - } - def persistAnalysis(analysis: CompileAnalysis, out: AbsolutePath): Task[Unit] = { // Important to memoize it, it's triggered by different clients Task(persist(out, analysis, result.setup, tracer, logger)).memoize @@ -460,7 +507,16 @@ object Compiler { val clientClassesDir = clientClassesObserver.classesDir clientLogger.debug(s"Triggering background tasks for $clientClassesDir") val updateClientState = - updateExternalClassesDirWithReadOnly(clientClassesDir, clientTracer, clientLogger) + updateExternalClassesDirWithReadOnly( + clientClassesDir, + clientTracer, + clientLogger, + compileInputs, + readOnlyClassesDir, + readOnlyCopyDenylist, + allInvalidatedClassFilesForProject, + allInvalidatedExtraCompileProducts + ) val writeAnalysisIfMissing = { if (compileOut.analysisOut.exists) Task.unit @@ -543,7 +599,12 @@ object Compiler { val firstTask = updateExternalClassesDirWithReadOnly( clientClassesDir, clientTracer, - clientLogger + clientLogger, + compileInputs, + readOnlyClassesDir, + readOnlyCopyDenylist, + allInvalidatedClassFilesForProject, + allInvalidatedExtraCompileProducts ) val secondTask = Task { @@ -597,6 +658,23 @@ object Compiler { reportedFatalWarnings ) } + case Failure(cause: xsbti.CompileFailed) + if isBestEffortMode && !containsBestEffortFailure(cause) => + // Copies required files to a bsp directory. + // For the Success case this is done by the enclosing method + fileManager.complete(true) + handleBestEffortSuccess( + compileInputs, + compileOut, + () => elapsed, + reporter, + backgroundTasksWhenNewSuccessfulAnalysis, + allInvalidatedClassFilesForProject, + allInvalidatedExtraCompileProducts, + previousSuccessfulProblems, + Some(cause) + ) + case Failure(_: xsbti.CompileCancelled) => handleCancellation case Failure(cause) => val errorCode = bsp.StatusCode.Error @@ -605,26 +683,84 @@ object Compiler { cause match { case f: xsbti.CompileFailed => - // We cannot guarantee reporter.problems == f.problems, so we aggregate them together - val reportedProblems = reporter.allProblemsPerPhase.toList - val rawProblemsFromReporter = reportedProblems.iterator.map(_.problem).toSet - val newProblems = f.problems().flatMap { p => - if (rawProblemsFromReporter.contains(p)) Nil - else List(ProblemPerPhase(p, None)) - } - val failedProblems = reportedProblems ++ newProblems.toList + val failedProblems = findFailedProblems(reporter, Some(f)) val backgroundTasks = toBackgroundTasks(backgroundTasksForFailedCompilation.toList) - Result.Failed(failedProblems, None, elapsed, backgroundTasks) + Result.Failed(failedProblems, None, elapsed, backgroundTasks, None) case t: Throwable => t.printStackTrace() val backgroundTasks = toBackgroundTasks(backgroundTasksForFailedCompilation.toList) - Result.Failed(Nil, Some(t), elapsed, backgroundTasks) + Result.Failed(Nil, Some(t), elapsed, backgroundTasks, None) } } } + def updateExternalClassesDirWithReadOnly( + clientClassesDir: AbsolutePath, + clientTracer: BraveTracer, + clientLogger: Logger, + compileInputs: CompileInputs, + readOnlyClassesDir: Path, + readOnlyCopyDenylist: mutable.HashSet[Path], + allInvalidatedClassFilesForProject: mutable.HashSet[File], + allInvalidatedExtraCompileProducts: mutable.HashSet[File] + ): Task[Unit] = Task.defer { + val descriptionMsg = s"Updating external classes dir with read only $clientClassesDir" + clientTracer.traceTaskVerbose(descriptionMsg) { _ => + Task.defer { + clientLogger.debug(descriptionMsg) + val invalidatedClassFiles = + allInvalidatedClassFilesForProject.iterator.map(_.toPath).toSet + val invalidatedExtraProducts = + allInvalidatedExtraCompileProducts.iterator.map(_.toPath).toSet + val invalidatedInThisProject = invalidatedClassFiles ++ invalidatedExtraProducts + val denyList = invalidatedInThisProject ++ readOnlyCopyDenylist.iterator + val config = + ParallelOps.CopyConfiguration(5, CopyMode.ReplaceIfMetadataMismatch, denyList) + val lastCopy = ParallelOps.copyDirectories(config)( + readOnlyClassesDir, + clientClassesDir.underlying, + compileInputs.ioScheduler, + enableCancellation = false, + compileInputs.logger + ) + + lastCopy.map { _ => + clientLogger.debug( + s"Finished copying classes from $readOnlyClassesDir to $clientClassesDir" + ) + () + } + } + } + } + + def findFailedProblems( + reporter: ZincReporter, + compileFailedMaybe: Option[xsbti.CompileFailed] + ): List[ProblemPerPhase] = { + // We cannot guarantee reporter.problems == f.problems, so we aggregate them together + val reportedProblems = reporter.allProblemsPerPhase.toList + val rawProblemsFromReporter = reportedProblems.iterator.map(_.problem).toSet + val newProblems: List[ProblemPerPhase] = compileFailedMaybe + .map { f => + f.problems() + .flatMap { p => + if (rawProblemsFromReporter.contains(p)) Nil + else List(ProblemPerPhase(p, None)) + } + .toList + } + .getOrElse(Nil) + reportedProblems ++ newProblems.toList + } + + def containsBestEffortFailure(cause: xsbti.CompileFailed) = + cause.problems().exists(_.message().contains("Unsuccessful best-effort compilation.")) || cause + .getCause() + .isInstanceOf[StackOverflowError] + /** * Bloop runs Scala compilation in the same process as the main server, * so the compilation process will use the same JDK that Bloop is using. @@ -714,6 +850,119 @@ object Compiler { .withOrder(inputs.compileOrder) } + /** + * Handles successful Best Effort compilation. + * + * Does not persist incremental compilation analysis, because as of time of commiting the compiler is not able + * to always run the necessary phases, nor is zinc adjusted to handle betasty files correctly. + * + * Returns a [[bloop.Result.Failed]] with generated CompileProducts and a hash value of inputs and outputs included. + */ + def handleBestEffortSuccess( + compileInputs: CompileInputs, + compileOut: CompileOutPaths, + elapsed: () => Long, + reporter: ZincReporter, + backgroundTasksWhenNewSuccessfulAnalysis: mutable.ListBuffer[CompileBackgroundTasks.Sig], + allInvalidatedClassFilesForProject: mutable.HashSet[File], + allInvalidatedExtraCompileProducts: mutable.HashSet[File], + previousSuccessfulProblems: List[ProblemPerPhase], + errorCause: Option[xsbti.CompileFailed] + ): Result = { + val uniqueInputs = compileInputs.uniqueInputs + val readOnlyClassesDir = compileOut.internalReadOnlyClassesDir.underlying + val readOnlyClassesDirPath = readOnlyClassesDir.toString + val newClassesDir = compileOut.internalNewClassesDir.underlying + + reporter.processEndCompilation( + previousSuccessfulProblems, + ch.epfl.scala.bsp.StatusCode.Error, + None, + None + ) + + val noOpPreviousResult = + updatePreviousResultWithRecentClasspathHashes( + compileInputs.previousResult, + uniqueInputs + ) + + val products = CompileProducts( + readOnlyClassesDir, + newClassesDir, + noOpPreviousResult, + noOpPreviousResult, + Set.empty, + Map.empty + ) + + // Delete all those class files that were invalidated in the external classes dir + val allInvalidated = + allInvalidatedClassFilesForProject ++ allInvalidatedExtraCompileProducts + + val backgroundTasksExecution = new CompileBackgroundTasks { + def trigger( + clientClassesObserver: ClientClassesObserver, + clientReporter: Reporter, + clientTracer: BraveTracer, + clientLogger: Logger + ): Task[Unit] = { + val clientClassesDir = clientClassesObserver.classesDir + val clientClassesDirPath = clientClassesDir.toString + val successBackgroundTasks = + backgroundTasksWhenNewSuccessfulAnalysis + .map(f => f(clientClassesDir, clientReporter, clientTracer)) + val allClientSyncTasks = Task.gatherUnordered(successBackgroundTasks.toList).flatMap { _ => + // Only start these tasks after the previous IO tasks in the external dir are done + val firstTask = updateExternalClassesDirWithReadOnly( + clientClassesDir, + clientTracer, + clientLogger, + compileInputs, + readOnlyClassesDir, + readOnlyCopyDenylist = mutable.HashSet.empty, + allInvalidatedClassFilesForProject, + allInvalidatedExtraCompileProducts + ) + + val secondTask = Task { + allInvalidated.foreach { f => + val path = AbsolutePath(f.toPath) + val syntax = path.syntax + if (syntax.startsWith(readOnlyClassesDirPath)) { + val rebasedFile = AbsolutePath( + syntax.replace(readOnlyClassesDirPath, clientClassesDirPath) + ) + if (rebasedFile.exists) { + Files.delete(rebasedFile.underlying) + } + } + } + } + Task + .gatherUnordered(List(firstTask, secondTask)) + .map(_ => ()) + } + + allClientSyncTasks.doOnFinish(_ => Task(clientReporter.reportEndCompilation())) + } + } + + val newHash = BestEffortUtils.hashResult( + products.newClassesDir, + compileInputs.sources, + compileInputs.classpath + ) + val failedProblems = findFailedProblems(reporter, errorCause) + Result.Failed( + failedProblems, + None, + elapsed(), + backgroundTasksExecution, + Some(BestEffortProducts(products, newHash)) + ) + } + def toBackgroundTasks( tasks: List[(AbsolutePath, Reporter, BraveTracer) => Task[Unit]] ): CompileBackgroundTasks = { diff --git a/backend/src/main/scala/bloop/util/BestEffortUtils.scala b/backend/src/main/scala/bloop/util/BestEffortUtils.scala new file mode 100644 index 0000000000..318507fe0d --- /dev/null +++ b/backend/src/main/scala/bloop/util/BestEffortUtils.scala @@ -0,0 +1,67 @@ +package bloop.util + +import java.security.MessageDigest + +import java.math.BigInteger +import java.nio.file.Files +import scala.collection.JavaConverters._ +import java.nio.file.Path +import bloop.io.AbsolutePath + +object BestEffortUtils { + + case class BestEffortProducts(compileProducts: bloop.CompileProducts, hash: String) + + /* Hashes results of a projects compilation, to mimic how it would have been handled in zinc. + * Returns SHA-1 of a project. + * + * Since currently for best-effort compilation we are unable to use neither incremental compilation, + * nor the data supplied by zinc (like the compilation analysis files, which are not able to be generated + * since the compiler is able to skip the necessary phases for now), this custom implementation + * is meant to keep the best-effort projects from being unnecessarily recompiled. + */ + def hashResult( + outputDir: Path, + sources: Array[AbsolutePath], + classpath: Array[AbsolutePath] + ): String = { + val md = MessageDigest.getInstance("SHA-1") + + md.update("".getBytes()) + if (Files.exists(outputDir)) { + Files.walk(outputDir).iterator().asScala.foreach { path => + if (Files.isRegularFile(path)) { + md.update(path.toString.getBytes()) + md.update(Files.readAllBytes(path)) + } + } + } + + md.update("".getBytes()) + sources.foreach { sourceFilePath => + val underlying = sourceFilePath.underlying + if (Files.exists(underlying) && Files.isRegularFile(underlying)) { + md.update(Files.readAllBytes(underlying)) + } + } + + md.update("".getBytes()) + classpath.map(_.underlying).foreach { classpathFile => + if (!Files.exists(classpathFile)) () + else if (Files.isRegularFile(classpathFile)) { + md.update(Files.readAllBytes(classpathFile)) + } else if (Files.isDirectory(classpathFile)) { + if (outputDir != classpathFile) { + Files.walk(classpathFile).iterator().asScala.foreach { file => + if (Files.isRegularFile(file)) { + md.update(Files.readAllBytes(file)) + } + } + } + } + } + + val digest = new BigInteger(1, md.digest()) + String.format(s"%040x", digest) + } +} diff --git a/frontend/src/it/scala/bloop/CommunityBuild.scala b/frontend/src/it/scala/bloop/CommunityBuild.scala index 3511168a10..1953f63fea 100644 --- a/frontend/src/it/scala/bloop/CommunityBuild.scala +++ b/frontend/src/it/scala/bloop/CommunityBuild.scala @@ -132,6 +132,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { resources = Nil, compileSetup = Config.CompileSetup.empty, genericClassesDir = dummyClassesDir, + isBestEffort = false, scalacOptions = Nil, javacOptions = Nil, sources = Nil, diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala b/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala index 28375e74fd..e741c74236 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala @@ -12,7 +12,8 @@ object BloopBspDefinitions { clientClassesRootDir: Option[Uri], semanticdbVersion: Option[String], supportedScalaVersions: Option[List[String]], - javaSemanticdbVersion: Option[String] + javaSemanticdbVersion: Option[String], + enableBestEffortMode: Option[Boolean] ) object BloopExtraBuildParams { @@ -21,7 +22,8 @@ object BloopBspDefinitions { clientClassesRootDir = None, semanticdbVersion = None, supportedScalaVersions = None, - javaSemanticdbVersion = None + javaSemanticdbVersion = None, + enableBestEffortMode = None ) implicit val codec: JsonValueCodec[BloopExtraBuildParams] = diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala index 717ebd9885..b11f08e508 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala @@ -275,6 +275,7 @@ final class BloopBspServices( } else { val javaSemanticDBVersion = extraBuildParams.flatMap(_.javaSemanticdbVersion) val scalaSemanticDBVersion = extraBuildParams.flatMap(_.semanticdbVersion) + val enableBestEffortMode = extraBuildParams.flatMap(_.enableBestEffortMode) val supportedScalaVersions = if (scalaSemanticDBVersion.nonEmpty) extraBuildParams.map(_.supportedScalaVersions.toList.flatten) @@ -286,7 +287,8 @@ final class BloopBspServices( scalaSemanticDBVersion, supportedScalaVersions, currentRefreshProjectsCommand, - currentTraceSettings + currentTraceSettings, + enableBestEffortMode ) ) else None @@ -432,6 +434,7 @@ final class BloopBspServices( } val isPipeline = compileArgs.exists(_ == "--pipeline") + val isBestEffort = compileArgs.exists(_ == "--best-effort") def compile(projects: List[Project]): Task[State] = { val config = ReporterConfig.defaultFormat.copy(reverseOrder = false) @@ -481,6 +484,7 @@ final class BloopBspServices( dag, createReporter, isPipeline, + isBestEffort, cancelCompilation, store, logger @@ -506,7 +510,7 @@ final class BloopBspServices( case Compiler.Result.GlobalError(problem, _) => List(problem) case Compiler.Result.Cancelled(problems, elapsed, _) => List(reportError(p, problems, elapsed)) - case f @ Compiler.Result.Failed(problems, t, elapsed, _) => + case f @ Compiler.Result.Failed(problems, t, elapsed, _, _) => previouslyFailedCompilations.put(p, f) val acc = List(reportError(p, problems, elapsed)) t match { diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index f3736ff6f6..cc46af71b6 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -28,6 +28,7 @@ import bloop.util.JavaRuntime import scalaz.Cord import xsbti.compile.ClasspathOptions import xsbti.compile.CompileOrder +import scala.util.Success final case class Project( name: String, @@ -39,6 +40,7 @@ final case class Project( resources: List[AbsolutePath], compileSetup: Config.CompileSetup, genericClassesDir: AbsolutePath, + isBestEffort: Boolean, scalacOptions: List[String], javacOptions: List[String], sources: List[AbsolutePath], @@ -360,6 +362,7 @@ object Project { compileResources, setup, AbsolutePath(project.classesDir), + isBestEffort = false, scala.map(_.options).getOrElse(Nil), project.java.map(_.options).getOrElse(Nil), project.sources.map(AbsolutePath.apply), @@ -418,6 +421,7 @@ object Project { configDir: AbsolutePath, scalaSemanticDBPlugin: Option[AbsolutePath], javaSemanticDBPlugin: Option[AbsolutePath], + enableBestEffortMode: Option[Boolean], logger: Logger ): Project = { val workspaceDir = project.workspaceDirectory.getOrElse(configDir.getParent) @@ -429,6 +433,30 @@ object Project { version != "3.0.0-M2" } + def canEnableBestEffortFlag(version: String): Boolean = { + val split = version.split('.') + if (split.length == 3) { + val major = split(0) + val minor = split(1) + (Try(major.toInt), Try(minor.toInt)) match { + case (Success(majorVer), Success(minorVer)) => majorVer >= 3 && minorVer >= 5 + case _ => false + } + } else false + } + + def enableBestEffortFlag(options: List[String]): List[String] = { + val bestEffortOpt = "-Ybest-effort" + val withBETastyOpt = "-Ywith-best-effort-tasty" + val optsWithBestEffort = + if (options.contains(bestEffortOpt)) options + else options :+ bestEffortOpt + val optsWithBETasty = + if (optsWithBestEffort.contains(withBETastyOpt)) optsWithBestEffort + else optsWithBestEffort :+ withBETastyOpt + optsWithBETasty + } + def enableScalaSemanticdb(options: List[String], pluginPath: AbsolutePath): List[String] = { val baseSemanticdbOptions = List( "-P:semanticdb:failures:warning", @@ -535,18 +563,35 @@ object Project { val scalacOptionsWithSemanticDB = enableScalaSemanticdb(rangedScalacOptions, pluginPath) projectWithRangePositions.copy(scalacOptions = scalacOptionsWithSemanticDB) } - javaSemanticDBPlugin match { - case None => - scalaProjectWithRangePositions - case Some(pluginPath) => - val javacOptionsWithSemanticDB = enableJavaSemanticdbOptions(javacOptions, pluginPath) - val classpathWithSemanticDB = - enableJavaSemanticdbClasspath(pluginPath, scalaProjectWithRangePositions.rawClasspath) - scalaProjectWithRangePositions.copy( - javacOptions = javacOptionsWithSemanticDB, - rawClasspath = classpathWithSemanticDB + + val withEnabledJavaSemanticDb = + javaSemanticDBPlugin match { + case None => + scalaProjectWithRangePositions + case Some(pluginPath) => + val javacOptionsWithSemanticDB = enableJavaSemanticdbOptions(javacOptions, pluginPath) + val classpathWithSemanticDB = + enableJavaSemanticdbClasspath(pluginPath, scalaProjectWithRangePositions.rawClasspath) + scalaProjectWithRangePositions.copy( + javacOptions = javacOptionsWithSemanticDB, + rawClasspath = classpathWithSemanticDB + ) + } + + val withEnabledBestEffortCompilation = + if ( + enableBestEffortMode.getOrElse(false) && + project.scalaInstance.exists(i => canEnableBestEffortFlag(i.version)) + ) { + val options = enableBestEffortFlag(withEnabledJavaSemanticDb.scalacOptions) + + project.copy( + isBestEffort = true, + scalacOptions = options ) - } + } else withEnabledJavaSemanticDb + + withEnabledBestEffortCompilation } def hasScalaSemanticDBEnabledInCompilerOptions(options: List[String]): Boolean = { diff --git a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala index c4efed1d88..9060161d39 100644 --- a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala +++ b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala @@ -47,6 +47,8 @@ import com.github.plokhotnyuk.jsoniter_scala.macros.JsonCodecMaker * server before loading the state and presentings projects to the client. * @param traceSettings are the settings provided by the user that customize how * the bloop server should behave. + * @param enableBestEffortMode signifies whether the best-effort compilation mode + * is used */ case class WorkspaceSettings( // Managed by bloop or build tool @@ -55,7 +57,8 @@ case class WorkspaceSettings( supportedScalaVersions: Option[List[String]], // Managed by the user refreshProjectsCommand: Option[List[String]], - traceSettings: Option[TraceSettings] + traceSettings: Option[TraceSettings], + enableBestEffortMode: Option[Boolean] ) { def withSemanticdbSettings: Option[(WorkspaceSettings, SemanticdbSettings)] = if (semanticDBVersion.nonEmpty || javaSemanticDBVersion.nonEmpty) { @@ -86,6 +89,7 @@ object WorkspaceSettings { Some(scalaSemanticDBVersion), Some(supportedScalaVersions), None, + None, None ) } diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index 3354778494..2d0193eb69 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -76,6 +76,7 @@ object BuildLoader { configDir, semanticdb.javaSemanticdbSettings, semanticdb.scalaSemanticdbSettings, + settings.enableBestEffortMode, logger ).map { transformedProjects => transformedProjects.map { @@ -106,6 +107,7 @@ object BuildLoader { configDir: AbsolutePath, javaSemanticSettings: Option[JavaSemanticdbSettings], scalaSemanticdbSettings: Option[ScalaSemanticdbSettings], + enableBestEffortMode: Option[Boolean], logger: Logger ): Task[List[(Project, Option[Project])]] = { @@ -124,7 +126,14 @@ object BuildLoader { logger ) { (scalaPlugin: Option[AbsolutePath], javaPlugin: Option[AbsolutePath]) => projects.map(p => - Project.enableMetalsSettings(p, configDir, scalaPlugin, javaPlugin, logger) -> Some(p) + Project.enableMetalsSettings( + p, + configDir, + scalaPlugin, + javaPlugin, + enableBestEffortMode, + logger + ) -> Some(p) ) } } @@ -175,7 +184,14 @@ object BuildLoader { logger ) { (scalaPlugin: Option[AbsolutePath], javaPlugin: Option[AbsolutePath]) => LoadedProject.ConfiguredProject( - Project.enableMetalsSettings(project, configDir, scalaPlugin, javaPlugin, logger), + Project.enableMetalsSettings( + project, + configDir, + scalaPlugin, + javaPlugin, + settings.enableBestEffortMode, + logger + ), project, settings ) diff --git a/frontend/src/main/scala/bloop/engine/Interpreter.scala b/frontend/src/main/scala/bloop/engine/Interpreter.scala index c4a32e8e18..545fd12f08 100644 --- a/frontend/src/main/scala/bloop/engine/Interpreter.scala +++ b/frontend/src/main/scala/bloop/engine/Interpreter.scala @@ -173,6 +173,7 @@ object Interpreter { dag, createReporter, cmd.pipeline, + bestEffort = false, Promise[Unit](), CompileClientStore.NoStore, state.logger diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index 19e20f4096..89937ba391 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -9,6 +9,7 @@ import bloop.CompileOutPaths import bloop.CompileProducts import bloop.Compiler import bloop.Compiler.Result.Success +import bloop.Compiler.Result.Failed import bloop.cli.ExitStatus import bloop.data.Project import bloop.data.WorkspaceSettings @@ -22,6 +23,7 @@ import bloop.engine.tasks.compilation._ import bloop.io.ParallelOps import bloop.io.ParallelOps.CopyMode import bloop.io.{Paths => BloopPaths} +import bloop.io.AbsolutePath import bloop.logging.DebugFilter import bloop.logging.Logger import bloop.logging.LoggerAction @@ -32,6 +34,7 @@ import bloop.reporter.ReporterAction import bloop.reporter.ReporterInputs import bloop.task.Task import bloop.tracing.BraveTracer +import bloop.util.BestEffortUtils.BestEffortProducts import monix.execution.CancelableFuture import monix.reactive.MulticastStrategy @@ -44,6 +47,7 @@ object CompileTask { dag: Dag[Project], createReporter: ReporterInputs[UseSiteLogger] => Reporter, pipeline: Boolean, + bestEffort: Boolean, cancelCompilation: Promise[Unit], store: CompileClientStore, rawLogger: UseSiteLogger @@ -80,7 +84,11 @@ object CompileTask { "client" -> clientName ) - def compile(graphInputs: CompileGraph.Inputs): Task[ResultBundle] = { + def compile( + graphInputs: CompileGraph.Inputs, + isBestEffort: Boolean, + isBestEffortDep: Boolean + ): Task[ResultBundle] = { val bundle = graphInputs.bundle val project = bundle.project val logger = bundle.logger @@ -164,7 +172,9 @@ object CompileTask { // Block on the task associated with this result that sets up the read-only classes dir waitOnReadClassesDir.flatMap { _ => // Only when the task is finished, we kickstart the compilation - inputs.flatMap(inputs => Compiler.compile(inputs)).map { result => + def compile(inputs: CompileInputs) = + Compiler.compile(inputs, isBestEffort, isBestEffortDep) + inputs.flatMap(inputs => compile(inputs)).map { result => def runPostCompilationTasks( backgroundTasks: CompileBackgroundTasks ): CancelableFuture[Unit] = { @@ -266,12 +276,12 @@ object CompileTask { } val client = state.client - CompileGraph.traverse(dag, client, store, setup(_), compile(_)).flatMap { pdag => + CompileGraph.traverse(dag, client, store, bestEffort, setup(_), compile).flatMap { pdag => val partialResults = Dag.dfs(pdag, mode = Dag.PreOrder) val finalResults = partialResults.map(r => PartialCompileResult.toFinalResult(r)) Task.gatherUnordered(finalResults).map(_.flatten).flatMap { results => val cleanUpTasksToRunInBackground = - markUnusedClassesDirAndCollectCleanUpTasks(results, rawLogger) + markUnusedClassesDirAndCollectCleanUpTasks(results, state, rawLogger) val failures = results.flatMap { case FinalNormalCompileResult(p, results) => @@ -372,6 +382,7 @@ object CompileTask { private def markUnusedClassesDirAndCollectCleanUpTasks( results: List[FinalCompileResult], + previousState: State, logger: Logger ): List[Task[Unit]] = { val cleanUpTasksToSpawnInBackground = mutable.ListBuffer[Task[Unit]]() @@ -379,6 +390,12 @@ object CompileTask { val resultBundle = finalResult.result val newSuccessful = resultBundle.successful val compilerResult = resultBundle.fromCompiler + val previousResult = + finalResult match { + case FinalNormalCompileResult(p, _) => + previousState.results.all.get(p) + case _ => None + } val populateNewProductsTask = newSuccessful.map(_.populatingProducts).getOrElse(Task.unit) val cleanUpPreviousLastSuccessful = resultBundle.previous match { case None => populateNewProductsTask @@ -386,7 +403,7 @@ object CompileTask { for { _ <- previousSuccessful.populatingProducts _ <- populateNewProductsTask - _ <- cleanUpPreviousResult(previousSuccessful, compilerResult, logger) + _ <- cleanUpPreviousResult(previousSuccessful, previousResult, compilerResult, logger) } yield () } @@ -426,6 +443,7 @@ object CompileTask { */ private def cleanUpPreviousResult( previousSuccessful: LastSuccessfulResult, + previousResult: Option[Compiler.Result], compilerResult: Compiler.Result, logger: Logger ): Task[Unit] = { @@ -448,6 +466,19 @@ object CompileTask { logger.debug(s"Scheduling to delete ${previousClassesDir} superseded by $newClassesDir") Some(previousClassesDir) } + case Failed(_, _, _, _, Some(BestEffortProducts(products, _))) => + val newClassesDir = products.newClassesDir + previousResult match { + case Some(Failed(_, _, _, _, Some(BestEffortProducts(previousProducts, _)))) => + val previousClassesDir = previousProducts.newClassesDir + if (previousClassesDir != newClassesDir) { + logger.debug( + s"Scheduling to delete ${previousClassesDir} superseded by $newClassesDir" + ) + Some(AbsolutePath(previousClassesDir)) + } else None + case _ => None + } case _ => None } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileDependenciesData.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileDependenciesData.scala index b866af8628..20292bd753 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileDependenciesData.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileDependenciesData.scala @@ -11,6 +11,7 @@ import bloop.io.AbsolutePath case class CompileDependenciesData( dependencyClasspath: Array[AbsolutePath], + bestEffortDirs: Seq[AbsolutePath], allInvalidatedClassFiles: Set[File], allGeneratedClassFilePaths: Map[String, File] ) { @@ -22,7 +23,7 @@ case class CompileDependenciesData( // Important: always place new classes dir before read-only classes dir val classesDirs = Array(newClassesDir, readOnlyClassesDir) val resources = Project.pickValidResources(project.resources) - resources ++ classesDirs ++ dependencyClasspath + resources ++ classesDirs ++ bestEffortDirs ++ dependencyClasspath } } @@ -33,6 +34,7 @@ object CompileDependenciesData { ): CompileDependenciesData = { val dependentClassesDir = new mutable.HashMap[AbsolutePath, Array[AbsolutePath]]() val dependentResources = new mutable.HashMap[AbsolutePath, Array[AbsolutePath]]() + val dependentBestEffortDirs = new mutable.ArrayBuffer[AbsolutePath]() val dependentInvalidatedClassFiles = new mutable.HashSet[File]() val dependentGeneratedClassFilePaths = new mutable.HashMap[String, File]() dependentProducts.foreach { @@ -47,6 +49,9 @@ object CompileDependenciesData { else Array(newClassesDir, readOnlyClassesDir) } + if (project.isBestEffort) { + dependentBestEffortDirs ++= classesDirs.map(_.resolve("META-INF").resolve("best-effort")) + } dependentClassesDir.put(genericClassesDir, classesDirs) case (project, Right(products)) => val genericClassesDir = project.genericClassesDir @@ -59,6 +64,11 @@ object CompileDependenciesData { } val resources = Project.pickValidResources(project.resources) + if (project.isBestEffort) { + dependentBestEffortDirs ++= classesDirs + .map(AbsolutePath(_).resolve("META-INF").resolve("best-effort")) + .toSeq + } dependentClassesDir.put(genericClassesDir, classesDirs.map(AbsolutePath(_))) dependentInvalidatedClassFiles.++=(products.invalidatedCompileProducts) dependentGeneratedClassFilePaths.++=(products.generatedRelativeClassFilePaths.iterator) @@ -83,6 +93,7 @@ object CompileDependenciesData { CompileDependenciesData( rewrittenClasspath, + dependentBestEffortDirs.toSeq, dependentInvalidatedClassFiles.toSet, dependentGeneratedClassFilePaths.toMap ) diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala index da644240aa..151dabc4b2 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala @@ -29,6 +29,7 @@ import bloop.reporter.ReporterAction import bloop.task.Task import bloop.util.JavaCompat.EnrichOptional import bloop.util.SystemProperties +import bloop.util.BestEffortUtils.BestEffortProducts import xsbti.compile.PreviousResult @@ -46,27 +47,12 @@ object CompileGraph { ): PartialSuccess = PartialSuccess(bundle, Task.now(result)) private def blockedBy(dag: Dag[PartialCompileResult]): Option[Project] = { - def blockedFromResults(results: List[PartialCompileResult]): Option[Project] = { - results match { - case Nil => None - case result :: _ => - result match { - case PartialEmpty => None - case _: PartialSuccess => None - case f: PartialFailure => Some(f.project) - case _: PartialFailures => blockedFromResults(results) - } - } - } - dag match { case Leaf(_: PartialSuccess) => None case Leaf(f: PartialFailure) => Some(f.project) - case Leaf(fs: PartialFailures) => blockedFromResults(fs.failures) case Leaf(PartialEmpty) => None case Parent(_: PartialSuccess, _) => None case Parent(f: PartialFailure, _) => Some(f.project) - case Parent(fs: PartialFailures, _) => blockedFromResults(fs.failures) case Parent(PartialEmpty, _) => None case Aggregate(dags) => dags.foldLeft(None: Option[Project]) { @@ -377,8 +363,9 @@ object CompileGraph { dag: Dag[Project], client: ClientInfo, store: CompileClientStore, + bestEffort: Boolean, computeBundle: BundleInputs => Task[CompileBundle], - compile: Inputs => Task[ResultBundle] + compile: (Inputs, Boolean, Boolean) => Task[ResultBundle] ): CompileTraversal = { val tasks = new mutable.HashMap[Dag[Project], CompileTraversal]() def register(k: Dag[Project], v: CompileTraversal): CompileTraversal = { @@ -398,7 +385,7 @@ object CompileGraph { PartialFailure(bundle.project, FailedOrCancelledPromise, Task.now(results)) } - def loop(dag: Dag[Project]): CompileTraversal = { + def loop(dag: Dag[Project], isBestEffortDep: Boolean): CompileTraversal = { tasks.get(dag) match { case Some(task) => task case None => @@ -406,7 +393,7 @@ object CompileGraph { case Leaf(project) => val bundleInputs = BundleInputs(project, dag, Map.empty) setupAndDeduplicate(client, bundleInputs, computeBundle) { bundle => - compile(Inputs(bundle, Map.empty)).map { results => + compile(Inputs(bundle, Map.empty), bestEffort, isBestEffortDep).map { results => results.fromCompiler match { case Compiler.Result.Ok(_) => Leaf(partialSuccess(bundle, results)) case _ => Leaf(toPartialFailure(bundle, results)) @@ -415,29 +402,41 @@ object CompileGraph { } case Aggregate(dags) => - val downstream = dags.map(loop) + val downstream = dags.map(loop(_, isBestEffortDep = false)) Task.gatherUnordered(downstream).flatMap { dagResults => Task.now(Parent(PartialEmpty, dagResults)) } case Parent(project, dependencies) => - val downstream = dependencies.map(loop) + val downstream = dependencies.map(loop(_, isBestEffortDep = false)) Task.gatherUnordered(downstream).flatMap { dagResults => + val depsSupportBestEffort = + dependencies.map(Dag.dfs(_, mode = Dag.PreOrder)).flatten.forall(_.isBestEffort) val failed = dagResults.flatMap(dag => blockedBy(dag).toList) - if (failed.nonEmpty) { - // Register the name of the projects we're blocked on (intransitively) - val blockedResult = Compiler.Result.Blocked(failed.map(_.name)) - val blocked = Task.now(ResultBundle(blockedResult, None, None)) - Task.now(Parent(PartialFailure(project, BlockURI, blocked), dagResults)) - } else { - val results: List[PartialSuccess] = { - val transitive = dagResults.flatMap(Dag.dfs(_, mode = Dag.PreOrder)).distinct - transitive.collect { case s: PartialSuccess => s } + + val allResults = Task.gatherUnordered { + val transitive = dagResults.flatMap(Dag.dfs(_, mode = Dag.PreOrder)).distinct + transitive.flatMap { + case PartialSuccess(bundle, result) => Some(result.map(r => bundle.project -> r)) + case PartialFailure(project, _, result) => Some(result.map(r => project -> r)) + case _ => None } + } - val projectResults = - results.map(ps => ps.result.map(r => ps.bundle.project -> r)) - Task.gatherUnordered(projectResults).flatMap { results => + allResults.flatMap { results => + val successfulBestEffort = !results.exists { + case (_, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => f.bestEffortProducts.isEmpty + case _ => false + } + val continue = bestEffort && depsSupportBestEffort && successfulBestEffort || failed.isEmpty + val dependsOnBestEffort = failed.nonEmpty && bestEffort && depsSupportBestEffort || isBestEffortDep + + if (!continue) { + // Register the name of the projects we're blocked on (intransitively) + val blockedResult = Compiler.Result.Blocked(failed.map(_.name)) + val blocked = Task.now(ResultBundle(blockedResult, None, None)) + Task.now(Parent(PartialFailure(project, BlockURI, blocked), dagResults)) + } else { val dependentProducts = new mutable.ListBuffer[(Project, BundleProducts)]() val dependentResults = new mutable.ListBuffer[(File, PreviousResult)]() results.foreach { @@ -448,6 +447,11 @@ object CompileGraph { dependentResults .+=(newProducts.newClassesDir.toFile -> newResult) .+=(newProducts.readOnlyClassesDir.toFile -> newResult) + case (p, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => + f.bestEffortProducts.foreach { + case BestEffortProducts(products, _) => + dependentProducts += (p -> Right(products)) + } case _ => () } @@ -455,9 +459,9 @@ object CompileGraph { val bundleInputs = BundleInputs(project, dag, dependentProducts.toMap) setupAndDeduplicate(client, bundleInputs, computeBundle) { bundle => val inputs = Inputs(bundle, resultsMap) - compile(inputs).map { results => + compile(inputs, bestEffort, dependsOnBestEffort).map { results => results.fromCompiler match { - case Compiler.Result.Ok(_) => + case Compiler.Result.Ok(_) if failed.isEmpty => Parent(partialSuccess(bundle, results), dagResults) case _ => Parent(toPartialFailure(bundle, results), dagResults) } @@ -471,7 +475,7 @@ object CompileGraph { } } - loop(dag) + loop(dag, isBestEffortDep = false) } private def errorToString(err: Throwable): String = { diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala index f5b6008431..1ed33a9510 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala @@ -43,8 +43,6 @@ object PartialCompileResult { case PartialEmpty => Task.now(FinalEmptyResult :: Nil) case PartialFailure(project, _, bundle) => bundle.map(b => FinalNormalCompileResult(project, b) :: Nil) - case PartialFailures(failures, _) => - Task.gatherUnordered(failures.map(toFinalResult(_))).map(_.flatten) case PartialSuccess(bundle, result) => result.map(res => FinalNormalCompileResult(bundle.project, res) :: Nil) } @@ -63,12 +61,6 @@ case class PartialFailure( ) extends PartialCompileResult with CacheHashCode {} -case class PartialFailures( - failures: List[PartialCompileResult], - result: Task[ResultBundle] -) extends PartialCompileResult - with CacheHashCode {} - case class PartialSuccess( bundle: SuccessfulCompileBundle, result: Task[ResultBundle] @@ -93,7 +85,7 @@ object FinalNormalCompileResult { object HasException { def unapply(res: FinalNormalCompileResult): Option[(Project, Either[String, Throwable])] = { res.result.fromCompiler match { - case Compiler.Result.Failed(_, Some(err), _, _) => + case Compiler.Result.Failed(_, Some(err), _, _, _) => Some((res.project, Right(err))) case Compiler.Result.GlobalError(problem, errOpt) => val err = errOpt.map(Right(_)).getOrElse(Left(problem)) @@ -127,7 +119,7 @@ object FinalCompileResult { case Compiler.Result.Blocked(on) => s"${projectName} (blocked on ${on.mkString(", ")})" case Compiler.Result.GlobalError(problem, _) => s"${projectName} (failed with global error ${problem})" - case Compiler.Result.Failed(problems, t, ms, _) => + case Compiler.Result.Failed(problems, t, ms, _, _) => val extra = t match { case Some(t) => s"exception '${t.getMessage}', " case None => "" diff --git a/frontend/src/test/scala/bloop/BuildLoaderSpec.scala b/frontend/src/test/scala/bloop/BuildLoaderSpec.scala index 42ebcc78ce..f4f96cfa85 100644 --- a/frontend/src/test/scala/bloop/BuildLoaderSpec.scala +++ b/frontend/src/test/scala/bloop/BuildLoaderSpec.scala @@ -306,7 +306,8 @@ object BuildLoaderSpec extends BaseSuite { traceEndAnnotation = Some("end"), enabled = Some(true) ) - ) + ), + None ) val state1 = loadState(workspace1, Nil, logger, Some(settings1)) diff --git a/frontend/src/test/scala/bloop/DagSpec.scala b/frontend/src/test/scala/bloop/DagSpec.scala index 49b7ef5d6c..cb85672860 100644 --- a/frontend/src/test/scala/bloop/DagSpec.scala +++ b/frontend/src/test/scala/bloop/DagSpec.scala @@ -28,7 +28,7 @@ class DagSpec { def dummyOrigin: Origin = TestUtil.syntheticOriginFor(dummyPath) def dummyProject(name: String, dependencies: List[String]): Project = Project(name, dummyPath, None, dependencies, Some(dummyInstance), Nil, Nil, compileOptions, - dummyPath, Nil, Nil, Nil, Nil, None, Nil, Nil, Config.TestOptions.empty, dummyPath, dummyPath, + dummyPath, isBestEffort = false, Nil, Nil, Nil, Nil, None, Nil, Nil, Config.TestOptions.empty, dummyPath, dummyPath, Project.defaultPlatform(logger, Nil, Nil), None, None, Nil, dummyOrigin) // format: ON diff --git a/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala b/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala index 7f7c6981c8..e2a0b78e30 100644 --- a/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala +++ b/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala @@ -144,15 +144,16 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { def compileTask( project: TestProject, originId: Option[String], - clearDiagnostics: Boolean = true + clearDiagnostics: Boolean = true, + arguments: Option[List[String]] = None ): Task[ManagedBspTestState] = { runAfterTargets(project) { target => // Handle internal state before sending compile request if (clearDiagnostics) diagnostics.clear() currentCompileIteration.increment(1) - rpcRequest(BuildTarget.compile, bsp.CompileParams(List(target), originId, None)).flatMap { - r => + rpcRequest(BuildTarget.compile, bsp.CompileParams(List(target), originId, arguments)) + .flatMap { r => // `headL` returns latest saved state from bsp because source is behavior subject Task .liftMonixTaskUncancellable( @@ -168,7 +169,7 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { serverStates ) } - } + } } } @@ -192,11 +193,12 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { project: TestProject, originId: Option[String] = None, clearDiagnostics: Boolean = true, - timeout: Long = 30 + timeout: Long = 30, + arguments: Option[List[String]] = None ): ManagedBspTestState = { // Use a default timeout of 30 seconds for every operation TestUtil.await(FiniteDuration(timeout, "s")) { - compileTask(project, originId, clearDiagnostics) + compileTask(project, originId, clearDiagnostics, arguments) } } diff --git a/frontend/src/test/scala/bloop/bsp/BspIntellijClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspIntellijClientSpec.scala index ee136b1a21..aa48e3fd7b 100644 --- a/frontend/src/test/scala/bloop/bsp/BspIntellijClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspIntellijClientSpec.scala @@ -59,7 +59,8 @@ class BspIntellijClientSpec( None, None, refreshProjectsCommand = Some(refreshProjectsCommand), - Some(TraceSettings.fromProperties(TraceProperties.default)) + Some(TraceSettings.fromProperties(TraceProperties.default)), + None ) WorkspaceSettings.writeToFile(configDir, workspaceSettings, logger) diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index 43a16d0a35..ca2807f10b 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -22,6 +22,7 @@ import bloop.logging.RecordingLogger import bloop.task.Task import bloop.util.TestProject import bloop.util.TestUtil +import bloop.Compiler object LocalBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Local) object TcpBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Tcp) @@ -57,7 +58,8 @@ class BspMetalsClientSpec( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), supportedScalaVersions = Some(List(testedScalaVersion)), - javaSemanticdbVersion = Some(javaSemanticdbVersion) + javaSemanticdbVersion = Some(javaSemanticdbVersion), + enableBestEffortMode = None ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => @@ -113,7 +115,8 @@ class BspMetalsClientSpec( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), supportedScalaVersions = Some(List(testedScalaVersion)), - javaSemanticdbVersion = Some(javaSemanticdbVersion) + javaSemanticdbVersion = Some(javaSemanticdbVersion), + enableBestEffortMode = None ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => @@ -157,7 +160,8 @@ class BspMetalsClientSpec( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), supportedScalaVersions = Some(List(testedScalaVersion)), - javaSemanticdbVersion = Some(javaSemanticdbVersion) + javaSemanticdbVersion = Some(javaSemanticdbVersion), + enableBestEffortMode = None ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => @@ -187,7 +191,8 @@ class BspMetalsClientSpec( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), supportedScalaVersions = Some(List(testedScalaVersion)), - javaSemanticdbVersion = Some(javaSemanticdbVersion) + javaSemanticdbVersion = Some(javaSemanticdbVersion), + enableBestEffortMode = None ) val `A` = TestProject(workspace, "A", Nil) val projects = List(`A`) @@ -240,7 +245,8 @@ class BspMetalsClientSpec( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), supportedScalaVersions = Some(List(testedScalaVersion)), - javaSemanticdbVersion = Some(javaSemanticdbVersion) + javaSemanticdbVersion = Some(javaSemanticdbVersion), + enableBestEffortMode = None ) val bspLogger = new BspClientLogger(logger) def bspCommand() = createBspCommand(configDir) @@ -465,7 +471,8 @@ class BspMetalsClientSpec( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), supportedScalaVersions = Some(List(testedScalaVersion)), - javaSemanticdbVersion = Some(javaSemanticdbVersion) + javaSemanticdbVersion = Some(javaSemanticdbVersion), + enableBestEffortMode = None ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => val compiledState = state.compile(`A`).toTestState @@ -492,7 +499,8 @@ class BspMetalsClientSpec( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), supportedScalaVersions = Some(List(testedScalaVersion)), - javaSemanticdbVersion = Some(javaSemanticdbVersion) + javaSemanticdbVersion = Some(javaSemanticdbVersion), + None ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => val compiledState = state.compile(`A`).toTestState @@ -503,6 +511,102 @@ class BspMetalsClientSpec( } } + val bestEffortScalaVersion = "3.5.0-RC1" + test("best-effort: compile dependency of failing project and produce semanticdb and betasty") { + TestUtil.withinWorkspace { workspace => + val `A` = TestProject( + workspace, + "A", + dummyBestEffortSources, + scalaVersion = Some(bestEffortScalaVersion) + ) + val `B` = TestProject( + workspace, + "B", + dummyBestEffortDepSources, + directDependencies = List(`A`), + scalaVersion = Some(bestEffortScalaVersion) + ) + val projects = List(`A`, `B`) + TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) + val extraParams = BloopExtraBuildParams( + ownsBuildFiles = None, + clientClassesRootDir = None, + semanticdbVersion = Some(semanticdbVersion), + supportedScalaVersions = Some(List(bestEffortScalaVersion)), + javaSemanticdbVersion = None, + enableBestEffortMode = Some(true) + ) + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + val compiledStateA = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState + assert(compiledStateA.status == ExitStatus.CompilationError) + assertSemanticdbFileFor("TypeError.scala", compiledStateA, "A") + assertBetastyFile("TypeError.betasty", compiledStateA, "A") + val compiledStateB = state.compile(`B`, arguments = Some(List("--best-effort"))).toTestState + assert(compiledStateB.status == ExitStatus.CompilationError) + assertSemanticdbFileFor("TypeErrorDependency.scala", compiledStateB, "B") + assertBetastyFile("TypeErrorDependency.betasty", compiledStateB, "B") + + val projectB = compiledStateB.build.getProjectFor("B").get + compiledStateB.results.all(projectB) match { + case Compiler.Result.Failed(problemsPerPhase, crash, _, _, _) => + assert(problemsPerPhase == List.empty) // No new errors should be found + assert(crash == None) + case result => fail(s"Result ${result} is not classified as failure") + } + } + } + } + + test("best-effort: regain artifacts after disconnecting and reconnecting to the client") { + TestUtil.withinWorkspace { workspace => + val `A` = TestProject( + workspace, + "A", + dummyBestEffortSources, + scalaVersion = Some(bestEffortScalaVersion) + ) + val `B` = TestProject( + workspace, + "B", + dummyBestEffortDepSources, + directDependencies = List(`A`), + scalaVersion = Some(bestEffortScalaVersion) + ) + val projects = List(`A`, `B`) + TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) + val extraParams = BloopExtraBuildParams( + ownsBuildFiles = None, + clientClassesRootDir = None, + semanticdbVersion = Some(semanticdbVersion), + supportedScalaVersions = Some(List(bestEffortScalaVersion)), + javaSemanticdbVersion = None, + enableBestEffortMode = Some(true) + ) + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + val compiledStateA = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState + val compiledStateB = state.compile(`B`, arguments = Some(List("--best-effort"))).toTestState + } + loadBspState( + workspace, + projects, + logger, + "Metals reconnected", + bloopExtraParams = extraParams + ) { state => + val compiledStateA = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState + assertSemanticdbFileFor("TypeError.scala", compiledStateA, "A") + assertBetastyFile("TypeError.betasty", compiledStateA, "A") + val compiledStateB = state.compile(`B`, arguments = Some(List("--best-effort"))).toTestState + assertSemanticdbFileFor("TypeErrorDependency.scala", compiledStateB, "B") + assertBetastyFile("TypeErrorDependency.betasty", compiledStateB, "B") + state.findBuildTarget(`A`) + } + } + } + test("compile is successful with semanticDB and javac processorpath") { TestUtil.withinWorkspace { workspace => val logger = new RecordingLogger(ansiCodesSupported = false) @@ -516,7 +620,8 @@ class BspMetalsClientSpec( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), supportedScalaVersions = Some(List(testedScalaVersion)), - javaSemanticdbVersion = Some(javaSemanticdbVersion) + javaSemanticdbVersion = Some(javaSemanticdbVersion), + None ) loadBspBuildFromResources(projectName, workspace, logger, "Metals", extraParams) { build => @@ -578,6 +683,19 @@ class BspMetalsClientSpec( private val dummyFooScalaAndBarJavaSources = dummyFooScalaSources ++ dummyBarJavaSources + private val dummyBestEffortSources = List( + """/TypeError.scala + |object TypeError: + | val num: Int = "" + |""".stripMargin + ) + private val dummyBestEffortDepSources = List( + """/TypeErrorDependency.scala + |object TypeErrorDependency: + | def num(): Int = TypeError.num + |""".stripMargin + ) + private def assertSemanticdbFileForProject( sourceFileName: String, state: TestState, @@ -597,26 +715,39 @@ class BspMetalsClientSpec( classesDir.resolve(s"META-INF/semanticdb/src/$sourceFileName.semanticdb") } - private def semanticdbFile(sourceFileName: String, state: TestState) = { - val projectA = state.build.getProjectFor("A").get + private def semanticdbFile(sourceFileName: String, state: TestState, projectName: String) = { + val projectA = state.build.getProjectFor(projectName).get val classesDir = state.client.getUniqueClassesDirFor(projectA, forceGeneration = true) val sourcePath = if (sourceFileName.startsWith("/")) sourceFileName else s"/$sourceFileName" - classesDir.resolve(s"META-INF/semanticdb/A/src/$sourcePath.semanticdb") + classesDir.resolve(s"META-INF/semanticdb/$projectName/src/$sourcePath.semanticdb") + } + + private def assertBetastyFile( + expectedBetastyRelativePath: String, + state: TestState, + projectName: String + ): Unit = { + val project = state.build.getProjectFor(projectName).get + val classesDir = state.client.getUniqueClassesDirFor(project, forceGeneration = true) + val beTastyFile = classesDir.resolve(s"META-INF/best-effort/$expectedBetastyRelativePath") + assertIsFile(beTastyFile) } private def assertSemanticdbFileFor( sourceFileName: String, - state: TestState + state: TestState, + projectName: String = "A" ): Unit = { - val file = semanticdbFile(sourceFileName, state) + val file = semanticdbFile(sourceFileName, state, projectName) assertIsFile(file) } private def assertNoSemanticdbFileFor( sourceFileName: String, - state: TestState + state: TestState, + projectName: String = "A" ): Unit = { - val file = semanticdbFile(sourceFileName, state) + val file = semanticdbFile(sourceFileName, state, projectName) assertNotFile(file) } diff --git a/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala b/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala index 240c3d5837..62b12cdcf0 100644 --- a/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala @@ -213,7 +213,8 @@ class BspProtocolSpec( Some(Uri(userClientClassesRootDir.toBspUri)), semanticdbVersion = None, supportedScalaVersions = None, - javaSemanticdbVersion = None + javaSemanticdbVersion = None, + enableBestEffortMode = None ) // Start first client and query for scalac options which creates client classes dirs diff --git a/frontend/src/test/scala/bloop/testing/BaseSuite.scala b/frontend/src/test/scala/bloop/testing/BaseSuite.scala index 3bb9aa6414..f49e4dcb52 100644 --- a/frontend/src/test/scala/bloop/testing/BaseSuite.scala +++ b/frontend/src/test/scala/bloop/testing/BaseSuite.scala @@ -250,7 +250,7 @@ abstract class BaseSuite extends TestSuite with BloopHelpers { ): Unit = { if (errors > 0) { result match { - case Compiler.Result.Failed(problems, t, _, _) => + case Compiler.Result.Failed(problems, t, _, _, _) => val count = Problem.count(problems) if (count.errors == 0 && errors != 0) { // If there's an exception count it as one error diff --git a/frontend/src/test/scala/bloop/util/TestUtil.scala b/frontend/src/test/scala/bloop/util/TestUtil.scala index 4d54475290..406af67c72 100644 --- a/frontend/src/test/scala/bloop/util/TestUtil.scala +++ b/frontend/src/test/scala/bloop/util/TestUtil.scala @@ -409,6 +409,7 @@ object TestUtil { scalaInstance = scalaInstance, rawClasspath = classpath, resources = Nil, + isBestEffort = false, compileSetup = Config.CompileSetup.empty.copy(order = compileOrder), genericClassesDir = classes, scalacOptions = Nil, From 16e52830e9b49c58bbc2b38bddcda55de9e8d314 Mon Sep 17 00:00:00 2001 From: Jan Chyb Date: Sun, 23 Jun 2024 22:24:33 +0200 Subject: [PATCH 2/3] Fix outdated betasty files not being properly deleted from clientDir --- .../scala/bloop/BloopClassFileManager.scala | 8 +- backend/src/main/scala/bloop/Compiler.scala | 27 ++++--- .../src/main/scala/bloop/io/ParallelOps.scala | 9 ++- .../bloop/engine/tasks/CompileTask.scala | 2 +- .../scala/bloop/bsp/BspMetalsClientSpec.scala | 80 +++++++++++++++++++ .../scala/bloop/testing/BloopHelpers.scala | 5 +- 6 files changed, 114 insertions(+), 17 deletions(-) diff --git a/backend/src/main/scala/bloop/BloopClassFileManager.scala b/backend/src/main/scala/bloop/BloopClassFileManager.scala index 02ed3d639f..d98be2a2d3 100644 --- a/backend/src/main/scala/bloop/BloopClassFileManager.scala +++ b/backend/src/main/scala/bloop/BloopClassFileManager.scala @@ -41,7 +41,7 @@ final class BloopClassFileManager( private[this] val generatedFiles = new mutable.HashSet[File] // Supported compile products by the class file manager - private[this] val supportedCompileProducts = List(".sjsir", ".nir", ".tasty") + private[this] val supportedCompileProducts = List(".sjsir", ".nir", ".tasty", ".betasty") // Files backed up during compilation private[this] val movedFiles = new mutable.HashMap[File, File] @@ -215,7 +215,8 @@ final class BloopClassFileManager( clientTracer: BraveTracer ) => { clientTracer.traceTaskVerbose("copy new products to external classes dir") { _ => - val config = ParallelOps.CopyConfiguration(5, CopyMode.ReplaceExisting, Set.empty) + val config = + ParallelOps.CopyConfiguration(5, CopyMode.ReplaceExisting, Set.empty, Set.empty) val clientExternalBestEffortDir = clientExternalClassesDir.underlying.resolve("META-INF/best-effort") @@ -291,7 +292,8 @@ final class BloopClassFileManager( else clientTracer.traceTask("populate empty classes dir") { _ => // Prepopulate external classes dir even though compilation failed - val config = ParallelOps.CopyConfiguration(1, CopyMode.NoReplace, Set.empty) + val config = + ParallelOps.CopyConfiguration(1, CopyMode.NoReplace, Set.empty, Set.empty) ParallelOps .copyDirectories(config)( Paths.get(readOnlyClassesDirPath), diff --git a/backend/src/main/scala/bloop/Compiler.scala b/backend/src/main/scala/bloop/Compiler.scala index f804eccbc2..b86c50b0ee 100644 --- a/backend/src/main/scala/bloop/Compiler.scala +++ b/backend/src/main/scala/bloop/Compiler.scala @@ -402,15 +402,19 @@ object Compiler { val clientClassesDir = clientClassesObserver.classesDir clientLogger.debug(s"Triggering background tasks for $clientClassesDir") val firstTask = Task { BloopPaths.delete(AbsolutePath(newClassesDir)) } - val secondTask = updateExternalClassesDirWithReadOnly( - clientClassesDir, - clientTracer, - clientLogger, - compileInputs, + val config = + ParallelOps.CopyConfiguration( + parallelUnits = 5, + CopyMode.ReplaceIfMetadataMismatch, + denylist = Set.empty, + denyDirs = Set.empty + ) + val secondTask = ParallelOps.copyDirectories(config)( compileProducts.newClassesDir, - readOnlyCopyDenylist = mutable.HashSet.empty, - allInvalidatedClassFilesForProject, - allInvalidatedExtraCompileProducts + clientClassesDir.underlying, + compileInputs.ioScheduler, + enableCancellation = false, + compileInputs.logger ) Task .gatherUnordered(List(firstTask, secondTask)) @@ -716,8 +720,11 @@ object Compiler { allInvalidatedExtraCompileProducts.iterator.map(_.toPath).toSet val invalidatedInThisProject = invalidatedClassFiles ++ invalidatedExtraProducts val denyList = invalidatedInThisProject ++ readOnlyCopyDenylist.iterator + // Let's not copy outdated betasty from readOnly, since we do not have a mechanism + // for tracking that otherwise + val denyDir = Set(readOnlyClassesDir.resolve("META-INF/best-effort")) val config = - ParallelOps.CopyConfiguration(5, CopyMode.ReplaceIfMetadataMismatch, denyList) + ParallelOps.CopyConfiguration(5, CopyMode.ReplaceIfMetadataMismatch, denyList, denyDir) val lastCopy = ParallelOps.copyDirectories(config)( readOnlyClassesDir, clientClassesDir.underlying, @@ -934,6 +941,8 @@ object Compiler { syntax.replace(readOnlyClassesDirPath, clientClassesDirPath) ) if (rebasedFile.exists) { + // In practice this deletes previous successful compilation artifacts + // (like '.tasty' and ".class"), which we do not need in clientDir Files.delete(rebasedFile.underlying) } } diff --git a/backend/src/main/scala/bloop/io/ParallelOps.scala b/backend/src/main/scala/bloop/io/ParallelOps.scala index 49b7709788..ae50e6c055 100644 --- a/backend/src/main/scala/bloop/io/ParallelOps.scala +++ b/backend/src/main/scala/bloop/io/ParallelOps.scala @@ -44,7 +44,8 @@ object ParallelOps { case class CopyConfiguration private ( parallelUnits: Int, mode: CopyMode, - denylist: Set[Path] + denylist: Set[Path], + denyDirs: Set[Path] ) case class FileWalk(visited: List[Path], target: List[Path]) @@ -87,7 +88,11 @@ object ParallelOps { def visitFile(file: Path, attributes: BasicFileAttributes): FileVisitResult = { if (isCancelled.get) FileVisitResult.TERMINATE else { - if (attributes.isDirectory || configuration.denylist.contains(file)) () + if ( + attributes.isDirectory || configuration.denylist.contains( + file + ) || configuration.denyDirs.find(file.startsWith(_)).isDefined + ) () else { val rebasedFile = currentTargetDirectory.resolve(file.getFileName) if (configuration.denylist.contains(rebasedFile)) () diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index 89937ba391..631f3f4022 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -365,7 +365,7 @@ object CompileTask { } else { // Denylist ensure final dir doesn't contain class files that don't map to source files val denylist = products.invalidatedCompileProducts.iterator.map(_.toPath).toSet - val config = ParallelOps.CopyConfiguration(5, CopyMode.NoReplace, denylist) + val config = ParallelOps.CopyConfiguration(5, CopyMode.NoReplace, denylist, Set.empty) val task = tracer.traceTaskVerbose("preparing new read-only classes directory") { _ => ParallelOps.copyDirectories(config)( products.readOnlyClassesDir, diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index ca2807f10b..c62ceaddea 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -23,6 +23,7 @@ import bloop.task.Task import bloop.util.TestProject import bloop.util.TestUtil import bloop.Compiler +import java.nio.file.Files object LocalBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Local) object TcpBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Tcp) @@ -607,6 +608,74 @@ class BspMetalsClientSpec( } } + test("best-effort: correctly manage betasty files when compiling correct and failing projects") { + val initFile = + """/ErrorFile.scala + |object A + |object B + |""".stripMargin + val updatedFile1WithError = + """|object A + |//object B + |error + |object C + |""".stripMargin + val updatedFile2WithoutError = + """|//object A + |object B + |//error + |object C + |""".stripMargin + val updatedFile3WithError = + """|//object A + |object B + |error + |//object C + |""".stripMargin + + TestUtil.withinWorkspace { workspace => + val `A` = TestProject( + workspace, + "A", + List(initFile), + scalaVersion = Some(bestEffortScalaVersion) + ) + def updateProject(content: String) = + Files.write(`A`.config.sources.head.resolve("ErrorFile.scala"), content.getBytes()) + val projects = List(`A`) + TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) + val extraParams = BloopExtraBuildParams( + ownsBuildFiles = None, + clientClassesRootDir = None, + semanticdbVersion = Some(semanticdbVersion), + supportedScalaVersions = Some(List(bestEffortScalaVersion)), + javaSemanticdbVersion = None, + enableBestEffortMode = Some(true) + ) + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + val compiledState = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState + assertBetastyFile("A.betasty", compiledState, "A") + assertBetastyFile("B.betasty", compiledState, "A") + updateProject(updatedFile1WithError) + val compiledState2 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState + assertBetastyFile("A.betasty", compiledState2, "A") + assertNoBetastyFile("B.betasty", compiledState2, "A") + assertBetastyFile("C.betasty", compiledState2, "A") + updateProject(updatedFile2WithoutError) + val compiledState3 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState + assertNoBetastyFile("A.betasty", compiledState3, "A") + assertBetastyFile("B.betasty", compiledState3, "A") + assertBetastyFile("C.betasty", compiledState3, "A") + updateProject(updatedFile3WithError) + val compiledState4 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState + assertNoBetastyFile("A.betasty", compiledState4, "A") + assertBetastyFile("B.betasty", compiledState4, "A") + assertNoBetastyFile("C.betasty", compiledState4, "A") + } + } + } + test("compile is successful with semanticDB and javac processorpath") { TestUtil.withinWorkspace { workspace => val logger = new RecordingLogger(ansiCodesSupported = false) @@ -733,6 +802,17 @@ class BspMetalsClientSpec( assertIsFile(beTastyFile) } + private def assertNoBetastyFile( + expectedBetastyRelativePath: String, + state: TestState, + projectName: String + ): Unit = { + val project = state.build.getProjectFor(projectName).get + val classesDir = state.client.getUniqueClassesDirFor(project, forceGeneration = true) + val beTastyFile = classesDir.resolve(s"META-INF/best-effort/$expectedBetastyRelativePath") + assertNotFile(beTastyFile) + } + private def assertSemanticdbFileFor( sourceFileName: String, state: TestState, diff --git a/frontend/src/test/scala/bloop/testing/BloopHelpers.scala b/frontend/src/test/scala/bloop/testing/BloopHelpers.scala index 47c2f387f8..b6cd52789f 100644 --- a/frontend/src/test/scala/bloop/testing/BloopHelpers.scala +++ b/frontend/src/test/scala/bloop/testing/BloopHelpers.scala @@ -83,7 +83,7 @@ trait BloopHelpers { val baseDir = sourceConfigDir.getParent val relativeConfigDir = RelativePath(sourceConfigDir.getFileName) - val config = ParallelOps.CopyConfiguration(5, CopyMode.ReplaceExisting, Set.empty) + val config = ParallelOps.CopyConfiguration(5, CopyMode.ReplaceExisting, Set.empty, Set.empty) val copyToNewWorkspace = ParallelOps.copyDirectories(config)( baseDir, workspace.underlying, @@ -302,7 +302,8 @@ trait BloopHelpers { } val backupDir = ParallelOps.copyDirectories( - ParallelOps.CopyConfiguration(2, ParallelOps.CopyMode.ReplaceExisting, Set.empty) + ParallelOps + .CopyConfiguration(2, ParallelOps.CopyMode.ReplaceExisting, Set.empty, Set.empty) )( classesDir, newClassesDir, From 48b9136360131adaf04ad4c8a029965d5233e5c0 Mon Sep 17 00:00:00 2001 From: Jan Chyb Date: Wed, 26 Jun 2024 17:45:09 +0200 Subject: [PATCH 3/3] Make best effort file management behavior clearer --- .../scala/bloop/BloopClassFileManager.scala | 9 ++-- backend/src/main/scala/bloop/Compiler.scala | 45 +++++++++---------- .../scala/bloop/bsp/BspMetalsClientSpec.scala | 42 +++++++++++++---- 3 files changed, 61 insertions(+), 35 deletions(-) diff --git a/backend/src/main/scala/bloop/BloopClassFileManager.scala b/backend/src/main/scala/bloop/BloopClassFileManager.scala index d98be2a2d3..3500a0d459 100644 --- a/backend/src/main/scala/bloop/BloopClassFileManager.scala +++ b/backend/src/main/scala/bloop/BloopClassFileManager.scala @@ -40,8 +40,6 @@ final class BloopClassFileManager( private[this] val weakClassFileInvalidations = new mutable.HashSet[Path]() private[this] val generatedFiles = new mutable.HashSet[File] - // Supported compile products by the class file manager - private[this] val supportedCompileProducts = List(".sjsir", ".nir", ".tasty", ".betasty") // Files backed up during compilation private[this] val movedFiles = new mutable.HashMap[File, File] @@ -140,7 +138,7 @@ final class BloopClassFileManager( val invalidatedExtraCompileProducts = classes.flatMap { classFile => val prefixClassName = classFile.getName().stripSuffix(".class") - supportedCompileProducts.flatMap { supportedProductSuffix => + BloopClassFileManager.supportedCompileProducts.flatMap { supportedProductSuffix => val productName = prefixClassName + supportedProductSuffix val productAssociatedToClassFile = new File(classFile.getParentFile, productName) if (!productAssociatedToClassFile.exists()) Nil @@ -186,7 +184,7 @@ final class BloopClassFileManager( .stripSuffix(".class") + supportedProductSuffix new File(classFile.getParentFile, productName) } - supportedCompileProducts.foreach { supportedProductSuffix => + BloopClassFileManager.supportedCompileProducts.foreach { supportedProductSuffix => val generatedProductName = productFile(generatedClassFile, supportedProductSuffix) val rebasedProductName = productFile(rebasedClassFile, supportedProductSuffix) @@ -318,6 +316,9 @@ final class BloopClassFileManager( } object BloopClassFileManager { + // Supported compile products by the class file manager + val supportedCompileProducts = List(".sjsir", ".nir", ".tasty", ".betasty") + def link(link: Path, target: Path): Try[Unit] = { Try { // Make sure parent directory for link exists diff --git a/backend/src/main/scala/bloop/Compiler.scala b/backend/src/main/scala/bloop/Compiler.scala index b86c50b0ee..677401e767 100644 --- a/backend/src/main/scala/bloop/Compiler.scala +++ b/backend/src/main/scala/bloop/Compiler.scala @@ -376,10 +376,10 @@ object Compiler { t, elapsed, _, - bestEffortProducts @ Some(BestEffortProducts(compileProducts, previousHash)) + bestEffortProducts @ Some(BestEffortProducts(previousCompilationResults, previousHash)) ) if isBestEffortMode => val newHash = BestEffortUtils.hashResult( - compileProducts.newClassesDir, + previousCompilationResults.newClassesDir, compileInputs.sources, compileInputs.classpath ) @@ -401,7 +401,18 @@ object Compiler { ): Task[Unit] = Task.defer { val clientClassesDir = clientClassesObserver.classesDir clientLogger.debug(s"Triggering background tasks for $clientClassesDir") + + // First, we delete newClassesDir, as it was created to store + // new compilation artifacts coming from scalac, which we will not + // have in this case and it's going to remain empty. val firstTask = Task { BloopPaths.delete(AbsolutePath(newClassesDir)) } + + // Then we copy previous best effort artifacts to a clientDir from the + // cached compilation result. + // This is useful if e.g. the client restarted after the last compilation + // and was assigned a new, empty directory. Since best-effort currently does + // not support incremental compilation, all necessary betasty files will come + // from a single previous compilation run, so that is all we need to copy. val config = ParallelOps.CopyConfiguration( parallelUnits = 5, @@ -410,7 +421,7 @@ object Compiler { denyDirs = Set.empty ) val secondTask = ParallelOps.copyDirectories(config)( - compileProducts.newClassesDir, + previousCompilationResults.newClassesDir, clientClassesDir.underlying, compileInputs.ioScheduler, enableCancellation = false, @@ -878,7 +889,6 @@ object Compiler { ): Result = { val uniqueInputs = compileInputs.uniqueInputs val readOnlyClassesDir = compileOut.internalReadOnlyClassesDir.underlying - val readOnlyClassesDirPath = readOnlyClassesDir.toString val newClassesDir = compileOut.internalNewClassesDir.underlying reporter.processEndCompilation( @@ -903,10 +913,6 @@ object Compiler { Map.empty ) - // Delete all those class files that were invalidated in the external classes dir - val allInvalidated = - allInvalidatedClassFilesForProject ++ allInvalidatedExtraCompileProducts - val backgroundTasksExecution = new CompileBackgroundTasks { def trigger( clientClassesObserver: ClientClassesObserver, @@ -915,7 +921,6 @@ object Compiler { clientLogger: Logger ): Task[Unit] = { val clientClassesDir = clientClassesObserver.classesDir - val clientClassesDirPath = clientClassesDir.toString val successBackgroundTasks = backgroundTasksWhenNewSuccessfulAnalysis .map(f => f(clientClassesDir, clientReporter, clientTracer)) @@ -933,20 +938,14 @@ object Compiler { ) val secondTask = Task { - allInvalidated.foreach { f => - val path = AbsolutePath(f.toPath) - val syntax = path.syntax - if (syntax.startsWith(readOnlyClassesDirPath)) { - val rebasedFile = AbsolutePath( - syntax.replace(readOnlyClassesDirPath, clientClassesDirPath) - ) - if (rebasedFile.exists) { - // In practice this deletes previous successful compilation artifacts - // (like '.tasty' and ".class"), which we do not need in clientDir - Files.delete(rebasedFile.underlying) - } - } - } + // Delete everything outside of betasty and semanticdb + val deletedCompileProducts = + BloopClassFileManager.supportedCompileProducts.filter(_ != ".betasty") :+ ".class" + Files + .walk(clientClassesDir.underlying) + .filter(path => Files.isRegularFile(path)) + .filter(path => deletedCompileProducts.exists(path.toString.endsWith(_))) + .forEach(Files.delete(_)) } Task .gatherUnordered(List(firstTask, secondTask)) diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index c62ceaddea..4e65ef357c 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -657,21 +657,25 @@ class BspMetalsClientSpec( val compiledState = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState assertBetastyFile("A.betasty", compiledState, "A") assertBetastyFile("B.betasty", compiledState, "A") + assertCompilationFile("A.class", compiledState, "A") updateProject(updatedFile1WithError) val compiledState2 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState assertBetastyFile("A.betasty", compiledState2, "A") assertNoBetastyFile("B.betasty", compiledState2, "A") assertBetastyFile("C.betasty", compiledState2, "A") + assertNoCompilationFile("A.class", compiledState, "A") updateProject(updatedFile2WithoutError) val compiledState3 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState assertNoBetastyFile("A.betasty", compiledState3, "A") assertBetastyFile("B.betasty", compiledState3, "A") assertBetastyFile("C.betasty", compiledState3, "A") + assertCompilationFile("B.class", compiledState, "A") updateProject(updatedFile3WithError) val compiledState4 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState assertNoBetastyFile("A.betasty", compiledState4, "A") assertBetastyFile("B.betasty", compiledState4, "A") assertNoBetastyFile("C.betasty", compiledState4, "A") + assertNoCompilationFile("B.class", compiledState, "A") } } } @@ -791,26 +795,48 @@ class BspMetalsClientSpec( classesDir.resolve(s"META-INF/semanticdb/$projectName/src/$sourcePath.semanticdb") } - private def assertBetastyFile( - expectedBetastyRelativePath: String, + private def assertCompilationFile( + expectedFilePath: String, state: TestState, projectName: String ): Unit = { val project = state.build.getProjectFor(projectName).get val classesDir = state.client.getUniqueClassesDirFor(project, forceGeneration = true) - val beTastyFile = classesDir.resolve(s"META-INF/best-effort/$expectedBetastyRelativePath") - assertIsFile(beTastyFile) + assertIsFile(classesDir.resolve(expectedFilePath)) } - private def assertNoBetastyFile( - expectedBetastyRelativePath: String, + private def assertNoCompilationFile( + expectedFilePath: String, state: TestState, projectName: String ): Unit = { val project = state.build.getProjectFor(projectName).get val classesDir = state.client.getUniqueClassesDirFor(project, forceGeneration = true) - val beTastyFile = classesDir.resolve(s"META-INF/best-effort/$expectedBetastyRelativePath") - assertNotFile(beTastyFile) + assertNotFile(classesDir.resolve(expectedFilePath)) + } + + private def assertBetastyFile( + expectedBetastyRelativePath: String, + state: TestState, + projectName: String + ): Unit = { + assertCompilationFile( + s"META-INF/best-effort/$expectedBetastyRelativePath", + state, + projectName + ) + } + + private def assertNoBetastyFile( + expectedBetastyRelativePath: String, + state: TestState, + projectName: String + ): Unit = { + assertNoCompilationFile( + s"META-INF/best-effort/$expectedBetastyRelativePath", + state, + projectName + ) } private def assertSemanticdbFileFor(