Fix incremental compilation when a Scala project depends on a Java pr…

…oject (#414)

* Upgrade ammonite to 1.1.2-30-53edc31

This is mainly to get com-lihaoyi/Ammonite#851 which
should reduce the amount of unnecessary work done by incremental
compilation in the Mill build. This requires some code changes since
this means we now depend on a more recent version of coursier, as a
side-effect this means that we do not depend on scalaz anymore.

Also use the same ammonite version in the Mill build and in
ScalaModule#ammoniteReplClasspath.

Also remove an incorrect dependency in the caffeine integration test.
This was always wrong but did not start failing until this commit,
probably due to dependencies appearing in a different order on the
classpath.

* Rename ScalaWorker to ZincWorker

Starting with the next commit, it will be used in Java-only projects
too, so the name is misleading.

* Upgrade to Zinc 1.2.1

* Fix incremental compilation when a Scala project depends on a Java project

Before this commit, JavaModule#compile simply called javac
unconditionally, thus generating new classfiles every time. But if a
Scala project depends on a Java project, this will throw off the
incremental compilation algorithm which will unnecessarily recompile
files. To avoid this we now use Zinc to compile Java projects too (as a
bonus this means that Java compilation becomes incremental). This
required some refactoring in ZincWorkerImpl to be able to compile stuff
without having to pass Scala-specific options.

The issue solved by this commit could be reproduced by running in the
Mill repository:

$ mill main.compile
$ mill -i
@ main.compile()

and observing that before this commit, the `main.compile()` call ended
up recompiling code.

build.sc

@@ -85,7 +85,8 @@ object main extends MillModule {
     )
 
     def ivyDeps = Agg(
-      ivy"com.lihaoyi:::ammonite:1.1.2-6-27842d9",
+      // Keep synchronized with ammonite in Versions.scala
+      ivy"com.lihaoyi:::ammonite:1.1.2-30-53edc31",
       // Necessary so we can share the JNA classes throughout the build process
       ivy"net.java.dev.jna:jna:4.5.0",
       ivy"net.java.dev.jna:jna-platform:4.5.0"
@@ -176,7 +177,7 @@ object scalalib extends MillModule {
 
     def ivyDeps = Agg(
       // Keep synchronized with zinc in Versions.scala
-      ivy"org.scala-sbt::zinc:1.1.7"
+      ivy"org.scala-sbt::zinc:1.2.1"
     )
     def testArgs = Seq(
       "-DMILL_SCALA_WORKER=" + runClasspath().map(_.path).mkString(",")

docs/pages/2 - Configuring Mill.md

@@ -72,20 +72,20 @@ def repositories = super.repositories ++ Seq(
 ```
 
 To add custom resolvers to the initial bootstrap of the build, you can create a 
-custom `ScalaWorkerModule`, and override the `scalaWorker` method in your 
+custom `ZincWorkerModule`, and override the `zincWorker` method in your 
 `ScalaModule` by pointing it to that custom object:
 
 ```scala
 import coursier.maven.MavenRepository
 
-object CustomScalaWorkerModule extends ScalaWorkerModule {
+object CustomZincWorkerModule extends ZincWorkerModule {
   def repositories() = super.repositories ++ Seq(
     MavenRepository("https://oss.sonatype.org/content/repositories/releases")
   )  
 }
 
 object YourBuild extends ScalaModule {
-  def scalaWorker = CustomScalaWorkerModule
+  def zincWorker = CustomZincWorkerModule
   // ... rest of your build definitions
 }
 ```

docs/pages/5 - Modules.md

@@ -152,7 +152,7 @@ mill foo.Bar/qux
 
 `ExternalModule`s are useful for someone providing a library for use with Mill
 that is shared by the entire build: for example,
-`mill.scalalib.ScalaWorkerApi/scalaWorker` provides a shared Scala compilation
+`mill.scalalib.ZincWorkerApi/zincWorker` provides a shared Scala compilation
 service & cache that is shared between all `ScalaModule`s, and
 `mill.scalalib.GenIdea/idea` lets you generate IntelliJ projects without
 needing to define your own `T.command` in your `build.sc` file

integration/test/resources/caffeine/build.sc

@@ -8,11 +8,11 @@ import deps.{benchmarkLibraries, benchmarkVersions, libraries, testLibraries, te
 
 trait CaffeineModule extends MavenModule{
   def repositories = super.repositories ++ Seq(
-    coursier.ivy.IvyRepository(
+    coursier.ivy.IvyRepository.parse(
       "https://dl.bintray.com/sbt/sbt-plugin-releases/" +
       coursier.ivy.Pattern.default.string,
       dropInfoAttributes = true
-    ),
+    ).toOption.get,
     MavenRepository("https://jcenter.bintray.com/"),
     MavenRepository("https://jitpack.io/"),
     MavenRepository("http://repo.spring.io/plugins-release")
@@ -25,7 +25,6 @@ trait CaffeineModule extends MavenModule{
       libraries.guava,
       testLibraries.mockito,
       testLibraries.hamcrest,
-      ivy"org.hamcrest:hamcrest-library:1.3",
       testLibraries.awaitility,
     ) ++
       testLibraries.testng ++
@@ -151,4 +150,4 @@ object simulator extends CaffeineModule {
 
     def ivyDeps = super.ivyDeps() ++ testLibraries.testng
   }
-}
+}

main/src/mill/modules/Jvm.scala

@@ -10,6 +10,7 @@ import java.util.jar.{JarEntry, JarFile, JarOutputStream}
 
 import ammonite.ops._
 import coursier.{Cache, Dependency, Fetch, Repository, Resolution}
+import coursier.util.{Gather, Task}
 import geny.Generator
 import mill.main.client.InputPumper
 import mill.eval.{PathRef, Result}
@@ -383,7 +384,7 @@ object Jvm {
   /**
     * Resolve dependencies using Coursier.
     *
-    * We do not bother breaking this out into the separate ScalaWorker classpath,
+    * We do not bother breaking this out into the separate ZincWorkerApi classpath,
     * because Coursier is already bundled with mill/Ammonite to support the
     * `import $ivy` syntax.
     */
@@ -413,17 +414,19 @@ object Jvm {
 
       def load(artifacts: Seq[coursier.Artifact]) = {
         val logger = None
-        val loadedArtifacts = scalaz.concurrent.Task.gatherUnordered(
+
+        import scala.concurrent.ExecutionContext.Implicits.global
+        val loadedArtifacts = Gather[Task].gather(
           for (a <- artifacts)
-            yield coursier.Cache.file(a, logger = logger).run
+            yield coursier.Cache.file[Task](a, logger = logger).run
               .map(a.isOptional -> _)
-        ).unsafePerformSync
+        ).unsafeRun
 
         val errors = loadedArtifacts.collect {
-          case (false, scalaz.-\/(x)) => x
-          case (true, scalaz.-\/(x)) if !x.notFound => x
+          case (false, Left(x)) => x
+          case (true, Left(x)) if !x.notFound => x
         }
-        val successes = loadedArtifacts.collect { case (_, scalaz.\/-(x)) => x }
+        val successes = loadedArtifacts.collect { case (_, Right(x)) => x }
         (errors, successes)
       }
 
@@ -459,8 +462,10 @@ object Jvm {
       mapDependencies = mapDependencies
     )
 
-    val fetch = Fetch.from(repositories, Cache.fetch())
-    val resolution = start.process.run(fetch).unsafePerformSync
+    val fetch = Fetch.from(repositories, Cache.fetch[Task]())
+
+    import scala.concurrent.ExecutionContext.Implicits.global
+    val resolution = start.process.run(fetch).unsafeRun()
     (deps.toSeq, resolution)
   }
 }

scalajslib/src/mill/scalajslib/ScalaJSModule.scala

@@ -15,7 +15,7 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer =>
   def scalaJSVersion: T[String]
 
   trait Tests extends TestScalaJSModule {
-    override def scalaWorker = outer.scalaWorker
+    override def zincWorker = outer.zincWorker
     override def scalaOrganization = outer.scalaOrganization()
     override def scalaVersion = outer.scalaVersion()
     override def scalaJSVersion = outer.scalaJSVersion()

scalalib/src/mill/scalalib/JavaModule.scala

@@ -17,12 +17,13 @@ import mill.util.Loose.Agg
   * Core configuration required to compile a single Scala compilation target
   */
 trait JavaModule extends mill.Module with TaskModule { outer =>
-  def scalaWorker: ScalaWorkerModule = mill.scalalib.ScalaWorkerModule
+  def zincWorker: ZincWorkerModule = mill.scalalib.ZincWorkerModule
 
   trait Tests extends TestModule{
     override def moduleDeps = Seq(outer)
     override def repositories = outer.repositories
     override def javacOptions = outer.javacOptions
+    override def zincWorker = outer.zincWorker
   }
   def defaultCommandName() = "run"
 
@@ -38,7 +39,16 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
   def finalMainClassOpt: T[Either[String, String]] = T{
     mainClass() match{
       case Some(m) => Right(m)
-      case None => Left("No main class specified or found")
+      case None =>
+        zincWorker.worker().discoverMainClasses(compile())match {
+          case Seq() => Left("No main class specified or found")
+          case Seq(main) => Right(main)
+          case mains =>
+            Left(
+              s"Multiple main classes found (${mains.mkString(",")}) " +
+                "please explicitly specify which one to use by overriding mainClass"
+            )
+        }
     }
   }
 
@@ -98,7 +108,7 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
   }
 
 
-  def repositories: Seq[Repository] = scalaWorker.repositories
+  def repositories: Seq[Repository] = zincWorker.repositories
 
   def platformSuffix = T{ "" }
 
@@ -133,12 +143,12 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
     } yield PathRef(path)
   }
 
-  def compile: T[CompilationResult] = T{
-    Lib.compileJava(
-      allSourceFiles().map(_.path.toIO).toArray,
-      compileClasspath().map(_.path.toIO).toArray,
-      javacOptions(),
-      upstreamCompileOutput()
+  def compile: T[CompilationResult] = T.persistent{
+    zincWorker.worker().compileJava(
+      upstreamCompileOutput(),
+      allSourceFiles().map(_.path),
+      compileClasspath().map(_.path),
+      javacOptions()
     )
   }
 
@@ -317,7 +327,7 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
     val (procId, procTombstone, token) = backgroundSetup(T.ctx().dest)
     try Result.Success(Jvm.interactiveSubprocess(
       "mill.scalalib.backgroundwrapper.BackgroundWrapper",
-      (runClasspath() ++ scalaWorker.backgroundWrapperClasspath()).map(_.path),
+      (runClasspath() ++ zincWorker.backgroundWrapperClasspath()).map(_.path),
       forkArgs(),
       forkEnv(),
       Seq(procId.toString, procTombstone.toString, token, finalMainClass()) ++ args,
@@ -332,7 +342,7 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
     val (procId, procTombstone, token) = backgroundSetup(T.ctx().dest)
     try Result.Success(Jvm.interactiveSubprocess(
       "mill.scalalib.backgroundwrapper.BackgroundWrapper",
-      (runClasspath() ++ scalaWorker.backgroundWrapperClasspath()).map(_.path),
+      (runClasspath() ++ zincWorker.backgroundWrapperClasspath()).map(_.path),
       forkArgs(),
       forkEnv(),
       Seq(procId.toString, procTombstone.toString, token, mainClass) ++ args,
@@ -384,7 +394,7 @@ trait TestModule extends JavaModule with TaskModule {
 
     Jvm.subprocess(
       mainClass = "mill.scalalib.TestRunner",
-      classPath = scalaWorker.scalalibClasspath().map(_.path),
+      classPath = zincWorker.scalalibClasspath().map(_.path),
       jvmArgs = forkArgs(),
       envArgs = forkEnv(),
       mainArgs =

scalalib/src/mill/scalalib/Lib.scala

@@ -27,34 +27,6 @@ object CompilationResult {
 case class CompilationResult(analysisFile: Path, classes: PathRef)
 
 object Lib{
-  def compileJava(sources: Array[java.io.File],
-                  classpath: Array[java.io.File],
-                  javaOpts: Seq[String],
-                  upstreamCompileOutput: Seq[CompilationResult])
-                 (implicit ctx: mill.util.Ctx) = {
-    val javac = ToolProvider.getSystemJavaCompiler()
-    if (javac == null) {
-      throw new Exception(
-        "Your Java installation is not a JDK, so it can't compile Java code;" +
-        " Please install the JDK version of Java")
-    }
-
-    rm(ctx.dest / 'classes)
-    mkdir(ctx.dest / 'classes)
-    val cpArgs =
-      if(classpath.isEmpty) Seq()
-      else Seq("-cp", classpath.mkString(File.pathSeparator))
-
-    val args = Seq("-d", ctx.dest / 'classes) ++ cpArgs ++ javaOpts ++ sources
-
-    javac.run(
-      ctx.log.inStream, ctx.log.outputStream, ctx.log.errorStream,
-      args.map(_.toString):_*
-    )
-    if (ls(ctx.dest / 'classes).isEmpty) mill.eval.Result.Failure("Compilation Failed")
-    else mill.eval.Result.Success(CompilationResult(ctx.dest / 'zinc, PathRef(ctx.dest / 'classes)))
-  }
-
   private val ReleaseVersion = raw"""(\d+)\.(\d+)\.(\d+)""".r
   private val MinorSnapshotVersion = raw"""(\d+)\.(\d+)\.([1-9]\d*)-SNAPSHOT""".r
   private val DottyVersion = raw"""0\.(\d+)\.(\d+).*""".r
@@ -104,7 +76,7 @@ object Lib{
   /**
     * Resolve dependencies using Coursier.
     *
-    * We do not bother breaking this out into the separate ScalaWorker classpath,
+    * We do not bother breaking this out into the separate ZincWorker classpath,
     * because Coursier is already bundled with mill/Ammonite to support the
     * `import $ivy` syntax.
     */