More granular cache invalidaton with multiple file

bsp/src/mill/bsp/BSP.scala

@@ -129,6 +129,7 @@ object BSP extends ExternalModule {
         ev.externalOutPath,
         ev.rootModule,
         ev.baseLogger,
+        ev.importTree,
         ev.classLoaderSig,
         ev.workerCache,
         ev.env,

integration/test/resources/invalidation/a/inputA.sc

@@ -0,0 +1,3 @@
+def input = T {
+  println("a")
+}

integration/test/resources/invalidation/b/inputB.sc

@@ -0,0 +1,5 @@
+import $file.inputD
+def input = T {
+  inputD.method()
+  println("b")
+}

integration/test/resources/invalidation/b/inputD.sc

@@ -0,0 +1,3 @@
+def method() = {
+  println("d")
+}

integration/test/resources/invalidation/build.sc

@@ -0,0 +1,10 @@
+import $file.a.inputA
+import $file.b.inputB
+import $file.inputC
+import $ivy.`org.scalaj::scalaj-http:2.4.2`
+
+def task = T {
+  inputA.input()
+  inputB.input()
+  inputC.input()
+}

integration/test/resources/invalidation/inputC.sc

@@ -0,0 +1,3 @@
+def input = T {
+  println("c")
+}

integration/test/src/ScriptsInvalidationTests.scala

@@ -0,0 +1,72 @@
+package mill.integration
+
+import mill.util.ScriptTestSuite
+import utest._
+
+import scala.collection.mutable
+
+class ScriptsInvalidationTests(fork: Boolean) extends ScriptTestSuite(fork) {
+  def workspaceSlug: String = "invalidation"
+  def scriptSourcePath: os.Path = os.pwd / "integration" / "test" / "resources" / workspaceSlug
+
+  val output = mutable.Buffer[String]()
+  val stdout = os.ProcessOutput((bytes, count) => output += new String(bytes, 0, count))
+  def runTask() = assert(eval(stdout, "task"))
+
+  override def utestBeforeEach(path: Seq[String]): Unit = {
+    output.clear()
+  }
+
+  val tests = Tests {
+    test("should not invalidate tasks in different untouched sc files") {
+      test("first run") {
+        initWorkspace()
+        runTask()
+
+        val result = output.map(_.trim).toList
+        val expected = List("a", "d", "b", "c")
+
+        assert(result == expected)
+      }
+
+      test("second run modifying script") {
+        val oldContent = os.read(scriptSourcePath / buildPath)
+        val newContent = s"""$oldContent
+                            |def newTask = T { }
+                            |""".stripMargin
+        os.write.over(workspacePath / buildPath, newContent)
+
+        runTask()
+
+        assert(output.isEmpty)
+      }
+    }
+    test("should invalidate tasks if leaf file is changed") {
+      test("first run") {
+        initWorkspace()
+        runTask()
+
+        val result = output.map(_.trim).toList
+        val expected = List("a", "d", "b", "c")
+
+        assert(result == expected)
+      }
+
+      test("second run modifying script") {
+        val inputD = os.sub / "b" / "inputD.sc"
+        val oldContent = os.read(scriptSourcePath / inputD)
+        val newContent = s"""$oldContent
+                            |def newTask = T { }
+                            |""".stripMargin
+        os.write.over(workspacePath / inputD, newContent)
+
+        runTask()
+
+        val result = output.map(_.trim).toList
+        val expected = List("d", "b")
+
+        assert(result == expected)
+      }
+    }
+  }
+}

integration/test/src/local/Tests.scala

@@ -10,3 +10,4 @@ object UpickleTests extends mill.integration.UpickleTests(fork = false)
 object PlayJsonTests extends mill.integration.PlayJsonTests(fork = false)
 object CaffeineTests extends mill.integration.CaffeineTests(fork = false)
 object DocAnnotationsTests extends mill.integration.DocAnnotationsTests(fork = false)
+object ScriptsInvalidationTests extends mill.integration.ScriptsInvalidationTests(fork = true)

main/core/src/mill/define/Graph.scala

@@ -35,23 +35,30 @@ object Graph {
     }
     output
   }
-
   /**
-   * Collects all transitive dependencies (targets) of the given targets,
-   * including the given targets.
+   * Collects all transitive dependencies (nodes) of the given nodes,
+   * including the given nodes.
    */
   def transitiveTargets(sourceTargets: Agg[Task[_]]): Agg[Task[_]] = {
-    val transitiveTargets = new Agg.Mutable[Task[_]]
-    def rec(t: Task[_]): Unit = {
-      if (transitiveTargets.contains(t)) () // do nothing
+    transitiveNodes(sourceTargets)
+  }
+
+  /**
+   * Collects all transitive dependencies (nodes) of the given nodes,
+   * including the given nodes.
+   */
+  def transitiveNodes[T <: GraphNode[T]](sourceNodes: Agg[T]): Agg[T] = {
+    val transitiveNodes = new Agg.Mutable[T]
+    def rec(t: T): Unit = {
+      if (transitiveNodes.contains(t)) () // do nothing
       else {
-        transitiveTargets.append(t)
+        transitiveNodes.append(t)
         t.inputs.foreach(rec)
       }
     }
 
-    sourceTargets.items.foreach(rec)
-    transitiveTargets
+    sourceNodes.items.foreach(rec)
+    transitiveNodes
   }
 
   /**

main/core/src/mill/define/GraphNode.scala

@@ -0,0 +1,8 @@
+package mill.define
+
+trait GraphNode[T] {
+  /**
+   * What other nodes does this node depend on?
+   */
+  def inputs: Seq[T]
+}

main/core/src/mill/define/ScriptNode.scala

@@ -0,0 +1,3 @@
+package mill.define
+
+case class ScriptNode(cls: String, inputs: Seq[ScriptNode]) extends GraphNode[ScriptNode]

main/core/src/mill/define/Task.scala

@@ -15,13 +15,7 @@ import scala.reflect.macros.blackbox.Context
  * Generally not instantiated manually, but instead constructed via the
  * [[Target.apply]] & similar macros.
  */
-abstract class Task[+T] extends Task.Ops[T] with Applyable[Task, T] {
-
-  /**
-   * What other Targets does this Target depend on?
-   */
-  val inputs: Seq[Task[_]]
-
+abstract class Task[+T] extends Task.Ops[T] with Applyable[Task, T] with GraphNode[Task[_]] {
   /**
    * Evaluate this target
    */

main/core/src/mill/eval/Evaluator.scala

@@ -52,19 +52,29 @@ case class Evaluator(
     externalOutPath: os.Path,
     rootModule: mill.define.BaseModule,
     baseLogger: ColorLogger,
+    importTree: Seq[ScriptNode],
     classLoaderSig: Seq[(Either[String, java.net.URL], Long)] = Evaluator.classLoaderSig,
     workerCache: mutable.Map[Segments, (Int, Any)] = mutable.Map.empty,
     env: Map[String, String] = Evaluator.defaultEnv,
     failFast: Boolean = true,
     threadCount: Option[Int] = Some(1)
 ) {
 
+  val (scriptsClassLoader, externalClassLoader) = classLoaderSig.partitionMap {
+    case (Right(elem), sig) => Right((elem, sig))
+    case (Left(elem), sig) => Left((elem, sig))
+  }
+
+  // We're interested of the whole file hash.
+  // So we sum the hash of both class and companion object (ends with `$`)
+  val scriptsSigMap = scriptsClassLoader.groupMapReduce(_._1.stripSuffix("$"))(_._2)(_ + _)
+
   val effectiveThreadCount: Int =
     this.threadCount.getOrElse(Runtime.getRuntime().availableProcessors())
 
   import Evaluator.Evaluated
 
-  val classLoaderSignHash = classLoaderSig.hashCode()
+  val externalClassLoaderSigHash = externalClassLoader.hashCode()
 
   val pathsResolver: EvaluatorPathsResolver = EvaluatorPathsResolver.default(outPath)
 
@@ -291,7 +301,23 @@ case class Evaluator(
       group.iterator.map(_.sideHash)
     )
 
-    val inputsHash = externalInputsHash + sideHashes + classLoaderSignHash
+    val scriptsHash = {
+      val cls = group.items.toList(1).asInstanceOf[NamedTask[_]].ctx.enclosingCls.getName
+      val classes = new Agg.Mutable[String]()
+      group.items.flatMap(i => i +: i.inputs.toSeq).foreach {
+        case namedTask: NamedTask[_] =>
+          // We don't care if it's the class of the companion object (class ends with `$`)
+          val cls = namedTask.ctx.enclosingCls.getName.stripSuffix("$")
+          classes.append(cls)
+        case _ =>
+      }
+      val importClasses = importTree.filter(e => classes.contains(e.cls))
+      val dependendentScripts = Graph.transitiveNodes(importClasses).map(_.cls)
+      val dependendentScriptsSig = dependendentScripts.map(s => s -> scriptsSigMap(s))
+      dependendentScriptsSig.hashCode()
+    }
+
+    val inputsHash = externalInputsHash + sideHashes + externalClassLoaderSigHash + scriptsHash
 
     terminal match {
       case Left(task) =>

main/src/mill/main/EvaluatorState.scala

@@ -2,12 +2,13 @@ package mill.main
 
 import scala.collection.mutable
 
-import mill.define.Segments
+import mill.define.{ScriptNode, Segments}
 
 case class EvaluatorState private[main] (
     rootModule: mill.define.BaseModule,
     classLoaderSig: Seq[(Either[String, java.net.URL], Long)],
     workerCache: mutable.Map[Segments, (Int, Any)],
     watched: Seq[(ammonite.interp.Watchable, Long)],
-    setSystemProperties: Set[String]
+    setSystemProperties: Set[String],
+    importTree: Seq[ScriptNode]
 )

main/src/mill/main/MainRunner.scala

@@ -129,7 +129,8 @@ class MainRunner(
               classLoaderSig = eval.classLoaderSig,
               workerCache = eval.workerCache,
               watched = interpWatched,
-              setSystemProperties = systemProperties.keySet
+              setSystemProperties = systemProperties.keySet,
+              importTree = eval.importTree
             ))
             val watched = () => {
               val alreadyStale = evalWatches.exists(p => p.sig != PathRef(p.path, p.quick).sig)

main/src/mill/main/ReplApplyHandler.scala

@@ -16,6 +16,7 @@ object ReplApplyHandler {
       colors: ammonite.util.Colors,
       pprinter0: pprint.PPrinter,
       rootModule: mill.define.BaseModule,
+      importTree: Seq[ScriptNode],
       discover: Discover[_],
       debugLog: Boolean,
       keepGoing: Boolean,
@@ -45,6 +46,7 @@ object ReplApplyHandler {
         os.pwd / "out",
         rootModule,
         logger,
+        importTree = importTree,
         failFast = !keepGoing,
         threadCount = threadCount
       ),

main/src/mill/main/RunScript.scala

@@ -64,15 +64,34 @@ object RunScript {
             interp.watch(path)
             val eval =
               for (rootModule <- evaluateRootModule(wd, path, interp, log))
-                yield EvaluatorState(
-                  rootModule,
-                  rootModule.getClass.getClassLoader.asInstanceOf[
-                    SpecialClassLoader
-                  ].classpathSignature,
-                  mutable.Map.empty[Segments, (Int, Any)],
-                  interp.watchedValues.toSeq,
-                  systemProperties.keySet
-                )
+                yield {
+                  val importTreeMap = interp.alreadyLoadedFiles.map { case (a, b) =>
+                    val filePath = a.filePathPrefix
+                    val importPaths = b.blockInfo.flatMap { b =>
+                      val relativePath = b.hookInfo.trees.map(_.prefix)
+                      relativePath.collect {
+                        case "$file" :: tail => filePath.init ++ tail
+                      }
+                    }
+                    val k = filePath.mkString(".")
+                    k -> ScriptNode(k, importPaths.map(s => ScriptNode(s.mkString("."), Seq.empty)))
+                  }.toMap
+
+                  val importTree = importTreeMap.map {
+                    case (k, v) => ScriptNode(k, v.inputs.map(i => importTreeMap(i.cls)))
+                  }.toSeq
+
+                  EvaluatorState(
+                    rootModule,
+                    rootModule.getClass.getClassLoader.asInstanceOf[
+                      SpecialClassLoader
+                    ].classpathSignature,
+                    mutable.Map.empty[Segments, (Int, Any)],
+                    interp.watchedValues.toSeq,
+                    systemProperties.keySet,
+                    importTree
+                  )
+                }
             (eval, interp.watchedValues)
         }
     }
@@ -85,6 +104,7 @@ object RunScript {
           wd / "out",
           s.rootModule,
           log,
+          importTree = s.importTree,
           s.classLoaderSig,
           s.workerCache,
           env,

main/test/src/util/ScriptTestSuite.scala

@@ -53,13 +53,20 @@ abstract class ScriptTestSuite(fork: Boolean) extends TestSuite {
     initialSystemProperties = sys.props.toMap
   )
   def eval(s: String*): Boolean = {
+    evalImpl(os.Inherit, s)
+  }
+  def eval(stdout: os.ProcessOutput, s: String*): Boolean = {
+    require(fork, "Custom stdout is only implemented for fork mode")
+    evalImpl(stdout, s)
+  }
+  private def evalImpl(stdout: os.ProcessOutput, s: Seq[String]): Boolean = {
     if (!fork) runner.runScript(workspacePath / buildPath, s.toList)
     else {
       try {
         os.proc(os.home / "mill-release", "-i", s).call(
           wd,
           stdin = os.Inherit,
-          stdout = os.Inherit,
+          stdout = stdout,
           stderr = os.Inherit
         )
         true

main/test/src/util/TestEvaluator.scala

@@ -63,6 +63,7 @@ class TestEvaluator(
     TestEvaluator.externalOutPath,
     module,
     logger,
+    importTree = Seq.empty,
     failFast = failFast,
     threadCount = threads
   )