diff --git a/bsp/worker/src/mill/bsp/worker/Utils.scala b/bsp/worker/src/mill/bsp/worker/Utils.scala index dba6b0faeef..b05c58ee9c8 100644 --- a/bsp/worker/src/mill/bsp/worker/Utils.scala +++ b/bsp/worker/src/mill/bsp/worker/Utils.scala @@ -45,7 +45,7 @@ object Utils { results: Evaluator.Results, task: mill.define.Task[_] ): StatusCode = { - results.results(task) match { + results.results(task).result match { case Success(_) => StatusCode.OK case Skipped => StatusCode.CANCELLED case _ => StatusCode.ERROR diff --git a/ci/mill-bootstrap.patch b/ci/mill-bootstrap.patch index 13c9465df92..42b80893553 100644 --- a/ci/mill-bootstrap.patch +++ b/ci/mill-bootstrap.patch @@ -1,5 +1,5 @@ diff --git a/build.sc b/build.sc -index 0f664c99c3..7d8788e7a8 100644 +index 0f664c99c3..d738fbe351 100644 --- a/build.sc +++ b/build.sc @@ -19,17 +19,18 @@ import com.github.lolgab.mill.mima.{ @@ -518,7 +518,7 @@ index 0f664c99c3..7d8788e7a8 100644 os.copy(examplePath, T.dest / exampleStr, createFolders = true) os.copy(bootstrapLauncher().path, T.dest / exampleStr / "mill") val zip = T.dest / s"$exampleStr.zip" -@@ -2018,47 +1735,6 @@ def exampleZips: Target[Seq[PathRef]] = T { +@@ -2018,51 +1735,10 @@ def exampleZips: Target[Seq[PathRef]] = T { } def uploadToGithub(authKey: String) = T.command { @@ -566,3 +566,18 @@ index 0f664c99c3..7d8788e7a8 100644 } def validate(ev: Evaluator): Command[Unit] = T.command { +- T.task(MainModule.evaluateTasks( ++ mill.main.RunScript.evaluateTasksNamed( + ev.withFailFast(false), + Seq( + "__.compile", +@@ -2075,7 +1751,8 @@ def validate(ev: Evaluator): Command[Unit] = T.command { + "docs.localPages" + ), + selectMode = SelectMode.Separated +- )(identity))() ++ ) ++ + () + } + diff --git a/example/src/mill/integration/ExampleTestSuite.scala b/example/src/mill/integration/ExampleTestSuite.scala index cd068300fc8..15d996f1a84 100644 --- a/example/src/mill/integration/ExampleTestSuite.scala +++ b/example/src/mill/integration/ExampleTestSuite.scala @@ -88,7 +88,7 @@ object ExampleTestSuite extends IntegrationTestSuite { BashTokenizer.tokenize(commandStr) match { case Seq(s"./$command", rest @ _*) => val evalResult = command match { - case "mill" => evalStdout(rest: _*) + case "mill" => evalStdout(rest) case cmd => val tokens = cmd +: rest val executable = workspaceRoot / os.RelPath(tokens.head) diff --git a/example/tasks/2-primary-tasks/build.sc b/example/tasks/2-primary-tasks/build.sc index 670ad446e97..d8483daebe2 100644 --- a/example/tasks/2-primary-tasks/build.sc +++ b/example/tasks/2-primary-tasks/build.sc @@ -34,10 +34,10 @@ def lineCount: T[Int] = T { > ./mill show lineCount Computing line count -18 +16 > ./mill show lineCount # line count already cached, doesn't need to be computed -18 +16 */ @@ -111,7 +111,7 @@ def hugeFileName = T{ /** Usage > ./mill show lineCount -18 +16 > ./mill show hugeFileName # This still runs `largestFile` even though `lineCount() < 999` Finding Largest File diff --git a/example/tasks/2-primary-tasks/src/Foo.java b/example/tasks/2-primary-tasks/src/Foo.java index 0c688a3e9ec..7f6eaa4135a 100644 --- a/example/tasks/2-primary-tasks/src/Foo.java +++ b/example/tasks/2-primary-tasks/src/Foo.java @@ -13,6 +13,4 @@ public static void main(String[] args) throws IOException{ System.out.println("foo.txt resource: " + br.readLine()); } } - - } diff --git a/integration/feature/editing/test/src/MultiLevelBuildTests.scala b/integration/feature/editing/test/src/MultiLevelBuildTests.scala index b962f70cd1c..c0a703033c5 100644 --- a/integration/feature/editing/test/src/MultiLevelBuildTests.scala +++ b/integration/feature/editing/test/src/MultiLevelBuildTests.scala @@ -58,7 +58,12 @@ object MultiLevelBuildTests extends IntegrationTestSuite { */ def checkWatchedFiles(expected0: Seq[os.Path]*) = { for ((expectedWatched0, (frame, path)) <- expected0.zip(loadFrames(expected0.length))) { - val frameWatched = frame.evalWatched.map(_.path).sorted + val frameWatched = frame + .evalWatched + .map(_.path) + .sorted.filter(_.startsWith(wsRoot)) + .filter(!_.segments.contains("mill-launcher")) + val expectedWatched = expectedWatched0.sorted assert(frameWatched == expectedWatched) } diff --git a/integration/feature/watch-source-input/repo/bar.txt b/integration/feature/watch-source-input/repo/bar.txt new file mode 100644 index 00000000000..8634afda766 --- /dev/null +++ b/integration/feature/watch-source-input/repo/bar.txt @@ -0,0 +1 @@ +initial-bar \ No newline at end of file diff --git a/integration/feature/watch-source-input/repo/baz.txt b/integration/feature/watch-source-input/repo/baz.txt new file mode 100644 index 00000000000..62fca0a765b --- /dev/null +++ b/integration/feature/watch-source-input/repo/baz.txt @@ -0,0 +1 @@ +initial-baz \ No newline at end of file diff --git a/integration/feature/watch-source-input/repo/build.sc b/integration/feature/watch-source-input/repo/build.sc new file mode 100644 index 00000000000..70bf339d6c2 --- /dev/null +++ b/integration/feature/watch-source-input/repo/build.sc @@ -0,0 +1,47 @@ +import mill._ + +println("Setting up build.sc") + +def foo = T.sources(millSourcePath / "foo1.txt", millSourcePath / "foo2.txt") +def bar = T.source(millSourcePath / "bar.txt") + +def qux = T{ + val fooMsg = "Running qux foo contents " + foo().map(p => os.read(p.path)).mkString(" ") + println(fooMsg) + + val barMsg = "Running qux bar contents " + os.read(bar().path) + println(barMsg) + + writeCompletionMarker("quxRan") + + fooMsg + " " + barMsg +} + +interp.watchValue(PathRef(millSourcePath / "watchValue.txt")) + +def baz = T.input(PathRef(millSourcePath / "baz.txt")) + +def lol = T{ + val barMsg = "Running lol baz contents " + os.read(baz().path) + println(barMsg) + + writeCompletionMarker("lolRan") + + barMsg +} + + +def writeCompletionMarker(name: String) = { + + Range(0, 10) + .map(i => os.pwd / "out" / s"$name$i") + .find(!os.exists(_)) + .foreach(os.write(_, "")) +} + +writeCompletionMarker("initialized") + +if (os.read(millSourcePath / "watchValue.txt").contains("exit")){ + Thread.sleep(1000) + System.exit(0) +} \ No newline at end of file diff --git a/integration/feature/watch-source-input/repo/foo1.txt b/integration/feature/watch-source-input/repo/foo1.txt new file mode 100644 index 00000000000..acc9bce3c86 --- /dev/null +++ b/integration/feature/watch-source-input/repo/foo1.txt @@ -0,0 +1 @@ +initial-foo1 \ No newline at end of file diff --git a/integration/feature/watch-source-input/repo/foo2.txt b/integration/feature/watch-source-input/repo/foo2.txt new file mode 100644 index 00000000000..b9491ded04b --- /dev/null +++ b/integration/feature/watch-source-input/repo/foo2.txt @@ -0,0 +1 @@ +initial-foo2 \ No newline at end of file diff --git a/integration/feature/watch-source-input/repo/watchValue.txt b/integration/feature/watch-source-input/repo/watchValue.txt new file mode 100644 index 00000000000..49e2fc36b56 --- /dev/null +++ b/integration/feature/watch-source-input/repo/watchValue.txt @@ -0,0 +1 @@ +initial-watchValue2 \ No newline at end of file diff --git a/integration/feature/watch-source-input/test/src/WatchSourceInputTests.scala b/integration/feature/watch-source-input/test/src/WatchSourceInputTests.scala new file mode 100644 index 00000000000..c2f869007c0 --- /dev/null +++ b/integration/feature/watch-source-input/test/src/WatchSourceInputTests.scala @@ -0,0 +1,153 @@ +package mill.integration + +import utest._ +import scala.concurrent.{Await, Future} +import scala.concurrent.duration.Duration +import scala.concurrent.duration.SECONDS +import scala.concurrent.ExecutionContext.Implicits.global + +/** + * Test to make sure that `--watch` works in the following cases: + * + * 1. `T.source` + * 2. `T.sources` + * 3. `T.input` + * 4. `interp.watchValue` + * 5. Implicitly watched files, like `build.sc` + */ +object WatchSourceInputTests extends IntegrationTestSuite { + + val maxDuration = 30000 + val tests = Tests { + val wsRoot = initWorkspace() + + def awaitCompletionMarker(name: String) = { + val maxTime = System.currentTimeMillis() + maxDuration + while (!os.exists(wsRoot / "out" / name)) { + if (System.currentTimeMillis() > maxTime) { + sys.error(s"awaitCompletionMarker($name) timed out") + } + Thread.sleep(100) + } + } + + def testWatchSource(show: Boolean) = { + val showArgs = if (show) Seq("show") else Nil + + val evalResult = Future { evalTimeoutStdout(maxDuration, "--watch", showArgs, "qux") } + val expectedPrints = collection.mutable.Buffer.empty[String] + val expectedShows = collection.mutable.Buffer.empty[String] + + awaitCompletionMarker("initialized0") + awaitCompletionMarker("quxRan0") + expectedPrints.append( + "Setting up build.sc", + "Running qux foo contents initial-foo1 initial-foo2", + "Running qux bar contents initial-bar" + ) + expectedShows.append( + "Running qux foo contents initial-foo1 initial-foo2 Running qux bar contents initial-bar" + ) + + os.write.over(wsRoot / "foo1.txt", "edited-foo1") + awaitCompletionMarker("quxRan1") + expectedPrints.append( + "Running qux foo contents edited-foo1 initial-foo2", + "Running qux bar contents initial-bar" + ) + expectedShows.append( + "Running qux foo contents edited-foo1 initial-foo2 Running qux bar contents initial-bar" + ) + + os.write.over(wsRoot / "foo2.txt", "edited-foo2") + awaitCompletionMarker("quxRan2") + expectedPrints.append( + "Running qux foo contents edited-foo1 edited-foo2", + "Running qux bar contents initial-bar" + ) + expectedShows.append( + "Running qux foo contents edited-foo1 edited-foo2 Running qux bar contents initial-bar" + ) + + os.write.over(wsRoot / "bar.txt", "edited-bar") + awaitCompletionMarker("quxRan3") + expectedPrints.append( + "Running qux foo contents edited-foo1 edited-foo2", + "Running qux bar contents edited-bar" + ) + expectedShows.append( + "Running qux foo contents edited-foo1 edited-foo2 Running qux bar contents edited-bar" + ) + + os.write.append(wsRoot / "build.sc", "\ndef unrelated = true") + awaitCompletionMarker("initialized1") + expectedPrints.append( + "Setting up build.sc", + "Running qux foo contents edited-foo1 edited-foo2", + "Running qux bar contents edited-bar" + ) + expectedShows.append( + "Running qux foo contents edited-foo1 edited-foo2 Running qux bar contents edited-bar" + ) + + os.write.over(wsRoot / "watchValue.txt", "exit") + awaitCompletionMarker("initialized2") + expectedPrints.append("Setting up build.sc") + + val res = Await.result(evalResult, Duration.apply(maxDuration, SECONDS)) + + val (shows, prints) = res.out.linesIterator.toVector.partition(_.startsWith("\"")) + + assert(prints == expectedPrints) + if (show) assert(shows == expectedShows.map('"' + _ + '"')) + } + + test("sources") { + + test("noshow") - testWatchSource(false) + test("show") - testWatchSource(true) + } + + def testWatchInput(show: Boolean) = { + val showArgs = if (show) Seq("show") else Nil + + val evalResult = Future { evalTimeoutStdout(maxDuration, "--watch", showArgs, "lol") } + val expectedPrints = collection.mutable.Buffer.empty[String] + val expectedShows = collection.mutable.Buffer.empty[String] + + awaitCompletionMarker("initialized0") + awaitCompletionMarker("lolRan0") + expectedPrints.append( + "Setting up build.sc", + "Running lol baz contents initial-baz" + ) + expectedShows.append("Running lol baz contents initial-baz") + + os.write.over(wsRoot / "baz.txt", "edited-baz") + awaitCompletionMarker("lolRan1") + expectedPrints.append("Running lol baz contents edited-baz") + expectedShows.append("Running lol baz contents edited-baz") + + os.write.over(wsRoot / "watchValue.txt", "edited-watchValue") + awaitCompletionMarker("initialized1") + expectedPrints.append("Setting up build.sc") + expectedShows.append("Running lol baz contents edited-baz") + + os.write.over(wsRoot / "watchValue.txt", "exit") + awaitCompletionMarker("initialized2") + expectedPrints.append("Setting up build.sc") + + val res = Await.result(evalResult, Duration.apply(maxDuration, SECONDS)) + + val (shows, prints) = res.out.linesIterator.toVector.partition(_.startsWith("\"")) + assert(prints == expectedPrints) + if (show) assert(shows == expectedShows.map('"' + _ + '"')) + } + + test("input") { + + test("noshow") - testWatchInput(false) + test("show") - testWatchInput(true) + } + } +} diff --git a/integration/src/mill/integration/IntegrationTestSuite.scala b/integration/src/mill/integration/IntegrationTestSuite.scala index e59e275e44d..75086dd1b15 100644 --- a/integration/src/mill/integration/IntegrationTestSuite.scala +++ b/integration/src/mill/integration/IntegrationTestSuite.scala @@ -1,21 +1,14 @@ package mill.integration -import mainargs.Flag -import mill.runner.MillCliConfig import mill.main.SelectMode -import mill.runner.{MillBuildBootstrap, MillMain, RunnerState, Watching} -import mill.api.SystemStreams -import mill.util.PrintLogger -import os.Path +import mill.runner.RunnerState +import os.{Path, Shellable} import utest._ -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, PrintStream} -import java.nio.file.NoSuchFileException import scala.util.control.NonFatal object IntegrationTestSuite { case class EvalResult(isSuccess: Boolean, out: String, err: String) - } abstract class IntegrationTestSuite extends TestSuite { @@ -37,16 +30,20 @@ abstract class IntegrationTestSuite extends TestSuite { var runnerState = RunnerState.empty - def eval(s: String*): Boolean = evalFork(os.Inherit, os.Inherit, s) + def eval(s: Shellable*): Boolean = evalFork(os.Inherit, os.Inherit, s, -1) + + def evalStdout(s: Shellable*): IntegrationTestSuite.EvalResult = { + evalTimeoutStdout(-1, s: _*) + } - def evalStdout(s: String*): IntegrationTestSuite.EvalResult = { + def evalTimeoutStdout(timeout: Long, s: Shellable*): IntegrationTestSuite.EvalResult = { val output = Seq.newBuilder[String] val error = Seq.newBuilder[String] val processOutput = os.ProcessOutput.Readlines(output += _) val processError = os.ProcessOutput.Readlines(error += _) - val result = evalFork(processOutput, processError, s) + val result = evalFork(processOutput, processError, s, timeout) IntegrationTestSuite.EvalResult( result, output.result().mkString("\n"), @@ -61,7 +58,8 @@ abstract class IntegrationTestSuite extends TestSuite { private def evalFork( stdout: os.ProcessOutput, stderr: os.ProcessOutput, - s: Seq[String] + s: Seq[Shellable], + timeout: Long ): Boolean = { val serverArgs = if (integrationTestMode == "server" || integrationTestMode == "local") Seq() @@ -75,7 +73,8 @@ abstract class IntegrationTestSuite extends TestSuite { stdin = os.Inherit, stdout = stdout, stderr = stderr, - env = millTestSuiteEnv + env = millTestSuiteEnv, + timeout = timeout ) true } catch { diff --git a/main/api/src/mill/api/Result.scala b/main/api/src/mill/api/Result.scala index ab55c0aaad3..efc85651890 100644 --- a/main/api/src/mill/api/Result.scala +++ b/main/api/src/mill/api/Result.scala @@ -11,6 +11,8 @@ sealed trait Result[+T] { def map[V](f: T => V): Result[V] def flatMap[V](f: T => Result[V]): Result[V] def asSuccess: Option[Result.Success[T]] = None + def asFailing: Option[Result.Failing[T]] = None + } object Result { @@ -56,6 +58,8 @@ object Result { sealed trait Failing[+T] extends Result[T] { def map[V](f: T => V): Failing[V] def flatMap[V](f: T => Result[V]): Failing[V] + override def asFailing: Option[Result.Failing[T]] = Some(this) + } /** diff --git a/main/api/src/mill/api/Val.scala b/main/api/src/mill/api/Val.scala new file mode 100644 index 00000000000..e80d92b393d --- /dev/null +++ b/main/api/src/mill/api/Val.scala @@ -0,0 +1,10 @@ +package mill.api + +/** + * A somewhat-type-safe wrapper around `Any`. Stores an un-typed value, but + * can only be created explicitly by wrapping in `Val(_)` and de-constructed + * explicitly via `.value`. That makes it much less likely to introduce bugs + * passing the wrong thing, e.g. `(Any, Int)` can be passed to `Any`, but + * `(Val, Int)` cannot be passed to `Val` + */ +case class Val(value: Any) diff --git a/main/eval/src/mill/eval/Evaluator.scala b/main/eval/src/mill/eval/Evaluator.scala index 32eeb2a8b3a..f7d6c6d88b8 100644 --- a/main/eval/src/mill/eval/Evaluator.scala +++ b/main/eval/src/mill/eval/Evaluator.scala @@ -10,7 +10,8 @@ import mill.api.{ PathRef, Result, Strict, - TestReporter + TestReporter, + Val } import mill.api.Result.{Aborted, Failing, OuterStack, Success} import mill.api.Strict.Agg @@ -35,14 +36,14 @@ class Evaluator private ( _rootModule: mill.define.BaseModule, _baseLogger: ColorLogger, _classLoaderSigHash: Int, - _workerCache: mutable.Map[Segments, (Int, Any)], + _workerCache: mutable.Map[Segments, (Int, Val)], _env: Map[String, String], _failFast: Boolean, _threadCount: Option[Int], _scriptImportGraph: Map[os.Path, (Int, Seq[os.Path])] ) { - import Evaluator.Terminal + import Evaluator._ def home: os.Path = _home @@ -65,7 +66,7 @@ class Evaluator private ( /** * Mutable worker cache. */ - def workerCache: mutable.Map[Segments, (Int, Any)] = _workerCache + def workerCache: mutable.Map[Segments, (Int, Val)] = _workerCache def env: Map[String, String] = _env /** @@ -122,14 +123,14 @@ class Evaluator private ( val (sortedGroups, transitive) = Evaluator.plan(goals) val evaluated = new Agg.Mutable[Task[_]] - val results = mutable.LinkedHashMap.empty[Task[_], mill.api.Result[(Any, Int)]] + var results = Map.empty[Task[_], TaskResult[(Val, Int)]] var someTaskFailed: Boolean = false val timings = mutable.ArrayBuffer.empty[(Either[Task[_], Labelled[_]], Int, Boolean)] for (((terminal, group), i) <- sortedGroups.items().zipWithIndex) { if (failFast && someTaskFailed) { // we exit early and set aborted state for all left tasks - group.iterator.foreach { task => results.put(task, Aborted) } + group.iterator.foreach { task => results += (task -> TaskResult(Aborted, None)) } } else { @@ -142,7 +143,7 @@ class Evaluator private ( val startTime = System.currentTimeMillis() // Increment the counter message by 1 to go from 1/10 to 10/10 instead of 0/10 to 9/10 - val counterMsg = s"${(i + 1)}/${sortedGroups.keyCount}" + val counterMsg = s"${i + 1}/${sortedGroups.keyCount}" val Evaluated(newResults, newEvaluated, cached) = evaluateGroupCached( terminal = terminal, @@ -154,10 +155,10 @@ class Evaluator private ( logger = contextLogger ) someTaskFailed = - someTaskFailed || newResults.exists(task => !task._2.isInstanceOf[Success[_]]) + someTaskFailed || newResults.exists(task => !task._2.result.isInstanceOf[Success[_]]) - for (ev <- newEvaluated) evaluated.append(ev) - for ((k, v) <- newResults) results.put(k, v) + evaluated.appendAll(newEvaluated) + results ++= newResults val endTime = System.currentTimeMillis() timings.append((terminal, (endTime - startTime).toInt, cached)) @@ -165,8 +166,9 @@ class Evaluator private ( } Evaluator.writeTimings(timings.toSeq, outPath) + Evaluator.Results( - rawValues = goals.indexed.map(results(_).map(_._1)), + rawValues = goals.indexed.map(results(_).result.map(_._1)), evaluated = evaluated, transitive = transitive, failing = getFailing(sortedGroups, results), @@ -176,15 +178,16 @@ class Evaluator private ( def getFailing( sortedGroups: MultiBiMap[Either[Task[_], Labelled[Any]], Task[_]], - results: collection.Map[Task[_], mill.api.Result[(Any, Int)]] - ): MultiBiMap.Mutable[Either[Task[_], Labelled[_]], Failing[_]] = { - val failing = new MultiBiMap.Mutable[Either[Task[_], Labelled[_]], mill.api.Result.Failing[_]] + results: collection.Map[Task[_], TaskResult[(Val, Int)]] + ): MultiBiMap.Mutable[Either[Task[_], Labelled[_]], Failing[Val]] = { + val failing = new MultiBiMap.Mutable[Either[Task[_], Labelled[_]], Result.Failing[Val]] for ((k, vs) <- sortedGroups.items()) { failing.addAll( k, Loose.Agg.from( - vs.items.flatMap(results.get).collect { case f: mill.api.Result.Failing[_] => - f.map(_._1) + vs.items.flatMap(results.get).collect { + case er @ TaskResult(f: Result.Failing[(Val, Int)], _) => + f.map(_._1) } ) ) @@ -254,7 +257,8 @@ class Evaluator private ( contextLogger ) - if (failFast && res.newResults.values.exists(_.asSuccess.isEmpty)) failed.set(true) + if (failFast && res.newResults.values.exists(_.result.asSuccess.isEmpty)) + failed.set(true) val endTime = System.currentTimeMillis() timeLog.timeTrace( @@ -275,21 +279,23 @@ class Evaluator private ( val finishedOptsMap = finishedOpts.toMap - val results = terminals + val results0: Vector[(Task[_], TaskResult[(Val, Int)])] = terminals .flatMap { t => sortedGroups.lookupKey(t).flatMap { t0 => finishedOptsMap(t) match { - case None => Some((t0, Aborted)) + case None => Some((t0, TaskResult(Aborted, None))) case Some(res) => res.newResults.get(t0).map(r => (t0, r)) } } } - .toMap + + val results: Map[Task[_], TaskResult[(Val, Int)]] = + results0.toMap timeLog.close() Evaluator.Results( - goals.indexed.map(results(_).map(_._1)), + goals.indexed.map(results(_).map(_._1).result), finishedOpts.map(_._2).flatMap(_.toSeq.flatMap(_.newEvaluated)), transitive, getFailing(sortedGroups, results), @@ -304,7 +310,7 @@ class Evaluator private ( protected def evaluateGroupCached( terminal: Terminal, group: Agg[Task[_]], - results: collection.Map[Task[_], mill.api.Result[(Any, Int)]], + results: Map[Task[_], TaskResult[(Val, Int)]], counterMsg: String, zincProblemReporter: Int => Option[CompileProblemReporter], testReporter: TestReporter, @@ -313,7 +319,7 @@ class Evaluator private ( val externalInputsHash = scala.util.hashing.MurmurHash3.orderedHash( group.items.flatMap(_.inputs).filter(!group.contains(_)) - .flatMap(results(_).asSuccess.map(_.value._2)) + .flatMap(results(_).result.asSuccess.map(_.value._2)) ) val sideHashes = scala.util.hashing.MurmurHash3.orderedHash( @@ -372,7 +378,7 @@ class Evaluator private ( destSegments(labelledNamedTask) ) - val cached: Option[(Any, Int)] = for { + val cached: Option[(Val, Int)] = for { cached <- try Some(upickle.default.read[Evaluator.Cached](paths.meta.toIO)) catch { @@ -390,16 +396,16 @@ class Evaluator private ( None case NonFatal(_) => None } - } yield (parsed, cached.valueHash) + } yield (Val(parsed), cached.valueHash) val previousWorker = labelledNamedTask.task.asWorker.flatMap { w => workerCache.synchronized { workerCache.get(w.ctx.segments) } } - val upToDateWorker: Option[Any] = previousWorker.flatMap { + val upToDateWorker: Option[Val] = previousWorker.flatMap { case (`inputsHash`, upToDate) => // worker cached and up-to-date Some(upToDate) - case (_, obsolete: AutoCloseable) => + case (_, Val(obsolete: AutoCloseable)) => // worker cached but obsolete, needs to be closed try { logger.debug(s"Closing previous worker: ${labelledNamedTask.segments.render}") @@ -422,8 +428,8 @@ class Evaluator private ( upToDateWorker.map((_, inputsHash)) orElse cached match { case Some((v, hashCode)) => - val newResults = mutable.LinkedHashMap.empty[Task[_], mill.api.Result[(Any, Int)]] - newResults(labelledNamedTask.task) = mill.api.Result.Success((v, hashCode)) + val newResults = mutable.LinkedHashMap.empty[Task[_], TaskResult[(Val, Int)]] + newResults(labelledNamedTask.task) = TaskResult(Result.Success((v, hashCode)), None) Evaluated(newResults, Nil, cached = true) @@ -437,7 +443,7 @@ class Evaluator private ( Evaluator.dynamicTickerPrefix.withValue(s"[$counterMsg] $targetLabel > ") { evaluateGroup( group, - results, + results.toMap, inputsHash, paths = Some(paths), maybeTargetLabel = Some(targetLabel), @@ -449,10 +455,10 @@ class Evaluator private ( } newResults(labelledNamedTask.task) match { - case mill.api.Result.Failure(_, Some((v, _))) => + case TaskResult(Result.Failure(_, Some((v, _))), _) => handleTaskResult(v, v.##, paths.meta, inputsHash, labelledNamedTask) - case mill.api.Result.Success((v, _)) => + case TaskResult(Result.Success((v, _)), _) => handleTaskResult(v, v.##, paths.meta, inputsHash, labelledNamedTask) case _ => @@ -479,7 +485,7 @@ class Evaluator private ( } def handleTaskResult( - v: Any, + v: Val, hashCode: Int, metaPath: os.Path, inputsHash: Int, @@ -495,9 +501,9 @@ class Evaluator private ( .task .writerOpt .asInstanceOf[Option[upickle.default.Writer[Any]]] - .map(w => upickle.default.writeJs(v)(w) -> v) + .map { w => upickle.default.writeJs(v.value)(w) } - for ((json, _) <- terminalResult) { + for (json <- terminalResult) { os.write.over( metaPath, upickle.default.stream( @@ -512,7 +518,7 @@ class Evaluator private ( protected def evaluateGroup( group: Agg[Task[_]], - results: collection.Map[Task[_], mill.api.Result[(Any, Int)]], + results: collection.Map[Task[_], TaskResult[(Val, Int)]], inputsHash: Int, paths: Option[EvaluatorPaths], maybeTargetLabel: Option[String], @@ -520,95 +526,99 @@ class Evaluator private ( reporter: Int => Option[CompileProblemReporter], testReporter: TestReporter, logger: mill.api.Logger - ): (mutable.LinkedHashMap[Task[_], mill.api.Result[(Any, Int)]], mutable.Buffer[Task[_]]) = { + ): (mutable.LinkedHashMap[Task[_], TaskResult[(Val, Int)]], mutable.Buffer[Task[_]]) = { - val newEvaluated = mutable.Buffer.empty[Task[_]] - val newResults = mutable.LinkedHashMap.empty[Task[_], mill.api.Result[(Any, Int)]] + def computeAll(enableTicker: Boolean) = { + val newEvaluated = mutable.Buffer.empty[Task[_]] + val newResults = mutable.LinkedHashMap.empty[Task[_], Result[(Val, Int)]] - val nonEvaluatedTargets = group.indexed.filterNot(results.contains) + val nonEvaluatedTargets = group.indexed.filterNot(results.contains) - // should we log progress? - val logRun = maybeTargetLabel.isDefined && { - val inputResults = for { - target <- nonEvaluatedTargets - item <- target.inputs.filterNot(group.contains) - } yield results(item).map(_._1) - inputResults.forall(_.isInstanceOf[mill.api.Result.Success[_]]) - } + // should we log progress? + val logRun = maybeTargetLabel.isDefined && { + val inputResults = for { + target <- nonEvaluatedTargets + item <- target.inputs.filterNot(group.contains) + } yield results(item).map(_._1) + inputResults.forall(_.result.isInstanceOf[Result.Success[_]]) + } - val tickerPrefix = maybeTargetLabel.map { targetLabel => - val prefix = s"[$counterMsg] $targetLabel " - if (logRun) logger.ticker(prefix) - prefix + "| " - } + val tickerPrefix = maybeTargetLabel.map { targetLabel => + val prefix = s"[$counterMsg] $targetLabel " + if (logRun && enableTicker) logger.ticker(prefix) + prefix + "| " + } - val multiLogger = new ProxyLogger(resolveLogger(paths.map(_.log), logger)) { - override def ticker(s: String): Unit = { - super.ticker(tickerPrefix.getOrElse("") + s) + val multiLogger = new ProxyLogger(resolveLogger(paths.map(_.log), logger)) { + override def ticker(s: String): Unit = { + if (enableTicker) super.ticker(tickerPrefix.getOrElse("") + s) + else () // do nothing + } } - } - // This is used to track the usage of `T.dest` in more than one Task - // But it's not really clear what issue we try to prevent here - // Vice versa, being able to use T.dest in multiple `T.task` - // is rather essential to split up larger tasks into small parts - // So I like to disable this detection for now - var usedDest = Option.empty[(Task[_], Array[StackTraceElement])] - for (task <- nonEvaluatedTargets) { - newEvaluated.append(task) - val targetInputValues = task.inputs - .map { x => newResults.getOrElse(x, results(x)) } - .collect { case mill.api.Result.Success((v, _)) => v } - - val res = - if (targetInputValues.length != task.inputs.length) mill.api.Result.Skipped - else { - val args = new Ctx( - args = targetInputValues.toArray[Any].toIndexedSeq, - dest0 = () => - paths match { - case Some(dest) => - if (usedDest.isEmpty) os.makeDir.all(dest.dest) - usedDest = Some((task, new Exception().getStackTrace)) - dest.dest - case None => - throw new Exception("No `dest` folder available here") - // } - }, - log = multiLogger, - home = home, - env = env, - reporter = reporter, - testReporter = testReporter, - workspace = rootModule.millSourcePath - ) with mill.api.Ctx.Jobs { - override def jobs: Int = effectiveThreadCount - } + // This is used to track the usage of `T.dest` in more than one Task + // But it's not really clear what issue we try to prevent here + // Vice versa, being able to use T.dest in multiple `T.task` + // is rather essential to split up larger tasks into small parts + // So I like to disable this detection for now + var usedDest = Option.empty[(Task[_], Array[StackTraceElement])] + for (task <- nonEvaluatedTargets) { + newEvaluated.append(task) + val targetInputValues = task.inputs + .map { x => newResults.getOrElse(x, results(x).result) } + .collect { case Result.Success((v, _)) => v } + + val res = { + if (targetInputValues.length != task.inputs.length) Result.Skipped + else { + val args = new Ctx( + args = targetInputValues.map(_.value).toIndexedSeq, + dest0 = () => + paths match { + case Some(dest) => + if (usedDest.isEmpty) os.makeDir.all(dest.dest) + usedDest = Some((task, new Exception().getStackTrace)) + dest.dest + case None => + throw new Exception("No `dest` folder available here") + // } + }, + log = multiLogger, + home = home, + env = env, + reporter = reporter, + testReporter = testReporter, + workspace = rootModule.millSourcePath + ) with mill.api.Ctx.Jobs { + override def jobs: Int = effectiveThreadCount + } - val out = System.out - val in = System.in - val err = System.err - mill.api.SystemStreams.withStreams(multiLogger.systemStreams) { - try task.evaluate(args) - catch { - case NonFatal(e) => - mill.api.Result.Exception( - e, - new OuterStack(new Exception().getStackTrace.toIndexedSeq) - ) + mill.api.SystemStreams.withStreams(multiLogger.systemStreams) { + try task.evaluate(args).map(Val(_)) + catch { + case NonFatal(e) => + Result.Exception( + e, + new OuterStack(new Exception().getStackTrace.toIndexedSeq) + ) + } } } - } - newResults(task) = for (v <- res) yield { - ( - v, - if (task.isInstanceOf[Worker[_]]) inputsHash - else v.## - ) + newResults(task) = for (v <- res) yield { + ( + v, + if (task.isInstanceOf[Worker[_]]) inputsHash + else v.## + ) + } } + multiLogger.close() + (newResults, newEvaluated) } + val (newResults, newEvaluated) = computeAll(enableTicker = true) + if (!failFast) maybeTargetLabel.foreach { targetLabel => val taskFailed = newResults.exists(task => !task._2.isInstanceOf[Success[_]]) if (taskFailed) { @@ -616,9 +626,21 @@ class Evaluator private ( } } - multiLogger.close() - - (newResults, newEvaluated) + ( + newResults.map { case (k, v) => + ( + k, + TaskResult( + v, + Some { () => + val (recalced, _) = computeAll(enableTicker = false) + recalced.apply(k) + } + ) + ) + }, + newEvaluated + ) } def resolveLogger(logPath: Option[os.Path], logger: mill.api.Logger): mill.api.Logger = @@ -685,7 +707,7 @@ class Evaluator private ( rootModule: mill.define.BaseModule = this.rootModule, baseLogger: ColorLogger = this.baseLogger, classLoaderSigHash: Int = this.classLoaderSigHash, - workerCache: mutable.Map[Segments, (Int, Any)] = this.workerCache, + workerCache: mutable.Map[Segments, (Int, Val)] = this.workerCache, env: Map[String, String] = this.env, failFast: Boolean = this.failFast, threadCount: Option[Int] = this.threadCount, @@ -712,7 +734,7 @@ class Evaluator private ( copy(rootModule = rootModule) def withBaseLogger(baseLogger: ColorLogger): Evaluator = copy(baseLogger = baseLogger) - def withWorkerCache(workerCache: mutable.Map[Segments, (Int, Any)]): Evaluator = + def withWorkerCache(workerCache: mutable.Map[Segments, (Int, Val)]): Evaluator = copy(workerCache = workerCache) def withEnv(env: Map[String, String]): Evaluator = copy(env = env) def withFailFast(failFast: Boolean): Evaluator = copy(failFast = failFast) @@ -764,20 +786,27 @@ object Evaluator { ) } + case class TaskResult[T](result: Result[T], recalcOpt: Option[() => Result[T]]) { + def map[V](f: T => V) = TaskResult[V]( + result.map(f), + recalcOpt.map(r => () => r().map(f)) + ) + } + case class Results( - rawValues: Seq[mill.api.Result[Any]], + rawValues: Seq[Result[Val]], evaluated: Agg[Task[_]], transitive: Agg[Task[_]], - failing: MultiBiMap[Either[Task[_], Labelled[_]], mill.api.Result.Failing[_]], - results: collection.Map[Task[_], mill.api.Result[Any]] + failing: MultiBiMap[Either[Task[_], Labelled[_]], Result.Failing[Val]], + results: collection.Map[Task[_], TaskResult[Val]] ) { - def values: Seq[Any] = rawValues.collect { case mill.api.Result.Success(v) => v } + def values: Seq[Val] = rawValues.collect { case Result.Success(v) => v } private def copy( - rawValues: Seq[Result[Any]] = rawValues, + rawValues: Seq[Result[Val]] = rawValues, evaluated: Agg[Task[_]] = evaluated, transitive: Agg[Task[_]] = transitive, - failing: MultiBiMap[Either[Task[_], Labelled[_]], Result.Failing[_]] = failing, - results: collection.Map[Task[_], Result[Any]] = results + failing: MultiBiMap[Either[Task[_], Labelled[_]], Result.Failing[Val]] = failing, + results: collection.Map[Task[_], TaskResult[Val]] ): Results = new Results( rawValues, evaluated, @@ -792,8 +821,8 @@ object Evaluator { Seq[Result[Any]], Agg[Task[_]], Agg[Task[_]], - MultiBiMap[Either[Task[_], Labelled[_]], Failing[_]], - collection.Map[Task[_], Result[Any]] + MultiBiMap[Either[Task[_], Labelled[_]], Failing[Val]], + collection.Map[Task[_], TaskResult[_]] )] = Some(( results.rawValues, results.evaluated, @@ -844,12 +873,12 @@ object Evaluator { } case class Evaluated( - newResults: collection.Map[Task[_], Result[(Any, Int)]], + newResults: collection.Map[Task[_], TaskResult[(Val, Int)]], newEvaluated: Seq[Task[_]], cached: Boolean ) { private[Evaluator] def copy( - newResults: collection.Map[Task[_], Result[(Any, Int)]] = newResults, + newResults: collection.Map[Task[_], TaskResult[(Val, Int)]], newEvaluated: Seq[Task[_]] = newEvaluated, cached: Boolean = cached ): Evaluated = new Evaluated(newResults, newEvaluated, cached) @@ -857,7 +886,7 @@ object Evaluator { object Evaluated { private[Evaluator] def unapply(evaluated: Evaluated) - : Option[(collection.Map[Task[_], Result[(Any, Int)]], Seq[Task[_]], Boolean)] = + : Option[(collection.Map[Task[_], TaskResult[(Val, Int)]], Seq[Task[_]], Boolean)] = Some((evaluated.newResults, evaluated.newEvaluated, evaluated.cached)) } @@ -885,14 +914,14 @@ object Evaluator { throw exceptionFactory(r) case r => // Input is a single-item Agg, so we also expect a single-item result - val Seq(e: T) = r.values + val Seq(Val(e: T)) = r.values e } def apply[T: ClassTag](tasks: Seq[Task[T]]): Seq[T] = evaluator.evaluate(tasks) match { case r if r.failing.items().nonEmpty => throw exceptionFactory(r) - case r => r.values.asInstanceOf[Seq[T]] + case r => r.values.map(_.value).asInstanceOf[Seq[T]] } } diff --git a/main/eval/test/src/mill/eval/EvaluationTests.scala b/main/eval/test/src/mill/eval/EvaluationTests.scala index 47d0d3241f8..fd2ee31f4be 100644 --- a/main/eval/test/src/mill/eval/EvaluationTests.scala +++ b/main/eval/test/src/mill/eval/EvaluationTests.scala @@ -43,7 +43,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { evaled.evaluated.indexed.partition(expEvaled.contains) assert( - evaled.values == Seq(expValue), + evaled.values.map(_.value) == Seq(expValue), matchingReturnedEvaled.toSet == expEvaled.toSet, extraEvaled == -1 || extra.length == extraEvaled ) @@ -53,7 +53,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { val evaled2 = evaluator.evaluate(Agg(target)) val expecteSecondRunEvaluated = Agg() assert( - evaled2.values == evaled.values, + evaled2.values.map(_.value) == evaled.values.map(_.value), evaled2.evaluated == expecteSecondRunEvaluated ) } diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index c3a54290856..ba6116b9cb6 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -2,10 +2,11 @@ package mill.main import java.util.concurrent.LinkedBlockingQueue import mill.{BuildInfo, T} -import mill.api.{Ctx, PathRef, Result, internal} -import mill.define.{Command, Segments, NamedTask, TargetImpl, Task} +import mill.api.{Ctx, Logger, PathRef, Result, internal} +import mill.define.{Command, NamedTask, Segments, TargetImpl, Task} import mill.eval.{Evaluator, EvaluatorPaths} -import mill.util.{PrintLogger, Watched} +import mill.main.SelectMode.Separated +import mill.util.{PrintLogger, Watchable} import pprint.{Renderer, Tree, Truncated} import ujson.Value @@ -25,32 +26,34 @@ object MainModule { } } - def evaluateTasks[T]( + private def show0( evaluator: Evaluator, targets: Seq[String], - selectMode: SelectMode - )(f: Seq[(Any, Option[ujson.Value])] => T): Result[Watched[Unit]] = { - RunScript.evaluateTasks(evaluator, targets, selectMode) match { + log: Logger, + watch0: Watchable => Unit + )(f: Seq[(Any, Option[(RunScript.TaskName, ujson.Value)])] => ujson.Value) = { + RunScript.evaluateTasksNamed( + evaluator.withBaseLogger( + // When using `show`, redirect all stdout of the evaluated tasks so the + // printed JSON is the only thing printed to stdout. + evaluator.baseLogger match { + case p: PrintLogger => p.withOutStream(p.errorStream) + case l => l + } + ), + targets, + Separated + ) match { case Left(err) => Result.Failure(err) - case Right((watched, Left(err))) => Result.Failure(err, Some(Watched((), watched))) - case Right((watched, Right(res))) => - f(res) - Result.Success(Watched((), watched)) - } - } + case Right((watched, Left(err))) => + watched.foreach(watch0) + Result.Failure(err) - @internal - def evaluateTasksNamed[T]( - evaluator: Evaluator, - targets: Seq[String], - selectMode: SelectMode - )(f: Seq[(Any, Option[(RunScript.TaskName, ujson.Value)])] => T): Result[Watched[Option[T]]] = { - RunScript.evaluateTasksNamed(evaluator, targets, selectMode) match { - case Left(err) => Result.Failure(err) - case Right((watched, Left(err))) => Result.Failure(err, Some(Watched(None, watched))) case Right((watched, Right(res))) => - val fRes = f(res) - Result.Success(Watched(Some(fRes), watched)) + val output = f(res) + watched.foreach(watch0) + log.outputStream.println(output.render(indent = 2)) + Result.Success(output) } } } @@ -60,6 +63,36 @@ object MainModule { * [[show]], [[inspect]], [[plan]], etc. */ trait MainModule extends mill.Module { + protected[mill] val watchedValues = mutable.Buffer.empty[Watchable] + protected[mill] val evalWatchedValues = mutable.Buffer.empty[Watchable] + + object interp { + + def watchValue[T](v0: => T)(implicit fn: sourcecode.FileName, ln: sourcecode.Line): T = { + val v = v0 + val watchable = Watchable.Value( + () => v0.hashCode, + v.hashCode(), + fn.value + ":" + ln.value + ) + watchedValues.append(watchable) + v + } + + def watch(p: os.Path): os.Path = { + val watchable = Watchable.Path(PathRef(p)) + watchedValues.append(watchable) + p + } + + def watch0(w: Watchable): Unit = { + watchedValues.append(w) + } + + def evalWatch0(w: Watchable): Unit = { + evalWatchedValues.append(w) + } + } implicit def millDiscover: mill.define.Discover[_] @@ -246,26 +279,11 @@ trait MainModule extends mill.Module { * to integrate Mill into external scripts and tooling. */ def show(evaluator: Evaluator, targets: String*): Command[ujson.Value] = T.command { - MainModule.evaluateTasksNamed( - evaluator.withBaseLogger( - // When using `show`, redirect all stdout of the evaluated tasks so the - // printed JSON is the only thing printed to stdout. - evaluator.baseLogger match { - case p: PrintLogger => p.withOutStream(p.errorStream) - case l => l - } - ), - targets, - SelectMode.Separated - ) { res: Seq[(Any, Option[(String, ujson.Value)])] => - val jsons = res.flatMap(_._2).map(_._2) - val output: ujson.Value = - if (jsons.size == 1) jsons.head - else { ujson.Arr.from(jsons) } - T.log.outputStream.println(output.render(indent = 2)) - output - }.map { res: Watched[Option[Value]] => - res.value.getOrElse(ujson.Null) + MainModule.show0(evaluator, targets, T.log, interp.evalWatch0) { res => + res.flatMap(_._2).map(_._2) match { + case Seq(single) => single + case multiple => multiple + } } } @@ -274,24 +292,8 @@ trait MainModule extends mill.Module { * to integrate Mill into external scripts and tooling. */ def showNamed(evaluator: Evaluator, targets: String*): Command[ujson.Value] = T.command { - MainModule.evaluateTasksNamed( - evaluator.withBaseLogger( - // When using `show`, redirect all stdout of the evaluated tasks so the - // printed JSON is the only thing printed to stdout. - evaluator.baseLogger match { - case p: PrintLogger => p.withOutStream(outStream = p.errorStream) - case l => l - } - ), - targets, - SelectMode.Separated - ) { res: Seq[(Any, Option[(String, ujson.Value)])] => - val nameAndJson = res.flatMap(_._2) - val output: ujson.Value = ujson.Obj.from(nameAndJson) - T.log.outputStream.println(output.render(indent = 2)) - output - }.map { res: Watched[Option[Value]] => - res.value.getOrElse(ujson.Null) + MainModule.show0(evaluator, targets, T.log, interp.evalWatch0) { res => + ujson.Obj.from(res.flatMap(_._2)) } } @@ -382,17 +384,19 @@ trait MainModule extends mill.Module { } /** - * The `init`` command generates a project based on a Giter8 template. It + * The `init` command generates a project based on a Giter8 template. It * prompts you to enter project name and creates a folder with that name. * You can use it to quickly generate a starter project. There are lots of * templates out there for many frameworks and tools! */ def init(evaluator: Evaluator, args: String*): Command[Unit] = T.command { - MainModule.evaluateTasks( + RunScript.evaluateTasksNamed( evaluator, Seq("mill.scalalib.giter8.Giter8Module/init") ++ args, - selectMode = SelectMode.Single - )(identity).map(_.value) + SelectMode.Single + ) + + () } private type VizWorker = ( diff --git a/main/src/mill/main/RootModule.scala b/main/src/mill/main/RootModule.scala index 7c19b0871a4..3ba1bfe54ab 100644 --- a/main/src/mill/main/RootModule.scala +++ b/main/src/mill/main/RootModule.scala @@ -1,7 +1,8 @@ package mill.main import mill.api.{PathRef, internal} -import mill.define.{Caller, Discover, Segments, Watchable} +import mill.util.Watchable +import mill.define.{Caller, Discover, Segments} import TokenReaders._ import scala.collection.mutable @@ -36,24 +37,6 @@ abstract class RootModule()(implicit // user-defined BaseModule can have a complete Discover[_] instance without // needing to tediously call `override lazy val millDiscover = Discover[this.type]` override lazy val millDiscover = baseModuleInfo.discover.asInstanceOf[Discover[this.type]] - - object interp { - - def watchValue[T](v0: => T): T = { - val v = v0 - val watchable = Watchable.Value(() => v0.hashCode, v.hashCode()) - watchedValues.append(watchable) - v - } - - def watch(p: os.Path): os.Path = { - val watchable = Watchable.Path(PathRef(p)) - watchedValues.append(watchable) - p - } - } - - protected[mill] val watchedValues = mutable.Buffer.empty[Watchable] } @internal diff --git a/main/src/mill/main/RunScript.scala b/main/src/mill/main/RunScript.scala index 36227805f89..713012cb224 100644 --- a/main/src/mill/main/RunScript.scala +++ b/main/src/mill/main/RunScript.scala @@ -2,61 +2,25 @@ package mill.main import mill.define._ import mill.eval.{Evaluator, EvaluatorPaths} -import mill.util.{EitherOps, Watched} -import mill.api.{PathRef, Result} +import mill.util.Watchable +import mill.api.{PathRef, Result, Val} import mill.api.Strict.Agg -import scala.reflect.ClassTag -import mill.main.ParseArgs.TargetsWithParams +import Evaluator._ object RunScript { type TaskName = String - def evaluateTasks( - evaluator: Evaluator, - scriptArgs: Seq[String], - selectMode: SelectMode - ): Either[String, (Seq[PathRef], Either[String, Seq[(Any, Option[ujson.Value])]])] = { - for (targets <- ResolveTasks.resolve(evaluator, scriptArgs, selectMode)) - yield { - val (watched, res) = evaluate(evaluator, Agg.from(targets.distinct)) - - val watched2 = for { - x <- res.toSeq - (Watched(_, extraWatched), _) <- x - w <- extraWatched - } yield w - - (watched ++ watched2, res) - } - } - def evaluateTasksNamed[T]( evaluator: Evaluator, scriptArgs: Seq[String], selectMode: SelectMode - ): Either[String, (Seq[PathRef], Either[String, Seq[(Any, Option[(TaskName, ujson.Value)])]])] = { + ): Either[ + String, + (Seq[Watchable], Either[String, Seq[(Any, Option[(TaskName, ujson.Value)])]]) + ] = { for (targets <- ResolveTasks.resolve(evaluator, scriptArgs, selectMode)) - yield { - val (watched, res) = evaluateNamed(evaluator, Agg.from(targets.distinct)) - - val watched2 = for { - x <- res.toSeq - (Watched(_, extraWatched), _) <- x - w <- extraWatched - } yield w - - (watched ++ watched2, res) - } - } - - def evaluate( - evaluator: Evaluator, - targets: Agg[Task[Any]] - ): (Seq[PathRef], Either[String, Seq[(Any, Option[ujson.Value])]]) = { - val (watched, results) = evaluateNamed(evaluator, targets) - // we drop the task name in the inner tuple - (watched, results.map(_.map(p => (p._1, p._2.map(_._2))))) + yield evaluateNamed(evaluator, Agg.from(targets.distinct)) } /** @@ -67,13 +31,19 @@ object RunScript { def evaluateNamed( evaluator: Evaluator, targets: Agg[Task[Any]] - ): (Seq[PathRef], Either[String, Seq[(Any, Option[(TaskName, ujson.Value)])]]) = { - val evaluated: Evaluator.Results = evaluator.evaluate(targets) + ): (Seq[Watchable], Either[String, Seq[(Any, Option[(TaskName, ujson.Value)])]]) = { + val evaluated: Results = evaluator.evaluate(targets) + val watched = evaluated.results .iterator .collect { - case (t: SourcesImpl, Result.Success(ps: Seq[PathRef])) => ps - case (t: SourceImpl, Result.Success(p: PathRef)) => Seq(p) + case (t: SourcesImpl, TaskResult(Result.Success(Val(ps: Seq[PathRef])), _)) => + ps.map(Watchable.Path(_)) + case (t: SourceImpl, TaskResult(Result.Success(Val(p: PathRef)), _)) => + Seq(Watchable.Path(p)) + case (t: InputImpl[_], TaskResult(_, Some(recalc))) => + val pretty = t.ctx0.fileName + ":" + t.ctx0.lineNum + Seq(Watchable.Value(() => recalc().hashCode(), recalc().hashCode(), pretty)) } .flatten .toSeq diff --git a/main/src/mill/main/VisualizeModule.scala b/main/src/mill/main/VisualizeModule.scala index ecf29d4330d..e4309949db5 100644 --- a/main/src/mill/main/VisualizeModule.scala +++ b/main/src/mill/main/VisualizeModule.scala @@ -42,7 +42,7 @@ trait VisualizeModule extends mill.define.TaskModule { ) val visualizeThread = new java.lang.Thread(() => while (true) { - val res = Result.create { + val res = Result.Success { val (targets, tasks, dest) = in.take() cl.loadClass("mill.main.graphviz.GraphvizTools") .getMethod("apply", classOf[Seq[_]], classOf[Seq[_]], classOf[os.Path]) diff --git a/main/src/mill/modules/CoursierSupport.scala b/main/src/mill/modules/CoursierSupport.scala index a56de052221..f2b9c5b3960 100644 --- a/main/src/mill/modules/CoursierSupport.scala +++ b/main/src/mill/modules/CoursierSupport.scala @@ -193,10 +193,11 @@ trait CoursierSupport { } if (errors.isEmpty) { - mill.Agg.from( - successes.map(os.Path(_)).filter(_.ext == "jar").map(PathRef(_, quick = true)) - ).filter(x => resolveFilter(x.path)) ++ localTestDeps.flatten - + Result.Success( + mill.Agg.from( + successes.map(os.Path(_)).filter(_.ext == "jar").map(PathRef(_, quick = true)) + ).filter(x => resolveFilter(x.path)) ++ localTestDeps.flatten + ) } else { val errorDetails = errors.map(e => s"${System.lineSeparator()} ${e.describe}").mkString Result.Failure( diff --git a/main/test/src/mill/main/MainModuleTests.scala b/main/test/src/mill/main/MainModuleTests.scala index aca9b114add..9a4115fcafa 100644 --- a/main/test/src/mill/main/MainModuleTests.scala +++ b/main/test/src/mill/main/MainModuleTests.scala @@ -1,6 +1,6 @@ package mill.main -import mill.api.{PathRef, Result} +import mill.api.{PathRef, Result, Val} import mill.{Agg, T} import mill.define.{Cross, Module} import mill.util.{TestEvaluator, TestUtil} @@ -49,7 +49,7 @@ object MainModuleTests extends TestSuite { val eval = new TestEvaluator(mainModule) test("single") { val res = eval.evaluator.evaluate(Agg(mainModule.inspect(eval.evaluator, "hello"))) - val Result.Success(value: String) = res.rawValues.head + val Result.Success(Val(value: String)) = res.rawValues.head assert( res.failing.keyCount == 0, value.startsWith("hello("), @@ -59,7 +59,7 @@ object MainModuleTests extends TestSuite { test("multi") { val res = eval.evaluator.evaluate(Agg(mainModule.inspect(eval.evaluator, "hello", "hello2"))) - val Result.Success(value: String) = res.rawValues.head + val Result.Success(Val(value: String)) = res.rawValues.head assert( res.failing.keyCount == 0, value.startsWith("hello("), @@ -77,7 +77,7 @@ object MainModuleTests extends TestSuite { assert(results.failing.keyCount == 0) - val Result.Success(value) = results.rawValues.head + val Result.Success(Val(value)) = results.rawValues.head assert(value == ujson.Arr.from(Seq("hello", "world"))) } @@ -92,7 +92,7 @@ object MainModuleTests extends TestSuite { assert(results.failing.keyCount == 0) - val Result.Success(value) = results.rawValues.head + val Result.Success(Val(value)) = results.rawValues.head assert(value == ujson.Arr.from(Seq( ujson.Arr.from(Seq("hello", "world")), @@ -109,7 +109,7 @@ object MainModuleTests extends TestSuite { assert(results.failing.keyCount == 0) - val Result.Success(value) = results.rawValues.head + val Result.Success(Val(value)) = results.rawValues.head assert(value == ujson.Obj.from(Map( "hello" -> ujson.Arr.from(Seq("hello", "world")) @@ -126,7 +126,7 @@ object MainModuleTests extends TestSuite { assert(results.failing.keyCount == 0) - val Result.Success(value) = results.rawValues.head + val Result.Success(Val(value)) = results.rawValues.head assert(value == ujson.Obj.from(Map( "hello" -> ujson.Arr.from(Seq("hello", "world")), diff --git a/main/testkit/src/mill/testkit/MillTestkit.scala b/main/testkit/src/mill/testkit/MillTestkit.scala index cfd7d3cc2fd..a923a63410d 100644 --- a/main/testkit/src/mill/testkit/MillTestkit.scala +++ b/main/testkit/src/mill/testkit/MillTestkit.scala @@ -1,8 +1,8 @@ package mill.testkit import mill._ -import mill.define.{Discover, TargetImpl, InputImpl} -import mill.api.{DummyInputStream, Result, SystemStreams} +import mill.define.{Discover, InputImpl, TargetImpl} +import mill.api.{DummyInputStream, Result, SystemStreams, Val} import mill.api.Result.OuterStack import mill.api.Strict.Agg @@ -109,7 +109,7 @@ trait MillTestKit { if (evaluated.failing.keyCount == 0) { Right( Tuple2( - evaluated.rawValues.head.asInstanceOf[Result.Success[T]].value, + evaluated.rawValues.head.asInstanceOf[Result.Success[Val]].value.value.asInstanceOf[T], evaluated.evaluated.collect { case t: TargetImpl[_] if module.millInternal.targets.contains(t) @@ -120,8 +120,16 @@ trait MillTestKit { ) } else { Left( - evaluated.failing.lookupKey(evaluated.failing.keys().next).items.next() - .asInstanceOf[Result.Failing[T]] + evaluated + .failing + .lookupKey(evaluated.failing.keys().next) + .items + .next() + .asFailing + .get + .map { (x: Val) => + x.value.asInstanceOf[T] + } ) } } @@ -132,7 +140,7 @@ trait MillTestKit { val cleaned = res.rawValues.map { case Result.Exception(ex, _) => Result.Exception(ex, new OuterStack(Nil)) - case x => x + case x => x.map(_.value) } assert( diff --git a/main/define/src/mill/define/Watchable.scala b/main/util/src/mill/util/Watchable.scala similarity index 80% rename from main/define/src/mill/define/Watchable.scala rename to main/util/src/mill/util/Watchable.scala index 66949e664d8..f465379937e 100644 --- a/main/define/src/mill/define/Watchable.scala +++ b/main/util/src/mill/util/Watchable.scala @@ -1,4 +1,4 @@ -package mill.define +package mill.util import mill.api.internal @@ -13,14 +13,16 @@ private[mill] trait Watchable { def poll(): Long def signature: Long def validate(): Boolean = poll() == signature + def pretty: String } @internal private[mill] object Watchable { case class Path(p: mill.api.PathRef) extends Watchable { def poll() = p.recomputeSig() def signature = p.sig + def pretty = p.toString } - case class Value(f: () => Long, signature: Long) extends Watchable { + case class Value(f: () => Long, signature: Long, pretty: String) extends Watchable { def poll() = f() } } diff --git a/main/util/src/mill/util/Watched.scala b/main/util/src/mill/util/Watched.scala deleted file mode 100644 index 4e7ee66a6ee..00000000000 --- a/main/util/src/mill/util/Watched.scala +++ /dev/null @@ -1,9 +0,0 @@ -package mill.util - -import mill.api.PathRef - -case class Watched[T](value: T, watched: Seq[PathRef]) -object Watched { - implicit def readWrite[T: upickle.default.ReadWriter]: upickle.default.ReadWriter[Watched[T]] = - upickle.default.macroRW[Watched[T]] -} diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index d2484e493ea..0b09606e8e3 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -1,12 +1,11 @@ package mill.runner -import mill.util.{ColorLogger, PrefixLogger, Util} +import mill.util.{ColorLogger, PrefixLogger, Util, Watchable} import mill.{BuildInfo, T} -import mill.api.{PathRef, internal} +import mill.api.{PathRef, Val, internal} import mill.eval.Evaluator - import mill.main.{RootModule, RunScript, SelectMode} import mill.main.TokenReaders._ -import mill.define.{Discover, Segments, Watchable} +import mill.define.{Discover, Segments} import java.net.URLClassLoader @@ -50,7 +49,7 @@ class MillBuildBootstrap( } Watching.Result( - watched = runnerState.frames.flatMap(_.evalWatched), + watched = runnerState.frames.flatMap(f => f.evalWatched ++ f.moduleWatched), error = runnerState.errorOpt, result = runnerState ) @@ -194,8 +193,8 @@ class MillBuildBootstrap( case ( Right(Seq( - runClasspath: Seq[PathRef], - scriptImportGraph: Map[os.Path, (Int, Seq[os.Path])] + Val(runClasspath: Seq[PathRef]), + Val(scriptImportGraph: Map[os.Path, (Int, Seq[os.Path])]) )), evalWatches, moduleWatches @@ -268,7 +267,7 @@ class MillBuildBootstrap( } def makeEvaluator( - workerCache: Map[Segments, (Int, Any)], + workerCache: Map[Segments, (Int, Val)], scriptImportGraph: Map[os.Path, (Int, Seq[os.Path])], rootModule: RootModule, millClassloaderSigHash: Int, @@ -333,17 +332,19 @@ object MillBuildBootstrap { evaluator: Evaluator, targetsAndParams: Seq[String] ): (Either[String, Seq[Any]], Seq[Watchable], Seq[Watchable]) = { - - val evalTaskResult = RunScript.evaluateTasks(evaluator, targetsAndParams, SelectMode.Separated) + rootModule.evalWatchedValues.clear() + val evalTaskResult = + RunScript.evaluateTasksNamed(evaluator, targetsAndParams, SelectMode.Separated) val moduleWatched = rootModule.watchedValues.toVector + val addedEvalWatched = rootModule.evalWatchedValues.toVector evalTaskResult match { case Left(msg) => (Left(msg), Nil, moduleWatched) - case Right((watchedPaths, evaluated)) => - val evalWatched = watchedPaths.map(Watchable.Path) + case Right((watched, evaluated)) => evaluated match { - case Left(msg) => (Left(msg), evalWatched, moduleWatched) - case Right(results) => (Right(results.map(_._1)), evalWatched, moduleWatched) + case Left(msg) => (Left(msg), watched ++ addedEvalWatched, moduleWatched) + case Right(results) => + (Right(results.map(_._1)), watched ++ addedEvalWatched, moduleWatched) } } } diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index 4c321a73fb1..af94a7d52f3 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -27,10 +27,10 @@ import scala.util.Try @internal class MillBuildRootModule()(implicit baseModuleInfo: RootModule.Info, - millBuildRootModule: MillBuildRootModule.Info + millBuildRootModuleInfo: MillBuildRootModule.Info ) extends RootModule() with ScalaModule { - override def millSourcePath = millBuildRootModule.projectRoot / os.up / "mill-build" + override def millSourcePath = millBuildRootModuleInfo.projectRoot / os.up / "mill-build" override def resolveDeps( deps: Task[Agg[BoundDep]], @@ -46,11 +46,18 @@ class MillBuildRootModule()(implicit override def scalaVersion = "2.13.10" - def parseBuildFiles: T[FileImportGraph] = T.input { - FileImportGraph.parseBuildFiles( - millBuildRootModule.topLevelProjectRoot, - millBuildRootModule.projectRoot / os.up - ) + def scriptSources = T.sources { + MillBuildRootModule + .parseBuildFiles(millBuildRootModuleInfo) + .seenScripts + .keys + .map(PathRef(_)) + .toSeq + } + + def parseBuildFiles = T { + scriptSources() + MillBuildRootModule.parseBuildFiles(millBuildRootModuleInfo) } override def repositoriesTask: Task[Seq[Repository]] = { @@ -93,10 +100,6 @@ class MillBuildRootModule()(implicit Seq(ivy"com.lihaoyi::mill-moduledefs:${Versions.millModuledefsVersion}") } - def scriptSources: T[Seq[PathRef]] = T.sources { - for ((p, s) <- parseBuildFiles().seenScripts.toSeq) yield PathRef(p) - } - override def generatedSources: T[Seq[PathRef]] = T { generateScriptSources() } @@ -106,12 +109,12 @@ class MillBuildRootModule()(implicit if (parsed.errors.nonEmpty) Result.Failure(parsed.errors.mkString("\n")) else { MillBuildRootModule.generateWrappedSources( - millBuildRootModule.projectRoot / os.up, + millBuildRootModuleInfo.projectRoot / os.up, scriptSources(), parsed.seenScripts, T.dest, - millBuildRootModule.enclosingClasspath, - millBuildRootModule.topLevelProjectRoot + millBuildRootModuleInfo.enclosingClasspath, + millBuildRootModuleInfo.topLevelProjectRoot ) Result.Success(Seq(PathRef(T.dest))) } @@ -129,11 +132,11 @@ class MillBuildRootModule()(implicit Lib.findSourceFiles(allSources(), Seq("scala", "java", "sc")).map(PathRef(_)) } - override def unmanagedClasspath: T[Agg[PathRef]] = mill.define.Target.input { - mill.api.Loose.Agg.from( - millBuildRootModule.enclosingClasspath.map(p => mill.api.PathRef(p, quick = true)) - ) ++ - lineNumberPluginClasspath() + def enclosingClasspath = T.sources { + millBuildRootModuleInfo.enclosingClasspath.map(p => mill.api.PathRef(p, quick = true)) + } + override def unmanagedClasspath: T[Agg[PathRef]] = T { + enclosingClasspath() ++ lineNumberPluginClasspath() } override def scalacPluginIvyDeps = Agg( @@ -185,6 +188,13 @@ object MillBuildRootModule { topLevelProjectRoot: os.Path ) + def parseBuildFiles(millBuildRootModuleInfo: MillBuildRootModule.Info) = { + FileImportGraph.parseBuildFiles( + millBuildRootModuleInfo.topLevelProjectRoot, + millBuildRootModuleInfo.projectRoot / os.up + ) + } + def generateWrappedSources( base: os.Path, scriptSources: Seq[PathRef], diff --git a/runner/src/mill/runner/RunnerState.scala b/runner/src/mill/runner/RunnerState.scala index 3c60d76a951..399d8c20fa5 100644 --- a/runner/src/mill/runner/RunnerState.scala +++ b/runner/src/mill/runner/RunnerState.scala @@ -1,9 +1,11 @@ package mill.runner -import mill.api.{PathRef, internal} -import mill.define.{BaseModule, Segments, Watchable} +import mill.api.{PathRef, Val, internal} +import mill.define.{BaseModule, Segments} +import mill.util.Watchable import upickle.default.{ReadWriter, macroRW} import mill.api.JsonFormatters._ +import mill.eval.Evaluator import mill.main.RootModule /** @@ -50,7 +52,7 @@ object RunnerState { @internal case class Frame( - workerCache: Map[Segments, (Int, Any)], + workerCache: Map[Segments, (Int, Val)], evalWatched: Seq[Watchable], moduleWatched: Seq[Watchable], scriptImportGraph: Map[os.Path, (Int, Seq[os.Path])], diff --git a/runner/src/mill/runner/Watching.scala b/runner/src/mill/runner/Watching.scala index a55f21bce62..2e65e36b161 100644 --- a/runner/src/mill/runner/Watching.scala +++ b/runner/src/mill/runner/Watching.scala @@ -1,8 +1,7 @@ package mill.runner import mill.api.internal -import mill.define.Watchable -import mill.util.ColorLogger +import mill.util.{ColorLogger, Watchable} import mill.api.SystemStreams import java.io.InputStream import scala.annotation.tailrec diff --git a/scalalib/src/mill/scalalib/GenIdeaImpl.scala b/scalalib/src/mill/scalalib/GenIdeaImpl.scala index 4f0c97cc613..10433d13257 100755 --- a/scalalib/src/mill/scalalib/GenIdeaImpl.scala +++ b/scalalib/src/mill/scalalib/GenIdeaImpl.scala @@ -243,7 +243,7 @@ case class GenIdeaImpl( evaluator.evaluate(resolveTasks) match { case r if r.failing.items().nonEmpty => throw GenIdeaException(s"Failure during resolving modules: ${Evaluator.formatFailing(r)}") - case r => r.values.asInstanceOf[Seq[ResolvedModule]] + case r => r.values.map(_.value).asInstanceOf[Seq[ResolvedModule]] } val moduleLabels = modules.map(_.swap).toMap @@ -537,6 +537,7 @@ case class GenIdeaImpl( ) ) .values + .map(_.value) val generatedSourcePaths = generatedSourcePathRefs.map(_.path) val normalSourcePaths = (allSourcesPathRefs @@ -550,6 +551,7 @@ case class GenIdeaImpl( .evaluate(Agg(x.scalaVersion)) .values .head + .value .asInstanceOf[String] ) case _ => None