From 26143723e5c4606a4fa19eac94c2baadd84ecedc Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Tue, 29 Aug 2023 17:53:03 +0200 Subject: [PATCH 01/19] Allow Mill CLI to select the meta-build frame it operates on Add a new CLI option `--frame` acception an `Int`. Default in `0` and means the root project, `1` is the parent meta-build, if defined, or the built-in bootstrap module, and so on. This is a first draft, to get more familiar with the recursive but mutable nature of our meta-build support. Don't hessitate to point out shortcomings. * Fixes https://github.com/com-lihaoyi/mill/issues/2658 Review by @lihaoyi --- .../src/mill/runner/MillBuildBootstrap.scala | 56 +++++++++++++++---- runner/src/mill/runner/MillCliConfig.scala | 19 +++++-- runner/src/mill/runner/MillMain.scala | 11 +++- 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 0b14b50a456..aa659f5f081 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -37,7 +37,8 @@ class MillBuildBootstrap( prevRunnerState: RunnerState, logger: ColorLogger, disableCallgraphInvalidation: Boolean, - needBuildSc: Boolean + needBuildSc: Boolean, + requestedFrame: Option[Int] ) { import MillBuildBootstrap._ @@ -66,6 +67,8 @@ class MillBuildBootstrap( val prevFrameOpt = prevRunnerState.frames.lift(depth) val prevOuterFrameOpt = prevRunnerState.frames.lift(depth - 1) + val requestedDepth = requestedFrame.filter(_ >= 0).getOrElse(0) + val nestedState = if (depth == 0) { // On this level we typically want assume a Mill project, which means we want to require an existing `build.sc`. @@ -112,7 +115,10 @@ class MillBuildBootstrap( val res = if (nestedState.errorOpt.isDefined) nestedState.add(errorOpt = nestedState.errorOpt) - else { + else if (depth == 0 && requestedDepth > nestedState.frames.size) { + // User has requested a frame depth, we actually don't have + nestedState.add(errorOpt = Some(s"The project has no meta-build frame ${requestedDepth}")) + } else { val validatedRootModuleOrErr = nestedState.frames.headOption match { case None => getChildRootModule(nestedState.bootstrapModuleOpt.get, depth, projectRoot) @@ -149,14 +155,40 @@ class MillBuildBootstrap( depth ) - if (depth != 0) processRunClasspath( - nestedState, - rootModule, - evaluator, - prevFrameOpt, - prevOuterFrameOpt - ) - else processFinalTargets(nestedState, rootModule, evaluator) + if (depth != 0) { + val retState = processRunClasspath( + nestedState, + rootModule, + evaluator, + prevFrameOpt, + prevOuterFrameOpt + ) + + if (retState.errorOpt.isEmpty && depth == requestedDepth) { + val evalRet = processFinalTargets(nestedState, rootModule, evaluator) + if (evalRet.errorOpt.isEmpty) retState + else evalRet + } else + retState + + } else { + if (depth == requestedDepth) { + processFinalTargets(nestedState, rootModule, evaluator) + } else { + // TODO what now, we already evaluated some level below, so we can just return with Success? + val evalState = RunnerState.Frame( + evaluator.workerCache.toMap, + Seq.empty, + Seq.empty, + Map.empty, + Map.empty, + None, + Nil, + evaluator + ) + nestedState.add(frame = evalState, errorOpt = None) + } + } } } // println(s"-evaluateRec($depth) " + recRoot(projectRoot, depth)) @@ -414,11 +446,11 @@ object MillBuildBootstrap { ) } - def recRoot(projectRoot: os.Path, depth: Int) = { + def recRoot(projectRoot: os.Path, depth: Int): os.Path = { projectRoot / Seq.fill(depth)("mill-build") } - def recOut(projectRoot: os.Path, depth: Int) = { + def recOut(projectRoot: os.Path, depth: Int): os.Path = { projectRoot / "out" / Seq.fill(depth)("mill-build") } } diff --git a/runner/src/mill/runner/MillCliConfig.scala b/runner/src/mill/runner/MillCliConfig.scala index b8d1b1a5a62..8ebe2b0ed6f 100644 --- a/runner/src/mill/runner/MillCliConfig.scala +++ b/runner/src/mill/runner/MillCliConfig.scala @@ -117,7 +117,11 @@ class MillCliConfig private ( code changes, and instead fall back to the previous coarse-grained implementation relying on the script `import $file` graph""" ) - val disableCallgraphInvalidation: Flag + val disableCallgraphInvalidation: Flag, + @arg( + doc = """Experimental: Select a meta-build to run the given targets.""" + ) + val frame: Option[Int] ) { override def toString: String = Seq( "home" -> home, @@ -139,7 +143,8 @@ class MillCliConfig private ( "silent" -> silent, "leftoverArgs" -> leftoverArgs, "color" -> color, - "disableCallgraphInvalidation" -> disableCallgraphInvalidation + "disableCallgraphInvalidation" -> disableCallgraphInvalidation, + "frame" -> frame ).map(p => s"${p._1}=${p._2}").mkString(getClass().getSimpleName + "(", ",", ")") } @@ -148,8 +153,8 @@ object MillCliConfig { * mainargs requires us to keep this apply method in sync with the private ctr of the class. * mainargs is designed to work with case classes, * but case classes can't be evolved in a binary compatible fashion. - * mainargs parses the class ctr for itss internal model, - * but used the companion's apply to actually create an instance of the config class, + * mainargs parses the class ctr for its internal model, + * but uses the companion's apply to actually create an instance of the config class, * hence we need both in sync. */ def apply( @@ -173,7 +178,8 @@ object MillCliConfig { silent: Flag = Flag(), leftoverArgs: Leftover[String] = Leftover(), color: Option[Boolean] = None, - disableCallgraphInvalidation: Flag = Flag() + disableCallgraphInvalidation: Flag = Flag(), + frame: Option[Int] = None ): MillCliConfig = new MillCliConfig( home = home, repl = repl, @@ -194,7 +200,8 @@ object MillCliConfig { silent = silent, leftoverArgs = leftoverArgs, color = color, - disableCallgraphInvalidation + disableCallgraphInvalidation, + frame = frame ) @deprecated("Bin-compat shim", "Mill after 0.11.0") diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala index 4c2ff9578c8..981084d9cb7 100644 --- a/runner/src/mill/runner/MillMain.scala +++ b/runner/src/mill/runner/MillMain.scala @@ -126,6 +126,11 @@ object MillMain { ) (false, RunnerState.empty) + // Check non-negative --frame option + case Right(config) if config.frame.exists(_ < 0) => + streams.err.println("--frame cannot be negative") + (false, RunnerState.empty) + case Right(config) => val logger = getLogger( streams, @@ -204,7 +209,8 @@ object MillMain { prevRunnerState = prevState.getOrElse(stateCache), logger = logger, disableCallgraphInvalidation = config.disableCallgraphInvalidation.value, - needBuildSc = needBuildSc(config) + needBuildSc = needBuildSc(config), + requestedFrame = config.frame ).evaluate() } ) @@ -224,8 +230,9 @@ object MillMain { ) BspContext.bspServerHandle.stop() } - loopRes + // return with evaluation result + loopRes } } if (config.ringBell.value) { From d1612300795d305f91b56ad435937740463602ed Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Wed, 30 Aug 2023 11:16:29 +0200 Subject: [PATCH 02/19] Added mill.runner.MillBuild to query for frameCount from the CLI --- runner/src/mill/runner/MillBuild.scala | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 runner/src/mill/runner/MillBuild.scala diff --git a/runner/src/mill/runner/MillBuild.scala b/runner/src/mill/runner/MillBuild.scala new file mode 100644 index 00000000000..36a0fdbf935 --- /dev/null +++ b/runner/src/mill/runner/MillBuild.scala @@ -0,0 +1,19 @@ +package mill.runner + +import mill.T +import mill.define.{Command, Discover, ExternalModule, Module} +import mill.eval.Evaluator.AllBootstrapEvaluators + +trait MillBuild extends Module { + + def frameCount(evaluators: AllBootstrapEvaluators): Command[Int] = T.command { + val count = evaluators.value.size + T.log.outputStream.println(s"${count}") + count + } + +} + +object MillBuild extends ExternalModule with MillBuild { + override lazy val millDiscover = Discover[this.type] +} From a3761a1864acdd3bb29ddff49cd3738afe78de14 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Wed, 30 Aug 2023 11:19:49 +0200 Subject: [PATCH 03/19] Don't evaluate higher level frames if we run with --frame option I applied one hack-ish way to avoid instantiating the higher level evaluators by just assigning `null`. This should be refactored, e.g. by representing a skipped frame by it's own case class. --- .../src/mill/runner/MillBuildBootstrap.scala | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index aa659f5f081..65b46869332 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -117,7 +117,23 @@ class MillBuildBootstrap( if (nestedState.errorOpt.isDefined) nestedState.add(errorOpt = nestedState.errorOpt) else if (depth == 0 && requestedDepth > nestedState.frames.size) { // User has requested a frame depth, we actually don't have - nestedState.add(errorOpt = Some(s"The project has no meta-build frame ${requestedDepth}")) + nestedState.add(errorOpt = Some(s"Invalid selected frame ${requestedDepth}. Valid range: 0 .. ${nestedState.frames.size}")) + } else if (depth < requestedDepth) { + // We already evaluated, hence we just need to make sure, we return a proper structure with all already existing watch data + val evalState = RunnerState.Frame( + prevFrameOpt.map(_.workerCache).getOrElse(Map.empty), + Seq.empty, + Seq.empty, + Map.empty, + Map.empty, + None, + Nil, + // FIXME we don't want to evaluator anything in this depth, so we just skip creating an evaluator, + // mainly because we didn't even constructed (compiled) it's classpath + // TODO: Instead, introduce a skipped frame type, so we can better comunicate that state + null + ) + nestedState.add(frame = evalState, errorOpt = None) } else { val validatedRootModuleOrErr = nestedState.frames.headOption match { case None => @@ -165,6 +181,7 @@ class MillBuildBootstrap( ) if (retState.errorOpt.isEmpty && depth == requestedDepth) { + // TODO: print some message and indicate actual evaluated frame val evalRet = processFinalTargets(nestedState, rootModule, evaluator) if (evalRet.errorOpt.isEmpty) retState else evalRet @@ -172,22 +189,7 @@ class MillBuildBootstrap( retState } else { - if (depth == requestedDepth) { - processFinalTargets(nestedState, rootModule, evaluator) - } else { - // TODO what now, we already evaluated some level below, so we can just return with Success? - val evalState = RunnerState.Frame( - evaluator.workerCache.toMap, - Seq.empty, - Seq.empty, - Map.empty, - Map.empty, - None, - Nil, - evaluator - ) - nestedState.add(frame = evalState, errorOpt = None) - } + processFinalTargets(nestedState, rootModule, evaluator) } } } From b7b116daa6784fa92acdf70d5840f975f83f2cee Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 11:51:12 +0200 Subject: [PATCH 04/19] Renamed `--frame` cli option to `--meta-level` --- runner/src/mill/runner/MillBuildBootstrap.scala | 10 +++++++--- runner/src/mill/runner/MillCliConfig.scala | 12 ++++++++---- runner/src/mill/runner/MillMain.scala | 4 ++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 65b46869332..6a6a026653a 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -38,7 +38,7 @@ class MillBuildBootstrap( logger: ColorLogger, disableCallgraphInvalidation: Boolean, needBuildSc: Boolean, - requestedFrame: Option[Int] + requestedMetaLevel: Option[Int] ) { import MillBuildBootstrap._ @@ -67,7 +67,7 @@ class MillBuildBootstrap( val prevFrameOpt = prevRunnerState.frames.lift(depth) val prevOuterFrameOpt = prevRunnerState.frames.lift(depth - 1) - val requestedDepth = requestedFrame.filter(_ >= 0).getOrElse(0) + val requestedDepth = requestedMetaLevel.filter(_ >= 0).getOrElse(0) val nestedState = if (depth == 0) { @@ -117,7 +117,11 @@ class MillBuildBootstrap( if (nestedState.errorOpt.isDefined) nestedState.add(errorOpt = nestedState.errorOpt) else if (depth == 0 && requestedDepth > nestedState.frames.size) { // User has requested a frame depth, we actually don't have - nestedState.add(errorOpt = Some(s"Invalid selected frame ${requestedDepth}. Valid range: 0 .. ${nestedState.frames.size}")) + nestedState.add(errorOpt = + Some( + s"Invalid selected frame ${requestedDepth}. Valid range: 0 .. ${nestedState.frames.size}" + ) + ) } else if (depth < requestedDepth) { // We already evaluated, hence we just need to make sure, we return a proper structure with all already existing watch data val evalState = RunnerState.Frame( diff --git a/runner/src/mill/runner/MillCliConfig.scala b/runner/src/mill/runner/MillCliConfig.scala index 8ebe2b0ed6f..bbe9951e957 100644 --- a/runner/src/mill/runner/MillCliConfig.scala +++ b/runner/src/mill/runner/MillCliConfig.scala @@ -119,9 +119,13 @@ class MillCliConfig private ( ) val disableCallgraphInvalidation: Flag, @arg( - doc = """Experimental: Select a meta-build to run the given targets.""" + name = "meta-level", + doc = + """Experimental: Select a meta-build level to run the given targets. + Level 0 is the normal project, level 1 the first meta-build, and so on. + The last level is the built-in synthetic meta-build which Mill uses to bootstrap the project.""" ) - val frame: Option[Int] + val metaLevel: Option[Int] ) { override def toString: String = Seq( "home" -> home, @@ -144,7 +148,7 @@ class MillCliConfig private ( "leftoverArgs" -> leftoverArgs, "color" -> color, "disableCallgraphInvalidation" -> disableCallgraphInvalidation, - "frame" -> frame + "metaLevel" -> metaLevel ).map(p => s"${p._1}=${p._2}").mkString(getClass().getSimpleName + "(", ",", ")") } @@ -201,7 +205,7 @@ object MillCliConfig { leftoverArgs = leftoverArgs, color = color, disableCallgraphInvalidation, - frame = frame + metaLevel = frame ) @deprecated("Bin-compat shim", "Mill after 0.11.0") diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala index 981084d9cb7..a808a06b79d 100644 --- a/runner/src/mill/runner/MillMain.scala +++ b/runner/src/mill/runner/MillMain.scala @@ -127,7 +127,7 @@ object MillMain { (false, RunnerState.empty) // Check non-negative --frame option - case Right(config) if config.frame.exists(_ < 0) => + case Right(config) if config.metaLevel.exists(_ < 0) => streams.err.println("--frame cannot be negative") (false, RunnerState.empty) @@ -210,7 +210,7 @@ object MillMain { logger = logger, disableCallgraphInvalidation = config.disableCallgraphInvalidation.value, needBuildSc = needBuildSc(config), - requestedFrame = config.frame + requestedMetaLevel = config.metaLevel ).evaluate() } ) From e06ce61121b018966674f6c679930e803e5f0a60 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 16:53:00 +0200 Subject: [PATCH 05/19] Rename MillBuild.frameCount to levelCount --- runner/src/mill/runner/MillBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/src/mill/runner/MillBuild.scala b/runner/src/mill/runner/MillBuild.scala index 36a0fdbf935..ecd9d8c1549 100644 --- a/runner/src/mill/runner/MillBuild.scala +++ b/runner/src/mill/runner/MillBuild.scala @@ -6,7 +6,7 @@ import mill.eval.Evaluator.AllBootstrapEvaluators trait MillBuild extends Module { - def frameCount(evaluators: AllBootstrapEvaluators): Command[Int] = T.command { + def levelCount(evaluators: AllBootstrapEvaluators): Command[Int] = T.command { val count = evaluators.value.size T.log.outputStream.println(s"${count}") count From 5dea10a58e445da22362e5cf37176afb45c47a51 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 16:53:25 +0200 Subject: [PATCH 06/19] Updated doc: --help --- docs/modules/ROOT/pages/Intro_to_Mill.adoc | 71 +++++++++++++--------- example/misc/5-module-run-task/build.sc | 4 +- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/docs/modules/ROOT/pages/Intro_to_Mill.adoc b/docs/modules/ROOT/pages/Intro_to_Mill.adoc index c30e9ae0ae8..3afe16f9852 100644 --- a/docs/modules/ROOT/pages/Intro_to_Mill.adoc +++ b/docs/modules/ROOT/pages/Intro_to_Mill.adoc @@ -124,35 +124,48 @@ Run `mill --help` for a complete list of options ---- Mill Build Tool, version {mill-version} usage: mill [options] [[target [target-options]] [+ [target ...]]] - -D --define Define (or overwrite) a system property. - -b --bell Ring the bell once if the run completes successfully, twice if it fails. - --bsp Enable BSP server mode. - --color Enable or disable colored output; by default colors are enabled in both - REPL and scripts mode if the console is interactive, and disabled - otherwise. - -d --debug Show debug output on STDOUT - --disable-ticker Disable ticker log (e.g. short-lived prints of stages and progress bars). - --enable-ticker Enable ticker log (e.g. short-lived prints of stages and progress bars). - -h --home (internal) The home directory of internally used Ammonite script engine; - where it looks for config and caches. - --help Print this help message and exit. - -i --interactive Run Mill in interactive mode, suitable for opening REPLs and taking user - input. This implies --no-server and no mill server will be used. Must be - the first argument. - --import Additional ivy dependencies to load into mill, e.g. plugins. - -j --jobs Allow processing N targets in parallel. Use 1 to disable parallel and 0 to - use as much threads as available processors. - -k --keep-going Continue build, even after build failures. - --no-server Run Mill in single-process mode. In this mode, no Mill server will be - started or used. Must be the first argument. - --repl This flag is no longer supported. - -s --silent Make ivy logs during script import resolution go silent instead of - printing; though failures will still throw exception. - -v --version Show mill version information and exit. - -w --watch Watch and re-run your scripts when they change. - target ... The name or a pattern of the target(s) you want to build, followed by any - parameters you wish to pass to those targets. To specify multiple target - names or patterns, use the `+` separator. + -D --define Define (or overwrite) a system property. + -b --bell Ring the bell once if the run completes successfully, twice if + it fails. + --bsp Enable BSP server mode. + --color Enable or disable colored output; by default colors are enabled + in both REPL and scripts mode if the console is interactive, and + disabled otherwise. + -d --debug Show debug output on STDOUT + --disable-callgraph-invalidation Disable the fine-grained callgraph-based target invalidation in + response to code changes, and instead fall back to the previous + coarse-grained implementation relying on the script `import + $file` graph + --disable-ticker Disable ticker log (e.g. short-lived prints of stages and + progress bars). + --enable-ticker Enable ticker log (e.g. short-lived prints of stages and + progress bars). + -h --home (internal) The home directory of internally used Ammonite script + engine; where it looks for config and caches. + --help Print this help message and exit. + -i --interactive Run Mill in interactive mode, suitable for opening REPLs and + taking user input. This implies --no-server and no mill server + will be used. Must be the first argument. + --import Additional ivy dependencies to load into mill, e.g. plugins. + -j --jobs Allow processing N targets in parallel. Use 1 to disable + parallel and 0 to use as much threads as available processors. + -k --keep-going Continue build, even after build failures. + --meta-level Experimental: Select a meta-build level to run the given + targets. Level 0 is the normal project, level 1 the first + meta-build, and so on. The last level is the built-in synthetic + meta-build which Mill uses to bootstrap the project. + --no-server Run Mill in single-process mode. In this mode, no Mill server + will be started or used. Must be the first argument. + --repl This flag is no longer supported. + -s --silent Make ivy logs during script import resolution go silent instead + of printing; though failures will still throw exception. + -v --version Show mill version information and exit. + -w --watch Watch and re-run your scripts when they change. + target ... The name or a pattern of the target(s) you want to build, + followed by any parameters you wish to pass to those targets. To + specify multiple target names or patterns, use the `+` + separator. + ---- All _options_ must be given before the first target. diff --git a/example/misc/5-module-run-task/build.sc b/example/misc/5-module-run-task/build.sc index de5be4f5cf6..73f0c2a362d 100644 --- a/example/misc/5-module-run-task/build.sc +++ b/example/misc/5-module-run-task/build.sc @@ -17,7 +17,7 @@ object bar extends ScalaModule{ def ivyDeps = Agg(ivy"com.lihaoyi::os-lib:0.9.1") } -// This example demonstrates using Mill `ScalaModule`s as build tasks: rather +// This example demonstrates using Mill ``ScalaModule``s as build tasks: rather // than defining the task logic in the `build.sc`, we instead put the build // logic within the `bar` module as `bar/src/Bar.scala`. In this example, we use // `Bar.scala` as a source-code pre-processor on the `foo` module source code: @@ -34,7 +34,7 @@ Foo.value: HELLO */ // This example does a trivial string-replace of "hello" with "HELLO", but is -// enough to demonstrate how you can use Mill `ScalaModule`s to implement your +// enough to demonstrate how you can use Mill ``ScalaModule``s to implement your // own arbitrarily complex transformations. This is useful for build logic that // may not fit nicely inside a `build.sc` file, whether due to the sheer lines // of code or due to dependencies that may conflict with the Mill classpath From 85ed21b389cceba6d2944bb457baa6241f3c3674 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 17:40:30 +0200 Subject: [PATCH 07/19] Documented the new `--mill-level` option --- docs/modules/ROOT/pages/Extending_Mill.adoc | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/modules/ROOT/pages/Extending_Mill.adoc b/docs/modules/ROOT/pages/Extending_Mill.adoc index e35461ee74d..7f66b9b7cdd 100644 --- a/docs/modules/ROOT/pages/Extending_Mill.adoc +++ b/docs/modules/ROOT/pages/Extending_Mill.adoc @@ -11,6 +11,44 @@ include::example/misc/3-import-file-ivy.adoc[] == The Mill Meta-Build +The Meta-Build manages the compilation of the `build.sc`. +If you don't configure it explicitly, the Meta-Build is built-in into Mill. + +To customize it, you need to explicitly enable it with `import $meta._`. +Once enabled, it lives in the `mill-build/` directory. +A Meta-Build is required to contain a `MillBuildRootModule`. + +Meta-Builds are recursive, which means, a Meta-Build can itself have Meta-Builds, and so on. + +To run a task on a meta-build by specifying the `--meta-level` option to select the meta-build level. + +=== Example: Format the `build.sc` + +As an example of running a task on the meta-build, you can format the `build.sc` with Scalafmt. +Everything is already provided by Mill. +You only need a `.scalafmt.conf` config file which at least needs configure the Scalafmt version. + +.Run Scalafmt on the `build.sc` (and potentially included files) +---- +$ mill --meta-level 1 mill.scalalib.scalafmt.ScalafmtModule/reformatAll sources +---- + +* `--meta-level 1` selects the first meta-build. Without any customization, this is the only built-in meta-build. +* `mill.scalalib.scalafmt.ScalafmtModule/reformatAll` is a generic task to format scala source files with Scalafmt. It requires the targets that refer to the source files as argument +* `sources` this selects the `sources` targets of the meta-build, which at least contains the `build.sc`. + +=== Example: Find plugin updates + +Mill plugins are defined as `ivyDeps` in the meta-build. +Hence, you can easily search for updates with the external `mill.scalalib.Dependency` module. + +.Check for Mill Plugin updates +---- +$ mill --meta-level 1 mill.scalalib.Dependency/showUpdates +---- + +=== Example: Customizing the Meta-Build + include::example/misc/4-mill-build-folder.adoc[] == Using ScalaModule.run as a task From c189d5e84fb233f0fad8961958b0bcd85ab92e51 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 17:56:24 +0200 Subject: [PATCH 08/19] Added test cases for --meta-level option to examples --- example/misc/4-mill-build-folder/build.sc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/example/misc/4-mill-build-folder/build.sc b/example/misc/4-mill-build-folder/build.sc index bcf6ff01081..a7a1e92b563 100644 --- a/example/misc/4-mill-build-folder/build.sc +++ b/example/misc/4-mill-build-folder/build.sc @@ -63,3 +63,19 @@ scalatagsVersion: 0.8.2 */ +// You can also run tasks on the meta-build by using the `--meta-level` +// cli option. + +/** Usage + +> ./mill --meta-level 1 show sources +[ +.../build.sc", +.../mill-build/src" +] + +> ./mill --meta-level 2 show sources +.../mill-build/build.sc" + + +*/ \ No newline at end of file From 82e6069c2067b1b38ab6b43ab0a54af333873d97 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 18:02:16 +0200 Subject: [PATCH 09/19] Fixed wrong name --- runner/src/mill/runner/MillCliConfig.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/src/mill/runner/MillCliConfig.scala b/runner/src/mill/runner/MillCliConfig.scala index bbe9951e957..6d3206ffc2b 100644 --- a/runner/src/mill/runner/MillCliConfig.scala +++ b/runner/src/mill/runner/MillCliConfig.scala @@ -183,7 +183,7 @@ object MillCliConfig { leftoverArgs: Leftover[String] = Leftover(), color: Option[Boolean] = None, disableCallgraphInvalidation: Flag = Flag(), - frame: Option[Int] = None + metaLevel: Option[Int] = None ): MillCliConfig = new MillCliConfig( home = home, repl = repl, @@ -205,7 +205,7 @@ object MillCliConfig { leftoverArgs = leftoverArgs, color = color, disableCallgraphInvalidation, - metaLevel = frame + metaLevel = metaLevel ) @deprecated("Bin-compat shim", "Mill after 0.11.0") From 81c9a8eae4e03351552e9f74376b3c8b7c05bd06 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 18:05:11 +0200 Subject: [PATCH 10/19] wording --- docs/modules/ROOT/pages/Extending_Mill.adoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/modules/ROOT/pages/Extending_Mill.adoc b/docs/modules/ROOT/pages/Extending_Mill.adoc index 7f66b9b7cdd..eec55e5009c 100644 --- a/docs/modules/ROOT/pages/Extending_Mill.adoc +++ b/docs/modules/ROOT/pages/Extending_Mill.adoc @@ -11,12 +11,12 @@ include::example/misc/3-import-file-ivy.adoc[] == The Mill Meta-Build -The Meta-Build manages the compilation of the `build.sc`. -If you don't configure it explicitly, the Meta-Build is built-in into Mill. +The meta-build manages the compilation of the `build.sc`. +If you don't configure it explicitly, a built-in synthetic meta-build is used. To customize it, you need to explicitly enable it with `import $meta._`. -Once enabled, it lives in the `mill-build/` directory. -A Meta-Build is required to contain a `MillBuildRootModule`. +Once enabled, the meta-build lives in the `mill-build/` directory. +It needs to contain a top-level module of type `MillBuildRootModule`. Meta-Builds are recursive, which means, a Meta-Build can itself have Meta-Builds, and so on. From 56d8a67acecd5f457aa245dcd1248a7ffe0aca38 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 18:06:44 +0200 Subject: [PATCH 11/19] . --- docs/modules/ROOT/pages/Extending_Mill.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/ROOT/pages/Extending_Mill.adoc b/docs/modules/ROOT/pages/Extending_Mill.adoc index eec55e5009c..8a7af8fe633 100644 --- a/docs/modules/ROOT/pages/Extending_Mill.adoc +++ b/docs/modules/ROOT/pages/Extending_Mill.adoc @@ -18,9 +18,9 @@ To customize it, you need to explicitly enable it with `import $meta._`. Once enabled, the meta-build lives in the `mill-build/` directory. It needs to contain a top-level module of type `MillBuildRootModule`. -Meta-Builds are recursive, which means, a Meta-Build can itself have Meta-Builds, and so on. +Meta-builds are recursive, which means, it can itself have a nested meta-builds, and so on. -To run a task on a meta-build by specifying the `--meta-level` option to select the meta-build level. +To run a task on a meta-build, you specifying the `--meta-level` option to select the meta-build level. === Example: Format the `build.sc` From b99cb7ec7fce20f1cdaee941e66866bfe33989f7 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 18:10:41 +0200 Subject: [PATCH 12/19] . --- docs/modules/ROOT/pages/Extending_Mill.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/modules/ROOT/pages/Extending_Mill.adoc b/docs/modules/ROOT/pages/Extending_Mill.adoc index 8a7af8fe633..0294fc03001 100644 --- a/docs/modules/ROOT/pages/Extending_Mill.adoc +++ b/docs/modules/ROOT/pages/Extending_Mill.adoc @@ -45,6 +45,8 @@ Hence, you can easily search for updates with the external `mill.scalalib.Depend .Check for Mill Plugin updates ---- $ mill --meta-level 1 mill.scalalib.Dependency/showUpdates +Found 1 dependency update for + de.tototec:de.tobiasroeser.mill.vcs.version_mill0.11_2.13 : 0.3.1-> 0.4.0 ---- === Example: Customizing the Meta-Build From ff0f242a3aae2eb602c97ebd667f0fbf72c30fbb Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 22:39:40 +0200 Subject: [PATCH 13/19] ScalaDoc --- runner/src/mill/runner/MillBuild.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runner/src/mill/runner/MillBuild.scala b/runner/src/mill/runner/MillBuild.scala index ecd9d8c1549..7f2c5dba0e3 100644 --- a/runner/src/mill/runner/MillBuild.scala +++ b/runner/src/mill/runner/MillBuild.scala @@ -6,6 +6,9 @@ import mill.eval.Evaluator.AllBootstrapEvaluators trait MillBuild extends Module { + /** + * Count of the nested build-levels, the main project and all its nested meta-builds. + */ def levelCount(evaluators: AllBootstrapEvaluators): Command[Int] = T.command { val count = evaluators.value.size T.log.outputStream.println(s"${count}") From d90f8aa9cc9978f01356dde4cb3f56d021e73d7a Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 31 Aug 2023 23:35:00 +0200 Subject: [PATCH 14/19] More consistent naming: frame => meta-level --- runner/src/mill/runner/MillBuildBootstrap.scala | 2 +- runner/src/mill/runner/MillMain.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 6a6a026653a..4952c21b086 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -119,7 +119,7 @@ class MillBuildBootstrap( // User has requested a frame depth, we actually don't have nestedState.add(errorOpt = Some( - s"Invalid selected frame ${requestedDepth}. Valid range: 0 .. ${nestedState.frames.size}" + s"Invalid selected meta-level ${requestedDepth}. Valid range: 0 .. ${nestedState.frames.size}" ) ) } else if (depth < requestedDepth) { diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala index a808a06b79d..c7873bd1c09 100644 --- a/runner/src/mill/runner/MillMain.scala +++ b/runner/src/mill/runner/MillMain.scala @@ -126,9 +126,9 @@ object MillMain { ) (false, RunnerState.empty) - // Check non-negative --frame option + // Check non-negative --meta-level option case Right(config) if config.metaLevel.exists(_ < 0) => - streams.err.println("--frame cannot be negative") + streams.err.println("--meta-level cannot be negative") (false, RunnerState.empty) case Right(config) => From 075fab1bf71d2250b841f9d71fa92c404430fe92 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Fri, 1 Sep 2023 07:39:46 +0200 Subject: [PATCH 15/19] Added an assert --- runner/src/mill/runner/MillBuildBootstrap.scala | 2 ++ runner/src/mill/runner/RunnerState.scala | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 4952c21b086..89617f07b07 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -300,6 +300,8 @@ class MillBuildBootstrap( evaluator: Evaluator ): RunnerState = { + assert(nestedState.frames.forall(_.evaluator != null)) + val (evaled, evalWatched, moduleWatches) = Evaluator.allBootstrapEvaluators.withValue( Evaluator.AllBootstrapEvaluators(Seq(evaluator) ++ nestedState.frames.map(_.evaluator)) ) { diff --git a/runner/src/mill/runner/RunnerState.scala b/runner/src/mill/runner/RunnerState.scala index 8a91535a021..587d611c31e 100644 --- a/runner/src/mill/runner/RunnerState.scala +++ b/runner/src/mill/runner/RunnerState.scala @@ -65,7 +65,7 @@ object RunnerState { evaluator: Evaluator ) { - def loggedData = { + def loggedData: Frame.Logged = { Frame.Logged( workerCache.map { case (k, (i, v)) => (k.render, Frame.WorkerInfo(System.identityHashCode(v), i)) From df392a2648cd30bce4bff3aa5ae8b2f4a2ca2c37 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Fri, 1 Sep 2023 08:01:56 +0200 Subject: [PATCH 16/19] Hold the evaluator in an Option to make non-existence safe --- runner/src/mill/runner/MillBuildBootstrap.scala | 16 ++++++++-------- runner/src/mill/runner/RunnerState.scala | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 89617f07b07..316dda4acd5 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -132,10 +132,10 @@ class MillBuildBootstrap( Map.empty, None, Nil, - // FIXME we don't want to evaluator anything in this depth, so we just skip creating an evaluator, + // We set this to None here. + // We don't want to evaluate anything in this depth, so we just skip creating an evaluator, // mainly because we didn't even constructed (compiled) it's classpath - // TODO: Instead, introduce a skipped frame type, so we can better comunicate that state - null + None ) nestedState.add(frame = evalState, errorOpt = None) } else { @@ -232,7 +232,7 @@ class MillBuildBootstrap( Map.empty, None, Nil, - evaluator + Option(evaluator) ) nestedState.add(frame = evalState, errorOpt = Some(error)) @@ -282,7 +282,7 @@ class MillBuildBootstrap( methodCodeHashSignatures, Some(classLoader), runClasspath, - evaluator + Option(evaluator) ) nestedState.add(frame = evalState) @@ -300,10 +300,10 @@ class MillBuildBootstrap( evaluator: Evaluator ): RunnerState = { - assert(nestedState.frames.forall(_.evaluator != null)) + assert(nestedState.frames.forall(_.evaluator.isDefined)) val (evaled, evalWatched, moduleWatches) = Evaluator.allBootstrapEvaluators.withValue( - Evaluator.AllBootstrapEvaluators(Seq(evaluator) ++ nestedState.frames.map(_.evaluator)) + Evaluator.AllBootstrapEvaluators(Seq(evaluator) ++ nestedState.frames.flatMap(_.evaluator)) ) { evaluateWithWatches(rootModule, evaluator, targetsAndParams) } @@ -316,7 +316,7 @@ class MillBuildBootstrap( Map.empty, None, Nil, - evaluator + Option(evaluator) ) nestedState.add(frame = evalState, errorOpt = evaled.left.toOption) diff --git a/runner/src/mill/runner/RunnerState.scala b/runner/src/mill/runner/RunnerState.scala index 587d611c31e..1f76ab34647 100644 --- a/runner/src/mill/runner/RunnerState.scala +++ b/runner/src/mill/runner/RunnerState.scala @@ -62,7 +62,7 @@ object RunnerState { methodCodeHashSignatures: Map[String, Int], classLoaderOpt: Option[RunnerState.URLClassLoader], runClasspath: Seq[PathRef], - evaluator: Evaluator + evaluator: Option[Evaluator] ) { def loggedData: Frame.Logged = { From 8f6920b9eb2c633654f6b3e2c9c6eca6e0bf3fe1 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Fri, 1 Sep 2023 08:03:49 +0200 Subject: [PATCH 17/19] Refactored build.sc#validate to be no longer a evalutator target --- build.sc | 65 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/build.sc b/build.sc index 428a2df2a4d..ad89b1756e7 100644 --- a/build.sc +++ b/build.sc @@ -4,9 +4,10 @@ import $file.ci.shared import $file.ci.upload import $ivy.`org.scalaj::scalaj-http:2.4.2` import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0` - import $ivy.`com.github.lolgab::mill-mima::0.0.23` import $ivy.`net.sourceforge.htmlcleaner:htmlcleaner:2.25` +import mill.define.NamedTask +import mill.main.Tasks // imports import com.github.lolgab.mill.mima.{CheckDirection, ProblemFilter, Mima} @@ -369,7 +370,7 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { ProblemFilter.exclude[Problem]("mill.eval.ProfileLogger*"), ProblemFilter.exclude[Problem]("mill.eval.GroupEvaluator*"), ProblemFilter.exclude[Problem]("mill.eval.Tarjans*"), - ProblemFilter.exclude[Problem]("mill.define.Ctx#Impl*"), + ProblemFilter.exclude[Problem]("mill.define.Ctx#Impl*") ) def mimaPreviousVersions: T[Seq[String]] = Settings.mimaBaseVersions @@ -486,33 +487,34 @@ object main extends MillStableScalaModule with BuildInfo { } object codesig extends MillPublishScalaModule { - override def ivyDeps = Agg(ivy"org.ow2.asm:asm-tree:9.5", Deps.osLib, ivy"com.lihaoyi::pprint:0.8.1") + override def ivyDeps = + Agg(ivy"org.ow2.asm:asm-tree:9.5", Deps.osLib, ivy"com.lihaoyi::pprint:0.8.1") def moduleDeps = Seq(util) - override lazy val test: CodeSigTests = new CodeSigTests{} + override lazy val test: CodeSigTests = new CodeSigTests {} trait CodeSigTests extends MillScalaTests { val caseKeys = interp.watchValue( os.walk(millSourcePath / "cases", maxDepth = 3) .map(_.subRelativeTo(millSourcePath / "cases").segments) - .collect{case Seq(a, b, c) => s"$a-$b-$c"} + .collect { case Seq(a, b, c) => s"$a-$b-$c" } ) - def testLogFolder = T{ T.dest } + def testLogFolder = T { T.dest } def caseEnvs[V](f1: CaseModule => Task[V])(s: String, f2: V => String) = { T.traverse(caseKeys) { i => f1(cases(i)).map(v => s"MILL_TEST_${s}_$i" -> f2(v)) } } - def forkEnv = T{ + def forkEnv = T { Map("MILL_TEST_LOGS" -> testLogFolder().toString) ++ - caseEnvs(_.compile)("CLASSES", _.classes.path.toString)() ++ - caseEnvs(_.compileClasspath)("CLASSPATH", _.map(_.path).mkString(","))() ++ - caseEnvs(_.sources)("SOURCES", _.head.path.toString)() + caseEnvs(_.compile)("CLASSES", _.classes.path.toString)() ++ + caseEnvs(_.compileClasspath)("CLASSPATH", _.map(_.path).mkString(","))() ++ + caseEnvs(_.sources)("SOURCES", _.head.path.toString)() } object cases extends Cross[CaseModule](caseKeys) - trait CaseModule extends ScalaModule with Cross.Module[String]{ + trait CaseModule extends ScalaModule with Cross.Module[String] { def caseName = crossValue - object external extends ScalaModule{ + object external extends ScalaModule { def scalaVersion = "2.13.10" } @@ -521,7 +523,7 @@ object main extends MillStableScalaModule with BuildInfo { val Array(prefix, suffix, rest) = caseName.split("-", 3) def millSourcePath = super.millSourcePath / prefix / suffix / rest def scalaVersion = "2.13.10" - def ivyDeps = T{ + def ivyDeps = T { if (!caseName.contains("realistic") && !caseName.contains("sourcecode")) super.ivyDeps() else Agg( Deps.fastparse, @@ -531,7 +533,7 @@ object main extends MillStableScalaModule with BuildInfo { Deps.mainargs, Deps.requests, Deps.osLib, - Deps.upickle, + Deps.upickle ) } } @@ -1454,7 +1456,7 @@ object docs extends Module { val pagesWd = T.dest / "modules" / "ROOT" / "pages" val partialsWd = T.dest / "modules" / "ROOT" / "partials" - os.copy(projectReadme().path, partialsWd / "project-readme.adoc", createFolders = true) + os.copy(projectReadme().path, partialsWd / "project-readme.adoc", createFolders = true) val renderedExamples: Seq[(os.SubPath, PathRef)] = T.traverse(example.exampleModules)(m => @@ -1763,23 +1765,24 @@ def uploadToGithub(authKey: String) = T.command { } } -def validate(ev: Evaluator): Command[Unit] = T.command { - mill.main.RunScript.evaluateTasksNamed( - ev.withFailFast(false), - Seq( - "__.compile", - "+", - "__.mimaReportBinaryIssues", - "+", - "mill.scalalib.scalafmt.ScalafmtModule/checkFormatAll", - "__.sources", - "+", - "docs.localPages" - ), - selectMode = SelectMode.Separated - ) +private def resolveTasks[T](taskNames: String*): Seq[NamedTask[T]] = { + mill.resolve.Resolve.Tasks.resolve( + build, + taskNames, + SelectMode.Separated + ).map(x => x.asInstanceOf[Seq[mill.define.NamedTask[T]]]).getOrElse(???) +} + +def validate(): Command[Unit] = { + val tasks = resolveTasks("__.compile", "__.minaReportBinaryIssues") + val sources = resolveTasks("__.sources") - () + T.command { + T.sequence(tasks)() + mill.scalalib.scalafmt.ScalafmtModule.checkFormatAll(Tasks(sources))() + docs.localPages() + () + } } /** Dummy module to let Scala-Steward find and bump dependency versions we use at runtime */ From 9e17b9894eea010a9fc932d83ca40b7d6fe9ff53 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Fri, 1 Sep 2023 14:49:28 +0200 Subject: [PATCH 18/19] Cleanup --- runner/src/mill/runner/MillBuildBootstrap.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 316dda4acd5..d638d155bbc 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -1,6 +1,5 @@ package mill.runner -import mill.util.{ColorLogger, PrefixLogger, Util, Watchable} -import mill.T +import mill.util.{ColorLogger, PrefixLogger, Watchable} import mill.main.BuildInfo import mill.api.{PathRef, Val, internal} import mill.eval.Evaluator @@ -123,7 +122,8 @@ class MillBuildBootstrap( ) ) } else if (depth < requestedDepth) { - // We already evaluated, hence we just need to make sure, we return a proper structure with all already existing watch data + // We already evaluated on a deeper level, hence we just need to make sure, + // we return a proper structure with all already existing watch data val evalState = RunnerState.Frame( prevFrameOpt.map(_.workerCache).getOrElse(Map.empty), Seq.empty, @@ -132,8 +132,7 @@ class MillBuildBootstrap( Map.empty, None, Nil, - // We set this to None here. - // We don't want to evaluate anything in this depth, so we just skip creating an evaluator, + // We don't want to evaluate anything in this depth (and above), so we just skip creating an evaluator, // mainly because we didn't even constructed (compiled) it's classpath None ) From 95bc8117930c7fd7d47ae0a6a6d8d6d28aef958e Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Fri, 1 Sep 2023 16:49:51 +0200 Subject: [PATCH 19/19] Don't print the count, we already have the show command --- runner/src/mill/runner/MillBuild.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/runner/src/mill/runner/MillBuild.scala b/runner/src/mill/runner/MillBuild.scala index 7f2c5dba0e3..bf3a94606d8 100644 --- a/runner/src/mill/runner/MillBuild.scala +++ b/runner/src/mill/runner/MillBuild.scala @@ -8,11 +8,10 @@ trait MillBuild extends Module { /** * Count of the nested build-levels, the main project and all its nested meta-builds. + * If you run this on a meta-build, the non-meta-builds are not included. */ def levelCount(evaluators: AllBootstrapEvaluators): Command[Int] = T.command { - val count = evaluators.value.size - T.log.outputStream.println(s"${count}") - count + evaluators.value.size } }