diff --git a/clientserver/src/mill/clientserver/Client.java b/clientserver/src/mill/clientserver/Client.java index c4cbc265f51..ccabc24dced 100644 --- a/clientserver/src/mill/clientserver/Client.java +++ b/clientserver/src/mill/clientserver/Client.java @@ -7,10 +7,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.channels.FileChannel; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Properties; +import java.util.*; public class Client { static void initServer(String lockBase, boolean setJnaNoSys) throws IOException,URISyntaxException{ @@ -51,6 +48,7 @@ static void initServer(String lockBase, boolean setJnaNoSys) throws IOException, } public static void main(String[] args) throws Exception{ boolean setJnaNoSys = System.getProperty("jna.nosys") == null; + Map env = System.getenv(); if (setJnaNoSys) { System.setProperty("jna.nosys", "true"); } @@ -82,7 +80,8 @@ public void run() { System.in, System.out, System.err, - args + args, + env ); System.exit(exitCode); } @@ -97,10 +96,12 @@ public static int run(String lockBase, InputStream stdin, OutputStream stdout, OutputStream stderr, - String[] args) throws Exception{ + String[] args, + Map env) throws Exception{ FileOutputStream f = new FileOutputStream(lockBase + "/run"); ClientServer.writeArgs(System.console() != null, args, f); + ClientServer.writeMap(env, f); f.close(); boolean serverInit = false; diff --git a/clientserver/src/mill/clientserver/ClientServer.java b/clientserver/src/mill/clientserver/ClientServer.java index e2e63dcffc0..15c20f41d11 100644 --- a/clientserver/src/mill/clientserver/ClientServer.java +++ b/clientserver/src/mill/clientserver/ClientServer.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; public class ClientServer { public static boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows"); @@ -20,23 +22,76 @@ public static String[] parseArgs(InputStream argStream) throws IOException { int argsLength = argStream.read(); String[] args = new String[argsLength]; for (int i = 0; i < args.length; i++) { - int n = argStream.read(); - byte[] arr = new byte[n]; - argStream.read(arr); - args[i] = new String(arr); + args[i] = readString(argStream); } return args; } public static void writeArgs(Boolean interactive, String[] args, - OutputStream argStream) throws IOException{ - argStream.write(interactive ? 1 : 0); - argStream.write(args.length); - int i = 0; - while (i < args.length){ - argStream.write(args[i].length()); - argStream.write(args[i].getBytes()); - i += 1; + OutputStream argStream) throws IOException { + argStream.write(interactive ? 1 : 0); + argStream.write(args.length); + int i = 0; + while (i < args.length) { + writeString(argStream, args[i]); + i += 1; + } + } + + /** + * This allows the mill client to pass the environment as he sees it to the + * server (as the server remains alive over the course of several runs and + * does not see the environment changes the client would) + */ + public static void writeMap(Map map, OutputStream argStream) throws IOException { + argStream.write(map.size()); + for (Map.Entry kv : map.entrySet()) { + writeString(argStream, kv.getKey()); + writeString(argStream, kv.getValue()); + } } - } -} \ No newline at end of file + + public static Map parseMap(InputStream argStream) throws IOException { + Map env = new HashMap<>(); + int mapLength = argStream.read(); + for (int i = 0; i < mapLength; i++) { + String key = readString(argStream); + String value = readString(argStream); + env.put(key, value); + } + return env; + } + + private static String readString(InputStream inputStream) throws IOException { + // Result is between 0 and 255, hence the loop. + int read = inputStream.read(); + int bytesToRead = read; + while(read == 255){ + read = inputStream.read(); + bytesToRead += read; + } + byte[] arr = new byte[bytesToRead]; + int readTotal = 0; + while (readTotal < bytesToRead) { + read = inputStream.read(arr, readTotal, bytesToRead - readTotal); + readTotal += read; + } + return new String(arr); + } + + private static void writeString(OutputStream outputStream, String string) throws IOException { + // When written, an int > 255 gets splitted. This logic performs the + // split beforehand so that the reading side knows that there is still + // more metadata to come before it's able to read the actual data. + // Could do with rewriting using logical masks / shifts. + byte[] bytes = string.getBytes(); + int toWrite = bytes.length; + while(toWrite >= 255){ + outputStream.write(255); + toWrite = toWrite - 255; + } + outputStream.write(toWrite); + outputStream.write(bytes); + } + +} diff --git a/clientserver/src/mill/clientserver/Server.scala b/clientserver/src/mill/clientserver/Server.scala index 24827ac24d7..4deac55f4c8 100644 --- a/clientserver/src/mill/clientserver/Server.scala +++ b/clientserver/src/mill/clientserver/Server.scala @@ -3,6 +3,7 @@ package mill.clientserver import java.io._ import java.net.Socket +import scala.collection.JavaConverters._ import org.scalasbt.ipcsocket._ trait ServerMain[T]{ @@ -21,7 +22,8 @@ trait ServerMain[T]{ mainInteractive: Boolean, stdin: InputStream, stdout: PrintStream, - stderr: PrintStream): (Boolean, Option[T]) + stderr: PrintStream, + env : Map[String, String]): (Boolean, Option[T]) } @@ -76,6 +78,7 @@ class Server[T](lockBase: String, val argStream = new FileInputStream(lockBase + "/run") val interactive = argStream.read() != 0; val args = ClientServer.parseArgs(argStream) + val env = ClientServer.parseMap(argStream) argStream.close() var done = false @@ -89,7 +92,9 @@ class Server[T](lockBase: String, sm.stateCache, interactive, socketIn, - stdout, stderr + stdout, + stderr, + env.asScala.toMap ) sm.stateCache = newStateCache diff --git a/clientserver/test/src/mill/clientserver/ClientServerTests.scala b/clientserver/test/src/mill/clientserver/ClientServerTests.scala index 2c9a57b0a44..ac2063ef664 100644 --- a/clientserver/test/src/mill/clientserver/ClientServerTests.scala +++ b/clientserver/test/src/mill/clientserver/ClientServerTests.scala @@ -2,6 +2,7 @@ package mill.clientserver import java.io._ import java.nio.file.Path +import scala.collection.JavaConverters._ import utest._ class EchoServer extends ServerMain[Int]{ def main0(args: Array[String], @@ -9,13 +10,21 @@ class EchoServer extends ServerMain[Int]{ mainInteractive: Boolean, stdin: InputStream, stdout: PrintStream, - stderr: PrintStream) = { + stderr: PrintStream, + env: Map[String, String]) = { val reader = new BufferedReader(new InputStreamReader(stdin)) val str = reader.readLine() - stdout.println(str + args(0)) + if (args.nonEmpty){ + stdout.println(str + args(0)) + } + env.toSeq.sortBy(_._1).foreach{ + case (key, value) => stdout.println(s"$key=$value") + } stdout.flush() - stderr.println(str.toUpperCase + args(0)) + if (args.nonEmpty){ + stderr.println(str.toUpperCase + args(0)) + } stderr.flush() (true, None) } @@ -35,37 +44,39 @@ object ClientServerTests extends TestSuite{ (tmpDir, locks) } + def spawnEchoServer(tmpDir : Path, locks: Locks): Unit = { + new Thread(() => new Server( + tmpDir.toString, + new EchoServer(), + () => (), + 1000, + locks + ).run()).start() + } + + def runClientAux(tmpDir : Path, locks: Locks) + (env : Map[String, String], args: Array[String]) = { + val (in, out, err) = initStreams() + Server.lockBlock(locks.clientLock){ + Client.run( + tmpDir.toString, + () => spawnEchoServer(tmpDir, locks), + locks, + in, + out, + err, + args, + env.asJava + ) + Thread.sleep(100) + (new String(out.toByteArray), new String(err.toByteArray)) + } + } + def tests = Tests{ 'hello - { val (tmpDir, locks) = init() - - def spawnEchoServer(): Unit = { - new Thread(() => new Server( - tmpDir.toString, - new EchoServer(), - () => (), - 1000, - locks - ).run()).start() - } - - - def runClient(arg: String) = { - val (in, out, err) = initStreams() - Server.lockBlock(locks.clientLock){ - Client.run( - tmpDir.toString, - () => spawnEchoServer(), - locks, - in, - out, - err, - Array(arg) - ) - Thread.sleep(100) - (new String(out.toByteArray), new String(err.toByteArray)) - } - } + def runClient(s: String) = runClientAux(tmpDir, locks)(Map.empty, Array(s)) // Make sure the simple "have the client start a server and // exchange one message" workflow works from end to end. @@ -116,5 +127,83 @@ object ClientServerTests extends TestSuite{ err3 == "HELLO World\n" ) } + + 'envVars - { + val (tmpDir, locks) = init() + + def runClient(env : Map[String, String]) = runClientAux(tmpDir, locks)(env, Array()) + + // Make sure the simple "have the client start a server and + // exchange one message" workflow works from end to end. + + assert( + locks.clientLock.probe(), + locks.serverLock.probe(), + locks.processLock.probe() + ) + + def longString(s : String) = Array.fill(1000)(s).mkString + val b1000 = longString("b") + val c1000 = longString("c") + val a1000 = longString("a") + + val env = Map( + "a" -> a1000, + "b" -> b1000, + "c" -> c1000 + ) + + + val (out1, err1) = runClient(env) + val expected = s"a=$a1000\nb=$b1000\nc=$c1000\n" + + assert( + out1 == expected, + err1 == "" + ) + + // Give a bit of time for the server to release the lock and + // re-acquire it to signal to the client that it's done + Thread.sleep(100) + + assert( + locks.clientLock.probe(), + !locks.serverLock.probe(), + !locks.processLock.probe() + ) + + val path = List( + "/Users/foo/Library/Haskell/bin", + "/usr/local/git/bin", + "/sw/bin/", + "/usr/local/bin", + "/usr/local/", + "/usr/local/sbin", + "/usr/local/mysql/bin", + "/usr/local/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/opt/X11/bin", + "/usr/local/MacGPG2/bin", + "/Library/TeX/texbin", + "/usr/local/bin/", + "/Users/foo/bin", + "/Users/foo/go/bin", + "~/.bloop" + ) + + val pathEnvVar = path.mkString(":") + val (out2, err2) = runClient(Map("PATH" -> pathEnvVar)) + + val expected2 = s"PATH=$pathEnvVar\n" + + assert( + out2 == expected2, + err2 == "" + ) + + } } } diff --git a/core/src/mill/eval/Evaluator.scala b/core/src/mill/eval/Evaluator.scala index 33141c0a24f..7b3634ad4a0 100644 --- a/core/src/mill/eval/Evaluator.scala +++ b/core/src/mill/eval/Evaluator.scala @@ -2,6 +2,8 @@ package mill.eval import java.net.URLClassLoader +import scala.collection.JavaConverters._ + import mill.util.Router.EntryPoint import ammonite.ops._ import ammonite.runtime.SpecialClassLoader @@ -32,7 +34,8 @@ case class Evaluator[T](home: Path, rootModule: mill.define.BaseModule, log: Logger, classLoaderSig: Seq[(Either[String, Path], Long)] = Evaluator.classLoaderSig, - workerCache: mutable.Map[Segments, (Int, Any)] = mutable.Map.empty){ + workerCache: mutable.Map[Segments, (Int, Any)] = mutable.Map.empty, + env : Map[String, String] = Evaluator.defaultEnv){ val classLoaderSignHash = classLoaderSig.hashCode() def evaluate(goals: Agg[Task[_]]): Evaluator.Results = { mkdir(outPath) @@ -271,7 +274,8 @@ case class Evaluator[T](home: Path, } }, multiLogger, - home + home, + env ) val out = System.out @@ -335,6 +339,8 @@ object Evaluator{ // in directly) we are forced to pass it in via a ThreadLocal val currentEvaluator = new ThreadLocal[mill.eval.Evaluator[_]] + val defaultEnv: Map[String, String] = System.getenv().asScala.toMap + case class Paths(out: Path, dest: Path, meta: Path, diff --git a/core/src/mill/util/Ctx.scala b/core/src/mill/util/Ctx.scala index 998181941d4..6c8b2afb692 100644 --- a/core/src/mill/util/Ctx.scala +++ b/core/src/mill/util/Ctx.scala @@ -23,6 +23,9 @@ object Ctx{ trait Home{ def home: Path } + trait Env{ + def env: Map[String, String] + } object Log{ implicit def logToCtx(l: Logger): Log = new Log { def log = l } } @@ -36,11 +39,13 @@ object Ctx{ class Ctx(val args: IndexedSeq[_], dest0: () => Path, val log: Logger, - val home: Path) + val home: Path, + val env : Map[String, String]) extends Ctx.Dest with Ctx.Log with Ctx.Args - with Ctx.Home{ + with Ctx.Home + with Ctx.Env { def dest = dest0() def length = args.length diff --git a/docs/pages/3 - Tasks.md b/docs/pages/3 - Tasks.md index ca6def42415..7197417777f 100644 --- a/docs/pages/3 - Tasks.md +++ b/docs/pages/3 - Tasks.md @@ -165,10 +165,10 @@ There are several APIs available to you within the body of a `T{...}` or `T.command{...}` block to help your write the code implementing your Target or Command: -### mill.util.Ctx.DestCtx +### mill.util.Ctx.Dest - `T.ctx().dest` -- `implicitly[mill.util.Ctx.DestCtx]` +- `implicitly[mill.util.Ctx.Dest]` This is the unique `out/classFiles/dest/` path or `out/run/dest/` path that is assigned to every Target or Command. It is cleared before your task runs, and @@ -177,10 +177,10 @@ artifacts. This is guaranteed to be unique for every `Target` or `Command`, so you can be sure that you will not collide or interfere with anyone else writing to those same paths. -### mill.util.Ctx.LogCtx +### mill.util.Ctx.Log - `T.ctx().log` -- `implicitly[mill.util.Ctx.LogCtx]` +- `implicitly[mill.util.Ctx.Log]` This is the default logger provided for every task. While your task is running, `System.out` and `System.in` are also redirected to this logger. The logs for a @@ -188,6 +188,25 @@ task are streamed to standard out/error as you would expect, but each task's specific output is also streamed to a log file on disk e.g. `out/run/log` or `out/classFiles/log` for you to inspect later. +### mill.util.Ctx.Env + +- `T.ctx().env` +- `implicitly[mill.util.Ctx.Env]` + +Mill keeps a long-lived JVM server to avoid paying the cost of recurrent +classloading. Because of this, running `System.getenv` in a task might not yield +up to date environment variables, since it will be initialised when the server +starts, rather than when the client executes. To circumvent this, mill's client +sends the environment variables to the server as it sees them, and the server +makes them available as a `Map[String, String]` via the `Ctx` API. + +If the intent is to always pull the latest environment values, the call should +be wrapped in an `Input` as such : + +```scala +def envVar = T.input { T.ctx().env.get("ENV_VAR") } +``` + ## Other Tasks - [Anonymous Tasks](#anonymous-tasks), defined using `T.task{...}` diff --git a/main/src/mill/Main.scala b/main/src/mill/Main.scala index e026dfe0ef9..c9ec00ca9bc 100644 --- a/main/src/mill/Main.scala +++ b/main/src/mill/Main.scala @@ -2,6 +2,8 @@ package mill import java.io.{InputStream, PrintStream} +import scala.collection.JavaConverters._ + import ammonite.main.Cli._ import ammonite.ops._ import ammonite.util.Util @@ -16,13 +18,15 @@ object ServerMain extends mill.clientserver.ServerMain[Evaluator.State]{ mainInteractive: Boolean, stdin: InputStream, stdout: PrintStream, - stderr: PrintStream) = Main.main0( + stderr: PrintStream, + env : Map[String, String]) = Main.main0( args, stateCache, mainInteractive, DummyInputStream, stdout, - stderr + stderr, + env ) } object Main { @@ -38,7 +42,8 @@ object Main { ammonite.Main.isInteractive(), System.in, System.out, - System.err + System.err, + System.getenv().asScala.toMap ) System.exit(if(result) 0 else 1) } @@ -48,7 +53,8 @@ object Main { mainInteractive: Boolean, stdin: InputStream, stdout: PrintStream, - stderr: PrintStream): (Boolean, Option[Evaluator.State]) = { + stderr: PrintStream, + env: Map[String, String]): (Boolean, Option[Evaluator.State]) = { import ammonite.main.Cli val removed = Set("predef-code", "no-home-predef") @@ -116,7 +122,8 @@ object Main { val runner = new mill.main.MainRunner( config.copy(colored = Some(mainInteractive)), stdout, stderr, stdin, - stateCache + stateCache, + env ) if (mill.clientserver.ClientServer.isJava9OrAbove) { diff --git a/main/src/mill/main/MainRunner.scala b/main/src/mill/main/MainRunner.scala index efebd5a5067..fed664fd139 100644 --- a/main/src/mill/main/MainRunner.scala +++ b/main/src/mill/main/MainRunner.scala @@ -20,7 +20,8 @@ class MainRunner(val config: ammonite.main.Cli.Config, outprintStream: PrintStream, errPrintStream: PrintStream, stdIn: InputStream, - stateCache0: Option[Evaluator.State] = None) + stateCache0: Option[Evaluator.State] = None, + env : Map[String, String]) extends ammonite.MainRunner( config, outprintStream, errPrintStream, stdIn, outprintStream, errPrintStream @@ -75,7 +76,8 @@ class MainRunner(val config: ammonite.main.Cli.Config, errPrintStream, errPrintStream, stdIn - ) + ), + env ) result match{ diff --git a/main/src/mill/main/RunScript.scala b/main/src/mill/main/RunScript.scala index 77930cc8ab1..75042dea0f3 100644 --- a/main/src/mill/main/RunScript.scala +++ b/main/src/mill/main/RunScript.scala @@ -29,7 +29,8 @@ object RunScript{ instantiateInterpreter: => Either[(Res.Failing, Seq[(Path, Long)]), ammonite.interp.Interpreter], scriptArgs: Seq[String], stateCache: Option[Evaluator.State], - log: Logger) + log: Logger, + env : Map[String, String]) : (Res[(Evaluator[Any], Seq[PathRef], Either[String, Seq[Js.Value]])], Seq[(Path, Long)]) = { val (evalState, interpWatched) = stateCache match{ @@ -53,7 +54,8 @@ object RunScript{ val evalRes = for(s <- evalState) - yield new Evaluator[Any](home, wd / 'out, wd / 'out, s.rootModule, log, s.classLoaderSig, s.workerCache) + yield new Evaluator[Any](home, wd / 'out, wd / 'out, s.rootModule, log, + s.classLoaderSig, s.workerCache, env) val evaluated = for{ evaluator <- evalRes diff --git a/main/test/src/mill/util/ScriptTestSuite.scala b/main/test/src/mill/util/ScriptTestSuite.scala index a2f2676a20e..f88007c5060 100644 --- a/main/test/src/mill/util/ScriptTestSuite.scala +++ b/main/test/src/mill/util/ScriptTestSuite.scala @@ -15,7 +15,7 @@ abstract class ScriptTestSuite(fork: Boolean) extends TestSuite{ val stdIn = new ByteArrayInputStream(Array()) lazy val runner = new mill.main.MainRunner( ammonite.main.Cli.Config(wd = workspacePath), - stdOutErr, stdOutErr, stdIn + stdOutErr, stdOutErr, stdIn, None, Map.empty ) def eval(s: String*) = { if (!fork) runner.runScript(workspacePath / "build.sc", s.toList)