diff --git a/scala/BUILD b/scala/BUILD index e69de29bb..db0424396 100644 --- a/scala/BUILD +++ b/scala/BUILD @@ -0,0 +1,41 @@ +load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library") + +java_binary( + name = "scala-worker", + main_class = "com.databricks.bazel.ScalaWorker", + visibility = ["//visibility:public"], + runtime_deps = [ + ":scala-worker-lib-import", + "@bazel_tools//src/main/protobuf:worker_protocol_proto", + "@zinc_0_3_10_SNAPSHOT_jar//jar", + "@scala_compiler_jar//jar", + "@incremental_compiler_0_13_9_jar//jar", + "@scala_library_jar//jar", + "@scala_reflect_jar//jar", + "@sbt_interface_0_13_9_jar//jar", + "@compiler_interface_0_13_9_sources_jar//jar", + "@nailgun_server_0_9_1_jar//jar", + ], +) + +java_import( + name = "scala-worker-lib-import", + jars = ["scala-worker-lib_deploy.jar"], +) + +scala_library( + name = "scala-worker-lib", + srcs = glob(["ScalaWorker.scala"]), + deps = [ + "@bazel_tools//src/main/protobuf:worker_protocol_proto", + "@zinc_0_3_10_SNAPSHOT_jar//jar", + "@scala_compiler_jar//jar", + "@incremental_compiler_0_13_9_jar//jar", + "@scala_library_jar//jar", + "@scala_reflect_jar//jar", + "@sbt_interface_0_13_9_jar//jar", + "@compiler_interface_0_13_9_sources_jar//jar", + "@nailgun_server_0_9_1_jar//jar", + ], +) + diff --git a/scala/ScalaWorker.scala b/scala/ScalaWorker.scala new file mode 100644 index 000000000..069768b36 --- /dev/null +++ b/scala/ScalaWorker.scala @@ -0,0 +1,220 @@ +package com.databricks.bazel + +import java.nio.charset.StandardCharsets.UTF_8 + +import com.google.devtools.build.lib.worker.WorkerProtocol.Input +import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest +import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse + +import java.io.ByteArrayOutputStream +import java.io.File +import java.io.IOException +import java.io.PrintStream +import java.nio.file.Files +import java.nio.file.Paths +import java.net.ServerSocket +import java.util.ArrayList +import java.util.LinkedHashMap +import java.util.{List => JList} +import java.util.Map.Entry +import java.util.UUID + +import scala.collection.JavaConverters._ +import scala.sys.process._ + +import com.typesafe.zinc.{Main => ZincMain, Nailgun, ZincClient} + + +/** + * An example implementation of a worker process that is used for integration tests. + */ +object ScalaWorker { + + // A UUID that uniquely identifies this running worker process. + private val workerUuid = UUID.randomUUID() + + // A counter that increases with each work unit processed. + private var workUnitCounter = 1 + + // If true, returns corrupt responses instead of correct protobufs. + private var poisoned = false + + // Keep state across multiple builds. + private val inputs = new LinkedHashMap[String, String]() + + private var serverArgs = "" + + private def getFreePort(): Int = { + val sock = new ServerSocket(0) + val port = sock.getLocalPort + sock.close() + port + } + + private var zincClient: ZincClient = _ + + private var zincPort: Int = 0 + + private var nailgunProcess: Process = _ + + private def attachShutdownHook() { + Runtime.getRuntime().addShutdownHook(new Thread() { + override def run() { + if (nailgunProcess != null) { + nailgunProcess.destroy() + } + } + }) + } + + private val serverOutput = new StringBuilder() + + private def startServer(classpath: String): Unit = { + attachShutdownHook() + zincPort = getFreePort() + + val logger = new ProcessLogger { + def buffer[T](fn: => T): T = fn + def err(s: => String): Unit = serverOutput.append(s).append("\n") + def out(s: => String): Unit = serverOutput.append(s).append("\n") + } + + // Options copied from Nailgun.scala in Zinc + val options = List("-cp", classpath, "-server", "-Xms1024m", "-Xmx3g", "-XX:MaxPermSize=384m", + "-XX:ReservedCodeCacheSize=192m") + val cmd = "java" :: options ++ Seq(classOf[Nailgun].getName, s"$zincPort") + val builder = Process(cmd) + this.nailgunProcess = builder.run(logger) + + serverArgs = cmd.mkString(" ") + zincClient = new ZincClient(port = zincPort) + } + + private def awaitServer() { + var count = 0 + while (!zincClient.serverAvailable && (count < 50)) { + try { Thread.sleep(100) } catch { case _: InterruptedException => } + count += 1 + } + } + + def main(args: Array[String]): Unit = { + if (args.contains("--persistent_worker")) { + startServer(args(0)) + runPersistentWorker(args) + } else { + // This is a single invocation of the example that exits after it processed the request. + ZincMain.run(args, cwd = None) + } + } + + private def listFiles(f: File): Seq[String] = { + val current = f.listFiles + val files = current.filter(_.isFile).map(_.getAbsolutePath) + val directories = current.filter(_.isDirectory) + files ++ directories.flatMap(listFiles) + } + + // Extract a src jar to a temporary directory and return the list of extracted files + private def expandSrcJar(path: String): Seq[String] = { + val tempDir = Files.createTempDirectory(null).toFile + Seq("unzip", "-q", path, "-d", tempDir.getAbsolutePath).!! + listFiles(tempDir) + } + + @throws[IOException] + private def runPersistentWorker(args: Array[String]) { + val originalStdOut = System.out + val originalStdErr = System.err + + while (true) { + try { + val request = WorkRequest.parseDelimitedFrom(System.in) + if (request == null) { + return + } + + inputs.clear() + + for (input <- request.getInputsList().asScala) { + inputs.put(input.getPath(), input.getDigest().toStringUtf8()) + } + + val baos = new ByteArrayOutputStream() + var exitCode = 0 + + val ps = new PrintStream(baos) + try { + System.setOut(ps) + System.setErr(ps) + + var clientArgs: Seq[String] = null + + try { + clientArgs = request.getArgumentsList.asScala.flatMap { arg => + // srcjars must be extracted before we can pass them to zinc + if (arg.endsWith(".srcjar")) { + expandSrcJar(arg) + } else { + Seq(arg) + } + } + awaitServer() + exitCode = zincClient.run( + args = clientArgs, + cwd = new File(System.getProperty("user.dir")), + out = ps, + err = ps + ) + } catch { + case e: Exception => + // We use System.out.println as not to accidentally write to real stdout + System.out.println("Startup Args:") + args.foreach(arg => System.out.println("Arg: " + arg)) + System.out.println("Server args: " + serverArgs) + System.out.println("Server output: " + serverOutput.toString) + System.out.println("Unexpanded Client Args:") + request.getArgumentsList.asScala.foreach(arg => System.out.println("Arg: " + arg)) + if (clientArgs != null) { + System.out.println("Expanded Client Args:") + clientArgs.foreach(arg => System.out.println("Arg: " + arg)) + } else { + System.out.println("======== CLIENT ARG EXPANSION MAY HAVE FAILED =======") + } + + e.printStackTrace() + exitCode = 1 + } + } finally { + System.setOut(originalStdOut) + System.setErr(originalStdErr) + } + + if (poisoned) { + System.out.println("I'm a poisoned worker and this is not a protobuf.") + } else { + WorkResponse.newBuilder() + .setOutput(baos.toString()) + .setExitCode(exitCode) + .build() + .writeDelimitedTo(System.out) + } + System.out.flush() + + /* + if (workerOptions.exitAfter > 0 && workUnitCounter > workerOptions.exitAfter) { + return + } + + if (workerOptions.poisonAfter > 0 && workUnitCounter > workerOptions.poisonAfter) { + poisoned = true + } + */ + } finally { + // Be a good worker process and consume less memory when idle. + System.gc() + } + } + } +} + diff --git a/scala/scala.bzl b/scala/scala.bzl index 34f1d7714..2869d74da 100644 --- a/scala/scala.bzl +++ b/scala/scala.bzl @@ -92,7 +92,73 @@ touch -t 198001010000 {manifest} arguments=[f.path for f in ctx.files.srcs]) def _compile_zinc(ctx, jars): - return None + worker = ctx.executable.worker + + tmp_out_dir = ctx.new_file(ctx.outputs.jar.path + "_tmp") + + flags = [ + "-fork-java", + "-scala-compiler", ctx.file._scala_compiler_jar.path, + "-scala-library", ctx.file._scala_library_jar.path, + "-scala-extra", ctx.file._scala_reflect_jar.path, + "-sbt-interface", ctx.file._sbt_interface_jar.path, + "-compiler-interface", ctx.file._compiler_interface_jar.path, + "-cp {jars}", + "-d", tmp_out_dir.path, + ] + flags = " ".join(flags) + flags = flags.format( + out=ctx.outputs.jar.path, + jars=":".join([j.path for j in jars])) + work_unit_args = ctx.new_file(ctx.configuration.bin_dir, ctx.label.name + "_args") + ctx.file_action(output = work_unit_args, content=flags) + + # Generate the "@"-file containing the command-line args for the unit of work. + argfile = ctx.new_file(ctx.configuration.bin_dir, "worker_input") + argfile_contents = "\n".join(["-argfile", work_unit_args.path] + [f.path for f in ctx.files.srcs]) + ctx.file_action(output=argfile, content=argfile_contents) + + # Classpath for the compiler/worker itself, these are not the compile time dependencies. + classpath_jars = [ + ctx.file._compiler_interface_jar, + ctx.file._incremental_compiler_jar, + ctx.file._scala_compiler_jar, + ctx.file._scala_library_jar, + ctx.file._scala_reflect_jar, + ctx.file._sbt_interface_jar, + ctx.file._zinc, + ctx.file._zinc_compiler_jar, + ctx.file._nailgun_server_jar, + ] + compiler_classpath = ":".join([f.path for f in classpath_jars]) + + ctx.action( + inputs=list(jars) + ctx.files.srcs + [ctx.outputs.manifest, argfile, work_unit_args] + classpath_jars, + outputs=[tmp_out_dir], + executable=worker, + progress_message="Zinc Worker: %s" % ctx.label.name, + mnemonic="Scala", + arguments=ctx.attr.worker_args + [compiler_classpath] + ["@" + argfile.path], + ) + + cmd = """ +set -e +# Make jar file deterministic by setting the timestamp of files +find {tmp_out} | xargs touch -t 198001010000 +touch -t 198001010000 {manifest} +jar cmf {manifest} {out} -C {tmp_out} . +""" + _get_res_cmd(ctx) + cmd = cmd.format( + tmp_out=tmp_out_dir.path, + out=ctx.outputs.jar.path, + manifest=ctx.outputs.manifest.path) + + ctx.action( + inputs=[tmp_out_dir, ctx.outputs.manifest], + outputs=[ctx.outputs.jar], + command=cmd, + progress_message="Building Jar: %s" % ctx.label) + def _get_res_cmd(ctx): res_cmd = "" @@ -242,6 +308,9 @@ def _lib(ctx, non_macro_lib, usezinc): def _scala_library_impl(ctx): return _lib(ctx, True, usezinc = False) +def _scala_worker_impl(ctx): + return _lib(ctx, True, usezinc = True) + def _scala_macro_library_impl(ctx): return _lib(ctx, False, usezinc = False) # don't build the ijar for macros @@ -279,7 +348,8 @@ _implicit_deps = { "_ijar": attr.label(executable=True, default=Label("//tools/defaults:ijar"), single_file=True, allow_files=True), "_scalac": attr.label(executable=True, default=Label("@scala//:bin/scalac"), single_file=True, allow_files=True), "_scalalib": attr.label(default=Label("@scala//:lib/scala-library.jar"), single_file=True, allow_files=True), - "_scalaxml": attr.label(default=Label("@scala//:lib/scala-xml_2.11-1.0.4.jar"), single_file=True, allow_files=True), + # "_scalaxml": attr.label(default=Label("@scala//:lib/scala-xml_2.11-1.0.4.jar"), single_file=True, allow_files=True), + "_scalaxml": attr.label(default=Label("@scala//:lib/scala-library.jar"), single_file=True, allow_files=True), "_scalasdk": attr.label(default=Label("@scala//:sdk"), allow_files=True), "_scalareflect": attr.label(default=Label("@scala//:lib/scala-reflect.jar"), single_file=True, allow_files=True), "_jar": attr.label(executable=True, default=Label("@bazel_tools//tools/jdk:jar"), single_file=True, allow_files=True), @@ -298,6 +368,67 @@ _common_attrs = { "jvm_flags": attr.string_list(), } +_zinc_compile_attrs = { + "_zinc": attr.label( + default=Label("@zinc//file"), + executable=True, + single_file=True, + allow_files=True), + "_zinc_compiler_jar": attr.label( + default=Label("@zinc_0_3_10_SNAPSHOT_jar//jar"), + single_file=True, + allow_files=True), + "_scala_compiler_jar": attr.label( + default=Label("@scala_compiler_jar//jar"), + single_file=True, + allow_files=True), + "_incremental_compiler_jar": attr.label( + default=Label("@incremental_compiler_0_13_9_jar//jar"), + single_file=True, + allow_files=True), + "_scala_library_jar": attr.label( + default=Label("@scala_library_jar//jar"), + single_file=True, + allow_files=True), + "_scala_reflect_jar": attr.label( + default=Label("@scala_reflect_jar//jar"), + single_file=True, + allow_files=True), + "_sbt_interface_jar": attr.label( + default=Label("@sbt_interface_0_13_9_jar//jar"), + single_file=True, + allow_files=True), + "_compiler_interface_jar": attr.label( + default=Label("@compiler_interface_0_13_9_sources_jar//jar"), + single_file=True, + allow_files=True), + "_nailgun_server_jar": attr.label( + default=Label("@nailgun_server_0_9_1_jar//jar"), + single_file=True, + allow_files=True), + +} + +scala_worker = rule( + implementation=_scala_worker_impl, + attrs={ + "main_class": attr.string(), + "exports": attr.label_list(allow_files=False), + # Worker Args + "worker": attr.label( + cfg=HOST_CFG, + default=Label("@io_bazel_rules_scala//scala:scala-worker"), + allow_files=True, + executable=True), + "worker_args": attr.string_list(), + } + _implicit_deps + _common_attrs + _zinc_compile_attrs, + outputs={ + "jar": "%{name}_deploy.jar", + "ijar": "%{name}_ijar.jar", + "manifest": "%{name}_MANIFEST.MF", + }, +) + scala_library = rule( implementation=_scala_library_impl, attrs={ @@ -381,6 +512,31 @@ filegroup( ) """ +SCALA_2_10_BUILD_FILE = """ +# scala.BUILD +exports_files([ + "bin/scala", + "bin/scalac", + "bin/scaladoc", + "lib/akka-actors.jar", + "lib/jline.jar", + "lib/scala-actors-migration.jar", + "lib/scala-actors.jar", + "lib/scala-compiler.jar", + "lib/scala-library.jar", + "lib/scala-reflect.jar", + "lib/scala-swing.jar", + "lib/scalap.jar", + "lib/typesafe-config.jar", +]) + +filegroup( + name = "sdk", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) +""" + def scala_repositories(): native.new_http_archive( name = "scala", @@ -394,3 +550,68 @@ def scala_repositories(): url = "https://oss.sonatype.org/content/groups/public/org/scalatest/scalatest_2.11/2.2.6/scalatest_2.11-2.2.6.jar", sha256 = "f198967436a5e7a69cfd182902adcfbcb9f2e41b349e1a5c8881a2407f615962", ) + +def scala_2_10_repositories(): + native.new_http_archive( + name = "scala", + strip_prefix = "scala-2.10.6", + sha256 = "54adf583dae6734d66328cafa26d9fa03b8c4cf607e27b9f3915f96e9bcd2d67", + url = "https://downloads.lightbend.com/scala/2.10.6/scala-2.10.6.tgz", + build_file_content = SCALA_2_10_BUILD_FILE, + ) + +def zinc_repositories(): + native.http_file( + name = "zinc", + url = "https://databricks-mvn.s3.amazonaws.com/binaries/zinc/12-29-15/zinc?AWSAccessKeyId=AKIAJ6V3VSHTA5RSYEQA&Expires=1482970631&Signature=tAP2QKWEnte6v3DpRFtAxGNkUps%3D", + sha256 = "255cbd2acb9e78ac30d20d3b57ba6fc4a38476b4eaa74173ba28c2839b4549df" + ) + + native.http_jar( + name = "scala_compiler_jar", + url = "https://databricks-mvn.s3.amazonaws.com/binaries/zinc/12-29-15/scala-compiler.jar?AWSAccessKeyId=AKIAJ6V3VSHTA5RSYEQA&Expires=1482962357&Signature=zDYAnLusbKWFFyhI3jokN%2FxissM%3D", + sha256 = "7ceaacf9b279b0e53c49234709623f55f6ce61613f14183a817e91e870da6bc8" + ) + + native.http_jar( + name = "incremental_compiler_0_13_9_jar", + url = "https://databricks-mvn.s3.amazonaws.com/binaries/zinc/12-29-15/incremental-compiler-0.13.9.jar?AWSAccessKeyId=AKIAJ6V3VSHTA5RSYEQA&Expires=1482971670&Signature=usTDvNkRldp8FSFys%2Fm0zKy0aHg%3D", + sha256 = "ddfbc88b9dd629118cad135ec32ec6cd1bc9969ca406cb780529a8cb037e1134" + ) + + native.http_jar( + name = "scala_library_jar", + url = "https://databricks-mvn.s3.amazonaws.com/binaries/zinc/12-29-15/scala-library.jar?AWSAccessKeyId=AKIAJ6V3VSHTA5RSYEQA&Expires=1482962357&Signature=fS5ZliC81RaOCArtLERGLOaCS2U%3D", + sha256 = "2aa6d7e5bb277c4072ac04433b9626aab586a313a41a57e192ea2acf430cdc29" + ) + + native.http_jar( + name = "sbt_interface_0_13_9_jar", + url = "https://databricks-mvn.s3.amazonaws.com/binaries/zinc/12-29-15/sbt-interface-0.13.9.jar?AWSAccessKeyId=AKIAJ6V3VSHTA5RSYEQA&Expires=1482962357&Signature=uu9taX3NAXzcieiTgcjFSOKqBVM%3D", + sha256 = "8004c0089728819896d678b3056b0ad0308e9760cb584b3cfc8eabde88f4e2bf" + ) + + native.http_jar( + name = "compiler_interface_0_13_9_sources_jar", + url = "https://databricks-mvn.s3.amazonaws.com/binaries/zinc/12-29-15/compiler-interface-0.13.9-sources.jar?AWSAccessKeyId=AKIAJ6V3VSHTA5RSYEQA&Expires=1482962357&Signature=bdNIS9%2BhpySfYIa7V9%2ByceDUClA%3D", + sha256 = "d124212ca6d83abe7ef4a275f545a2ac1d3fc8a43ac49d5e2a40054783062127" + ) + + native.http_jar( + name = "scala_reflect_jar", + url = "https://databricks-mvn.s3.amazonaws.com/binaries/zinc/12-29-15/scala-reflect.jar?AWSAccessKeyId=AKIAJ6V3VSHTA5RSYEQA&Expires=1482962357&Signature=btywJi2tZAudjoCVlMyjagDbQ2o%3D", + sha256 = "ad9b8ec8f7cb6a1d68d3b50a5d6cc61143b783f85523122871d98bac20dd48e3" + ) + + native.http_jar( + name = "zinc_0_3_10_SNAPSHOT_jar", + url = "https://databricks-mvn.s3.amazonaws.com/binaries/zinc/12-29-15/zinc-0.3.10-SNAPSHOT.jar?AWSAccessKeyId=AKIAJ6V3VSHTA5RSYEQA&Expires=1482968757&Signature=5mojfgtEVjsYUgoVWX54QSiUIu8%3D", + sha256 = "1db98ace1e69a7b7f757f7726e494816583ed44ca46ccd3ed11563772dacb915" + ) + + native.http_jar( + name = "nailgun_server_0_9_1_jar", + url = "http://central.maven.org/maven2/com/martiansoftware/nailgun-server/0.9.1/nailgun-server-0.9.1.jar", + sha256 = "4518faa6bf4bd26fccdc4d85e1625dc679381a08d56872d8ad12151dda9cef25" + ) +