From b11684768f1fb7d6104c2c105864566833a6092b Mon Sep 17 00:00:00 2001 From: Daniel Muller Date: Mon, 24 Aug 2020 10:24:21 -0600 Subject: [PATCH] feat(typescript): worker mode for ts_project --- internal/node/node.bzl | 12 ++++- packages/typescript/internal/ts_project.bzl | 53 ++++++++++++++++++- .../typescript/internal/worker/BUILD.bazel | 27 ++++++++++ .../internal/worker/worker_adapter.js | 40 ++++++++++++++ .../test/ts_project/worker/BUILD.bazel | 6 +++ .../typescript/test/ts_project/worker/big.ts | 3 ++ .../test/ts_project/worker/tsconfig.json | 6 +++ .../test/ts_project/worker/worker_adapter.js | 40 ++++++++++++++ 8 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 packages/typescript/internal/worker/BUILD.bazel create mode 100644 packages/typescript/internal/worker/worker_adapter.js create mode 100644 packages/typescript/test/ts_project/worker/BUILD.bazel create mode 100644 packages/typescript/test/ts_project/worker/big.ts create mode 100644 packages/typescript/test/ts_project/worker/tsconfig.json create mode 100644 packages/typescript/test/ts_project/worker/worker_adapter.js diff --git a/internal/node/node.bzl b/internal/node/node.bzl index 56c6d3dd71..b9b32e958e 100644 --- a/internal/node/node.bzl +++ b/internal/node/node.bzl @@ -37,7 +37,7 @@ def _trim_package_node_modules(package_name): for n in package_name.split("/"): if n == "node_modules": break - segments += [n] + segments.append(n) return "/".join(segments) def _compute_node_modules_root(ctx): @@ -250,7 +250,15 @@ fi expanded_args = [expand_location_into_runfiles(ctx, a, ctx.attr.data) for a in expanded_args] # Next expand predefined variables & custom variables - expanded_args = [ctx.expand_make_variables("templated_args", e, {}) for e in expanded_args] + rule_dir = [f for f in [ + ctx.bin_dir.path, + ctx.label.workspace_root, + ctx.label.package, + ] if f] + additional_substitutions = {} + additional_substitutions["@D"] = "/".join([o for o in rule_dir if o]) + additional_substitutions["RULEDIR"] = "/".join([o for o in rule_dir if o]) + expanded_args = [ctx.expand_make_variables("templated_args", e, additional_substitutions) for e in expanded_args] substitutions = { # TODO: Split up results of multifile expansions into separate args and qoute them with diff --git a/packages/typescript/internal/ts_project.bzl b/packages/typescript/internal/ts_project.bzl index fcd0f2d13c..03c9b6e1c7 100644 --- a/packages/typescript/internal/ts_project.bzl +++ b/packages/typescript/internal/ts_project.bzl @@ -2,6 +2,7 @@ load("@build_bazel_rules_nodejs//:providers.bzl", "DeclarationInfo", "NpmPackageInfo", "declaration_info", "js_module_info", "run_node") load("@build_bazel_rules_nodejs//internal/linker:link_node_modules.bzl", "module_mappings_aspect") +load("@build_bazel_rules_nodejs//internal/node:node.bzl", "nodejs_binary") _DEFAULT_TSC = ( # BEGIN-INTERNAL @@ -22,7 +23,14 @@ _ATTRS = { # if you swap out the `compiler` attribute (like with ngtsc) # that compiler might allow more sources than tsc does. "srcs": attr.label_list(allow_files = True, mandatory = True), - "tsc": attr.label(default = Label(_DEFAULT_TSC), executable = True, cfg = "host"), + "supports_workers": attr.bool( + doc = """Experimental! Use only with caution. + +Allows you to enable the Bazel Worker strategy for this project. +This requires that the tsc binary support it.""", + default = False, + ), + "tsc": attr.label(default = Label(_DEFAULT_TSC), executable = True, cfg = "target"), "tsconfig": attr.label(mandatory = True, allow_single_file = [".json"]), } @@ -50,6 +58,13 @@ def _join(*elements): def _ts_project_impl(ctx): arguments = ctx.actions.args() + execution_requirements = {} + + if ctx.attr.supports_workers: + # Set to use a multiline param-file for worker mode + arguments.use_param_file("@%s", use_always = True) + arguments.set_param_file_format("multiline") + execution_requirements["supports-workers"] = "1" # Add user specified arguments *before* rule supplied arguments arguments.add_all(ctx.attr.args) @@ -128,6 +143,7 @@ def _ts_project_impl(ctx): arguments = [arguments], outputs = outputs, executable = "tsc", + execution_requirements = execution_requirements, progress_message = "Compiling TypeScript project %s [tsc -p %s]" % ( ctx.label, ctx.file.tsconfig.short_path, @@ -238,6 +254,7 @@ def ts_project_macro( emit_declaration_only = False, tsc = None, validate = True, + supports_workers = False, declaration_dir = None, out_dir = None, root_dir = None, @@ -366,6 +383,11 @@ def ts_project_macro( validate: boolean; whether to check that the tsconfig settings match the attributes. + supports_workers: Experimental! Use only with caution. + + Allows you to enable the Bazel Worker strategy for this project. + This requires that the tsc binary support it. + root_dir: a string specifying a subdirectory under the input package which should be consider the root directory of all the input files. Equivalent to the TypeScript --rootDir option. @@ -421,6 +443,35 @@ def ts_project_macro( ) extra_deps.append("_validate_%s_options" % name) + is_windows = select({ + "@bazel_tools//src/conditions:host_windows": True, + "//conditions:default": False, + }) + supports_workers = supports_workers and not is_windows + if supports_workers: + worker_name = name + "_worker" + nodejs_binary( + name = worker_name, + data = [ + tsconfig, + "//packages/typescript/internal/worker:copy_worker_js", + "//packages/typescript/internal/worker:copy_worker_proto", + "//packages/typescript/internal/worker:worker_adapter", + tsc, + ], + entry_point = "//packages/typescript/internal/worker:worker_adapter", + templated_args = [ + "--nobazel_patch_module_resolver", + "$(rootpath {tsc})".format(tsc = tsc), + "--project", + "$(execpath {tsconfig})".format(tsconfig = tsconfig), + "--outDir", + "$(RULEDIR)", + "--watch", + ], + ) + tsc = ":" + worker_name + typings_out_dir = declaration_dir if declaration_dir else out_dir ts_project( diff --git a/packages/typescript/internal/worker/BUILD.bazel b/packages/typescript/internal/worker/BUILD.bazel new file mode 100644 index 0000000000..8af83feb35 --- /dev/null +++ b/packages/typescript/internal/worker/BUILD.bazel @@ -0,0 +1,27 @@ +load("//internal/common:copy_to_bin.bzl", "copy_to_bin") +load("//third_party/github.com/bazelbuild/bazel-skylib:rules/copy_file.bzl", "copy_file") + +# Copy the proto file to a matching third_party/... nested directory +# so the runtime require() statements still work +_worker_proto_dir = "third_party/github.com/bazelbuild/bazel/src/main/protobuf" + +genrule( + name = "copy_worker_js", + srcs = ["//packages/worker:npm_package"], + outs = ["worker.js"], + cmd = "cp $(execpath //packages/worker:npm_package)/index.js $@", + visibility = ["//visibility:public"], +) + +copy_file( + name = "copy_worker_proto", + src = "@build_bazel_rules_typescript//%s:worker_protocol.proto" % _worker_proto_dir, + out = "%s/worker_protocol.proto" % _worker_proto_dir, + visibility = ["//visibility:public"], +) + +copy_to_bin( + name = "worker_adapter", + srcs = ["worker_adapter.js"], + visibility = ["//visibility:public"], +) diff --git a/packages/typescript/internal/worker/worker_adapter.js b/packages/typescript/internal/worker/worker_adapter.js new file mode 100644 index 0000000000..1eae15672f --- /dev/null +++ b/packages/typescript/internal/worker/worker_adapter.js @@ -0,0 +1,40 @@ +const child_process = require('child_process'); + +const worker = require('./worker'); + +const workerArg = process.argv.indexOf('--persistent_worker') +if (workerArg > 0) { + process.argv.splice(workerArg, 1) + + if (process.platform !== 'linux' && process.platform !== 'darwin') { + throw new Error('Worker mode is only supported on unix type systems.'); + } + + worker.runWorkerLoop(awaitOneBuild); +} + +const [tscBin, ...tscArgs] = process.argv.slice(2); + +const child = child_process.spawn( + tscBin, + tscArgs, + {stdio: 'pipe'}, +); + +function awaitOneBuild() { + child.kill('SIGCONT') + + return new Promise((res) => { + function awaitBuild(s) { + if (s.includes('Watching for file changes.')) { + child.kill('SIGSTOP') + + const success = s.includes('Found 0 errors.'); + res(success); + + child.stdout.removeListener('data', awaitBuild); + } + }; + child.stdout.on('data', awaitBuild); + }); +} \ No newline at end of file diff --git a/packages/typescript/test/ts_project/worker/BUILD.bazel b/packages/typescript/test/ts_project/worker/BUILD.bazel new file mode 100644 index 0000000000..d88f89fa68 --- /dev/null +++ b/packages/typescript/test/ts_project/worker/BUILD.bazel @@ -0,0 +1,6 @@ +load("//packages/typescript:index.bzl", "ts_project") + +ts_project( + declaration = True, + supports_workers = True, +) diff --git a/packages/typescript/test/ts_project/worker/big.ts b/packages/typescript/test/ts_project/worker/big.ts new file mode 100644 index 0000000000..c47d1b0914 --- /dev/null +++ b/packages/typescript/test/ts_project/worker/big.ts @@ -0,0 +1,3 @@ +// TODO: make it big enough to slow down tsc + +export const a: number = 45; diff --git a/packages/typescript/test/ts_project/worker/tsconfig.json b/packages/typescript/test/ts_project/worker/tsconfig.json new file mode 100644 index 0000000000..1e6e3b3652 --- /dev/null +++ b/packages/typescript/test/ts_project/worker/tsconfig.json @@ -0,0 +1,6 @@ +{ + "compilerOptions": { + "types": [], + "declaration": true + } +} \ No newline at end of file diff --git a/packages/typescript/test/ts_project/worker/worker_adapter.js b/packages/typescript/test/ts_project/worker/worker_adapter.js new file mode 100644 index 0000000000..1eae15672f --- /dev/null +++ b/packages/typescript/test/ts_project/worker/worker_adapter.js @@ -0,0 +1,40 @@ +const child_process = require('child_process'); + +const worker = require('./worker'); + +const workerArg = process.argv.indexOf('--persistent_worker') +if (workerArg > 0) { + process.argv.splice(workerArg, 1) + + if (process.platform !== 'linux' && process.platform !== 'darwin') { + throw new Error('Worker mode is only supported on unix type systems.'); + } + + worker.runWorkerLoop(awaitOneBuild); +} + +const [tscBin, ...tscArgs] = process.argv.slice(2); + +const child = child_process.spawn( + tscBin, + tscArgs, + {stdio: 'pipe'}, +); + +function awaitOneBuild() { + child.kill('SIGCONT') + + return new Promise((res) => { + function awaitBuild(s) { + if (s.includes('Watching for file changes.')) { + child.kill('SIGSTOP') + + const success = s.includes('Found 0 errors.'); + res(success); + + child.stdout.removeListener('data', awaitBuild); + } + }; + child.stdout.on('data', awaitBuild); + }); +} \ No newline at end of file