Skip to content

Commit

Permalink
feat(typescript): create a better ts_project worker
Browse files Browse the repository at this point in the history
  • Loading branch information
Dan Muller committed Jan 26, 2021
1 parent 3eb4260 commit 7111c73
Show file tree
Hide file tree
Showing 13 changed files with 334 additions and 153 deletions.
3 changes: 3 additions & 0 deletions examples/react_webpack/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ sass(
ts_project(
# Experimental: Start a tsc daemon to watch for changes to make recompiles faster.
supports_workers = True,
# Show an example of how to run the daemon with a custom tsc.
typescript_module = "@npm//ttypescript",
typescript_module_name = "ttypescript",
deps = [
"@npm//@types",
"@npm//csstype",
Expand Down
3 changes: 2 additions & 1 deletion examples/react_webpack/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
"typescript": "^3.6.3",
"webpack": "^4.41.0",
"webpack-cli": "^3.3.9",
"webpack-dev-server": "^3.8.2"
"webpack-dev-server": "^3.8.2",
"ttypescript": "^1.5.12"
},
"scripts": {
"build": "bazel build //...",
Expand Down
7 changes: 6 additions & 1 deletion examples/react_webpack/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@
"lib": [
"ES2015",
"DOM"
],
"plugins": [
{
"transform": "transformer-module"
},
]
},
// When using ts_project in worker mode, we run outside the Bazel sandbox (unless using --worker_sandboxing).
Expand All @@ -12,4 +17,4 @@
"*.tsx",
"*.ts"
]
}
}
29 changes: 28 additions & 1 deletion examples/react_webpack/yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -1807,7 +1807,7 @@ has-values@^1.0.0:
is-number "^3.0.0"
kind-of "^4.0.0"

has@^1.0.1:
has@^1.0.1, has@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796"
integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==
Expand Down Expand Up @@ -2109,6 +2109,13 @@ is-buffer@^1.1.5:
resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==

is-core-module@^2.1.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.2.0.tgz#97037ef3d52224d85163f5597b2b63d9afed981a"
integrity sha512-XRAfAdyyY5F5cOXn7hYQDqh2Xmii+DEfIcQGxK/uNwMHhIkPWO0g8msXcbzLe+MpGoR951MlqM/2iIlU4vKDdQ==
dependencies:
has "^1.0.3"

is-data-descriptor@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56"
Expand Down Expand Up @@ -3032,6 +3039,11 @@ path-key@^2.0.0, path-key@^2.0.1:
resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40"
integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=

path-parse@^1.0.6:
version "1.0.6"
resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c"
integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==

[email protected]:
version "0.1.7"
resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c"
Expand Down Expand Up @@ -3462,6 +3474,14 @@ resolve-url@^0.2.1:
resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a"
integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=

resolve@>=1.9.0:
version "1.19.0"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c"
integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==
dependencies:
is-core-module "^2.1.0"
path-parse "^1.0.6"

ret@~0.1.10:
version "0.1.15"
resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"
Expand Down Expand Up @@ -4080,6 +4100,13 @@ [email protected]:
resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6"
integrity sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=

ttypescript@^1.5.12:
version "1.5.12"
resolved "https://registry.yarnpkg.com/ttypescript/-/ttypescript-1.5.12.tgz#27a8356d7d4e719d0075a8feb4df14b52384f044"
integrity sha512-1ojRyJvpnmgN9kIHmUnQPlEV1gq+VVsxVYjk/NfvMlHSmYxjK5hEvOOU2MQASrbekTUiUM7pR/nXeCc8bzvMOQ==
dependencies:
resolve ">=1.9.0"

type-is@~1.6.17, type-is@~1.6.18:
version "1.6.18"
resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131"
Expand Down
2 changes: 1 addition & 1 deletion packages/typescript/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ pkg_npm(
":npm_version_check",
"//packages/typescript/internal:BUILD",
"//packages/typescript/internal:ts_project_options_validator.js",
"//packages/typescript/internal/worker",
"//packages/typescript/internal/worker:filegroup",
] + select({
# FIXME: fix stardoc on Windows; //packages/typescript:index.md generation fails with:
# ERROR: D:/b/62unjjin/external/npm_bazel_typescript/BUILD.bazel:36:1: Couldn't build file
Expand Down
54 changes: 19 additions & 35 deletions packages/typescript/internal/ts_project.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,6 @@ _DEFAULT_TSC = (
"//typescript/bin:tsc"
)

_DEFAULT_TSC_BIN = (
# BEGIN-INTERNAL
"@npm" +
# END-INTERNAL
"//:node_modules/typescript/bin/tsc"
)

_DEFAULT_TYPESCRIPT_MODULE = (
# BEGIN-INTERNAL
"@npm" +
Expand Down Expand Up @@ -49,7 +42,7 @@ _ATTRS = {
# that compiler might allow more sources than tsc does.
"srcs": attr.label_list(allow_files = True, mandatory = True),
"supports_workers": attr.bool(default = False),
"tsc": attr.label(default = Label(_DEFAULT_TSC), executable = True, cfg = "target"),
"tsc": attr.label(default = Label(_DEFAULT_TSC), executable = True, cfg = "host"),
"tsconfig": attr.label(mandatory = True, allow_single_file = [".json"]),
}

Expand Down Expand Up @@ -209,6 +202,7 @@ def _ts_project_impl(ctx):
inputs = inputs,
arguments = [arguments],
outputs = outputs,
mnemonic = "TsProject",
executable = "tsc",
execution_requirements = execution_requirements,
progress_message = "%s %s [tsc -p %s]" % (
Expand Down Expand Up @@ -337,8 +331,8 @@ def ts_project_macro(
emit_declaration_only = False,
ts_build_info_file = None,
tsc = None,
worker_tsc_bin = _DEFAULT_TSC_BIN,
worker_typescript_module = _DEFAULT_TYPESCRIPT_MODULE,
typescript_module = _DEFAULT_TYPESCRIPT_MODULE,
typescript_module_name = "typescript",
validate = True,
supports_workers = False,
declaration_dir = None,
Expand Down Expand Up @@ -506,14 +500,13 @@ def ts_project_macro(
For example, `tsc = "@my_deps//typescript/bin:tsc"`
Or you can pass a custom compiler binary instead.
worker_tsc_bin: Label of the TypeScript compiler binary to run when running in worker mode.
typescript_module: Label of the package containing all data deps of tsc.
For example, `tsc = "@my_deps//node_modules/typescript/bin/tsc"`
Or you can pass a custom compiler binary instead.
For example, `tsc = "@my_deps//typescript"`
worker_typescript_module: Label of the package containing all data deps of worker_tsc_bin.
typescript_module_name: Module name which resolves to typescript_module when required
For example, `tsc = "@my_deps//typescript"`
For example, `tsc = "typescript"`
validate: boolean; whether to check that the tsconfig settings match the attributes.
Expand Down Expand Up @@ -637,30 +630,24 @@ def ts_project_macro(
nodejs_binary(
name = tsc_worker,
data = [
# BEGIN-INTERNAL
# Users get this dependency transitively from @bazel/typescript
# but that's our own code, so we don't.
"@npm//protobufjs",
# END-INTERNAL
Label("//packages/typescript/internal/worker:worker"),
Label(worker_tsc_bin),
Label(worker_typescript_module),
Label(
# BEGIN-INTERNAL
"@npm" +
# END-INTERNAL
"//protobufjs",
),
Label(typescript_module),
Label("//packages/typescript/internal/worker:filegroup"),
tsconfig,
],
entry_point = Label("//packages/typescript/internal/worker:worker_adapter"),
templated_args = [
"$(execpath {})".format(Label(worker_tsc_bin)),
"--project",
"$(execpath {})".format(tsconfig),
# FIXME: should take out_dir into account
"--outDir",
"$(RULEDIR)",
# FIXME: what about other settings like declaration_dir, root_dir, etc
"--typescript_module_name",
typescript_module_name,
],
)

tsc = ":" + tsc_worker

typings_out_dir = declaration_dir if declaration_dir else out_dir
tsbuildinfo_path = ts_build_info_file if ts_build_info_file else name + ".tsbuildinfo"
js_outs = []
Expand Down Expand Up @@ -701,9 +688,6 @@ Check the srcs attribute to see that some .ts files are present (or .js files wi
buildinfo_out = tsbuildinfo_path if composite or incremental else None,
tsc = tsc,
link_workspace_root = link_workspace_root,
supports_workers = select({
"@bazel_tools//src/conditions:host_windows": False,
"//conditions:default": supports_workers,
}),
supports_workers = supports_workers,
**kwargs
)
27 changes: 15 additions & 12 deletions packages/typescript/internal/worker/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,8 +1,18 @@
# BEGIN-INTERNAL

load("//internal/common:copy_to_bin.bzl", "copy_to_bin")
load("//packages/typescript:checked_in_ts_project.bzl", "checked_in_ts_project")
load("//third_party/github.com/bazelbuild/bazel-skylib:rules/copy_file.bzl", "copy_file")

# To update index.js run:
# bazel run //packages/typescript/internal/worker:worker_adapter_check_compiled.update

checked_in_ts_project(
name = "worker_adapter",
src = "worker_adapter.ts",
checked_in_js = "index.js",
visibility = ["//visibility:public"],
deps = ["@npm//@types/node"],
)

# Copy the proto file to a matching third_party/... nested directory
# so the runtime require() statements still work
_worker_proto_dir = "third_party/github.com/bazelbuild/bazel/src/main/protobuf"
Expand All @@ -22,14 +32,6 @@ copy_file(
visibility = ["//visibility:public"],
)

copy_to_bin(
name = "worker_adapter",
srcs = [
"worker_adapter.js",
],
visibility = ["//visibility:public"],
)

filegroup(
name = "package_contents",
srcs = [
Expand All @@ -41,12 +43,13 @@ filegroup(
# END-INTERNAL

exports_files([
"worker_adapter.js",
"index.js",
])

filegroup(
name = "worker",
name = "filegroup",
srcs = [
"index.js",
"third_party/github.com/bazelbuild/bazel/src/main/protobuf/worker_protocol.proto",
"worker.js",
"worker_adapter.js",
Expand Down
93 changes: 93 additions & 0 deletions packages/typescript/internal/worker/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/* THIS FILE GENERATED FROM .ts; see BUILD.bazel */ /* clang-format off */"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const fs = require("fs");
const ts = require("typescript");
const MNEMONIC = "TsProject";
const worker = require("./worker");
let createWatchCompilerHost;
const formatHost = {
getCanonicalFileName: (path) => path,
getCurrentDirectory: ts.sys.getCurrentDirectory,
getNewLine: () => ts.sys.newLine,
};
const reportDiagnostic = (diagnostic) => {
worker.log(ts.formatDiagnostic(diagnostic, formatHost));
};
const reportWatchStatusChanged = (diagnostic) => {
worker.debug(ts.formatDiagnostic(diagnostic, formatHost));
};
function createWatchProgram(options, tsconfigPath, setTimeout) {
const host = createWatchCompilerHost(tsconfigPath, options, Object.assign(Object.assign({}, ts.sys), { setTimeout }), ts.createEmitAndSemanticDiagnosticsBuilderProgram, reportDiagnostic, reportWatchStatusChanged);
return ts.createWatchProgram(host);
}
let workerRequestTimestamp;
let cachedWatchedProgram;
let consolidateChangesCallback;
let cachedWatchProgramArgs;
function getWatchProgram(args) {
const newWatchArgs = args.join(" ");
if (cachedWatchedProgram &&
cachedWatchProgramArgs &&
cachedWatchProgramArgs !== newWatchArgs) {
cachedWatchedProgram.close();
cachedWatchedProgram = undefined;
cachedWatchProgramArgs = undefined;
}
if (!cachedWatchedProgram) {
const parsedArgs = ts.parseCommandLine(args);
const tsconfigPath = args[args.indexOf("--project") + 1];
cachedWatchProgramArgs = newWatchArgs;
cachedWatchedProgram = createWatchProgram(parsedArgs.options, tsconfigPath, (callback) => {
consolidateChangesCallback = callback;
});
}
return cachedWatchedProgram;
}
function emitOnce(args) {
const watchProgram = getWatchProgram(args);
if (consolidateChangesCallback) {
consolidateChangesCallback();
}
return new Promise((res) => {
var _a;
workerRequestTimestamp = Date.now();
const result = (_a = watchProgram) === null || _a === void 0 ? void 0 : _a.getProgram().emit(undefined, undefined, {
isCancellationRequested: function (timestamp) {
return timestamp !== workerRequestTimestamp;
}.bind(null, workerRequestTimestamp),
throwIfCancellationRequested: function (timestamp) {
if (timestamp !== workerRequestTimestamp) {
throw new ts.OperationCanceledException();
}
}.bind(null, workerRequestTimestamp),
});
res(result && result.diagnostics.length === 0);
});
}
function main() {
const typescriptModuleName = process.argv[process.argv.indexOf('--typescript_module_name') + 1];
try {
const customTypescriptModule = require(typescriptModuleName);
createWatchCompilerHost = customTypescriptModule.createWatchCompilerHost;
}
catch (e) {
worker.log('typescript_module_name must be a requirable module path');
throw e;
}
if (process.argv.includes("--persistent_worker")) {
worker.log(`Running ${MNEMONIC} as a Bazel worker`);
worker.runWorkerLoop(emitOnce);
}
else {
worker.log(`Running ${MNEMONIC} as a standalone process`);
worker.log(`Started a new process to perform this action. Your build might be misconfigured, try
--strategy=${MNEMONIC}=worker`);
let argsFilePath = process.argv.pop();
if (argsFilePath.startsWith('@')) {
argsFilePath = argsFilePath.slice(1);
}
const args = fs.readFileSync(argsFilePath).toString().split('\n');
emitOnce(args).finally(() => { var _a; return (_a = cachedWatchedProgram) === null || _a === void 0 ? void 0 : _a.close(); });
}
}
main();
Loading

0 comments on commit 7111c73

Please sign in to comment.