Skip to content

Commit

Permalink
feat(rollup): add worker support to rollup_bundle
Browse files Browse the repository at this point in the history
Adds rollup_bundle(supports_workers), disabled by default. This enables
rollup to be run in a worker process to perform incremental builds when
notified of file changes from bazel.

Worker mode currently supports only a very small subset of the standard
rollup CLI arguments.
  • Loading branch information
jbedard authored and alexeagle committed Mar 27, 2020
1 parent a031c3e commit 7cc42d6
Show file tree
Hide file tree
Showing 7 changed files with 334 additions and 4 deletions.
25 changes: 24 additions & 1 deletion packages/rollup/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

load("@build_bazel_rules_nodejs//:tools/defaults.bzl", "codeowners", "pkg_npm")
load("//third_party/github.com/bazelbuild/bazel-skylib:rules/copy_file.bzl", "copy_file")

codeowners(
teams = ["@jbedard"],
Expand All @@ -33,6 +34,25 @@ genrule(
visibility = ["//docs:__pkg__"],
)

# Copy the proto file to a matching third_party/... nested directory
# so the runtime require() statements still work
_worker_proto_dir = "third_party/github.com/bazelbuild/bazel/src/main/protobuf"

genrule(
name = "copy_worker_js",
srcs = ["//packages/worker:npm_package"],
outs = ["worker.js"],
cmd = "cp $(execpath //packages/worker:npm_package)/index.js $@",
visibility = ["//visibility:public"],
)

copy_file(
name = "copy_worker_proto",
src = "@build_bazel_rules_typescript//%s:worker_protocol.proto" % _worker_proto_dir,
out = "%s/worker_protocol.proto" % _worker_proto_dir,
visibility = ["//visibility:public"],
)

pkg_npm(
name = "npm_package",
srcs = [
Expand All @@ -48,5 +68,8 @@ pkg_npm(
# external/npm_bazel_rollup/docs.raw: Generating proto for Starlark doc for docs failed (Exit 1)
"@bazel_tools//src/conditions:windows": [],
"//conditions:default": [":generate_README"],
}),
}) + [
":copy_worker_proto",
":worker.js",
],
)
15 changes: 15 additions & 0 deletions packages/rollup/src/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary")
load("@build_bazel_rules_nodejs//tools/stardoc:index.bzl", "stardoc")

package(default_visibility = ["//visibility:public"])
Expand Down Expand Up @@ -42,8 +43,22 @@ filegroup(
name = "package_contents",
srcs = [
"index.bzl",
"index.js",
"package.json",
"rollup.config.js",
"rollup_bundle.bzl",
],
)

nodejs_binary(
name = "rollup-worker",
data = [
# NOTE: we ought to link to the original location instead of copying
# "@build_bazel_rules_nodejs//packages/worker:npm_package",
"@build_bazel_rules_nodejs//packages/rollup:worker.js",
"@build_bazel_rules_nodejs//packages/rollup:copy_worker_proto",
"@npm//protobufjs",
"@npm//rollup",
],
entry_point = "index.js",
)
7 changes: 6 additions & 1 deletion packages/rollup/src/index.from_src.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,9 @@

load(":index.bzl", _rollup_bundle = "rollup_bundle")

rollup_bundle = _rollup_bundle
def rollup_bundle(**kwargs):
_rollup_bundle(
# Override this attribute to point to our local one, not @npm
rollup_worker_bin = "@npm_bazel_rollup//:rollup-worker",
**kwargs
)
244 changes: 244 additions & 0 deletions packages/rollup/src/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,244 @@
/**
* @license
* Copyright 2020 The Bazel Authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
*
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const path = require('path');
const rollup = require('rollup');

const MNEMONIC = 'Rollup';
const PID = process.pid;

let worker;
try {
worker = require('@bazel/worker');
} catch {
// TODO: rely on the linker to link the first-party package
const helper = process.env['BAZEL_NODE_RUNFILES_HELPER'];
if (!helper) throw new Error('No runfiles helper and no @bazel/worker npm package');
const runfiles = require(helper);
const workerRequire = runfiles.resolve('build_bazel_rules_nodejs/packages/rollup/worker.js');
if (!workerRequire)
throw new Error(`build_bazel_rules_nodejs/packages/rollup/worker.js missing in runfiles ${
JSON.stringify(runfiles.manifest)}, ${runfiles.dir}`);
worker = require(workerRequire);
}

// Store the cache forever to re-use on each build
let cacheMap = {};
async function runRollup(cacheKey, inputOptions, outputOptions) {
let cache = cacheMap[cacheKey];

const rollupStartTime = Date.now();

const bundle = await rollup.rollup({...inputOptions, cache});

const rollupEndTime = Date.now();
worker.debug(
`${MNEMONIC}[${PID}][${cacheKey}].rollup()`, (rollupEndTime - rollupStartTime) / 1000);

cacheMap[cacheKey] = bundle.cache;

try {
await bundle.write(outputOptions);
} catch (e) {
worker.log(e);
return false;
}

const bundleEndTime = Date.now();
worker.debug(`${MNEMONIC}[${PID}][${cacheKey}].write()`, (bundleEndTime - rollupEndTime) / 1000);

return true;
}

// Run rollup, will use + re-populate the cache
async function runRollupBundler(args /*, inputs */) {
const {inputOptions, outputOptions} = await parseCLIArgs(args);

return runRollup('main', inputOptions, outputOptions);
}

// Load the config file.
// Must be rollup-ed first to allow use of es6 within the config.
// See the rollup CLI version:
// https://github.com/rollup/rollup/blob/v1.31.0/cli/run/loadConfigFile.ts#L14
async function loadConfigFile(configFile) {
const cjsConfigFile = configFile + '.cjs.js';

// inputOptions: https://github.com/rollup/rollup/blob/v1.31.0/cli/run/loadConfigFile.ts#L21-L28
const inputOptions = {
external: id => (id[0] !== '.' && !path.isAbsolute(id)) || id.slice(-5, id.length) === '.json',
input: configFile,
treeshake: false,
preserveSymlinks: true,
};

// outputOptions: https://github.com/rollup/rollup/blob/v1.31.0/cli/run/loadConfigFile.ts#L35-L38
const outputOptions = {
exports: 'named',
format: 'cjs',
file: cjsConfigFile,
};

await runRollup('configFile', inputOptions, outputOptions);

// Read the config file:
// https://github.com/rollup/rollup/blob/v1.31.0/cli/run/loadConfigFile.ts#L54-L61
//
// Supports:
// * async results
// * commonjs "default" export
let config = await Promise.resolve(require(cjsConfigFile));
if (config.default) {
config = config.default;
}

// Does NOT support (unlike rollup CLI):
// * factory function
// * multiple configs for multiple outputs
if (Array.isArray(config) || typeof config === 'function') {
throw new Error('Arrays + factory configs unsupported');
}

return config;
}

// Processing of --environment CLI options into environment vars
// https://github.com/rollup/rollup/blob/v1.31.0/cli/run/index.ts#L50-L57
function extractEnvironmentVariables(vars) {
vars.split(',').forEach(pair => {
const [key, ...value] = pair.split(':');
if (value.length) {
process.env[key] = value.join(':');
} else {
process.env[key] = String(true);
}
});
}

// Parse a subset of supported CLI arguments required for the rollup_bundle rule API.
// Returns input/outputOptions for the rollup.bundle/write() API
// input: https://rollupjs.org/guide/en/#inputoptions-object
// output: https://rollupjs.org/guide/en/#outputoptions-object
async function parseCLIArgs(args) {
let inputOptions = {
onwarn(...warnArgs) {
worker.log(...warnArgs);
},
};

let outputOptions = {};

let configFile = null;

// Input files to rollup
let inputs = [];

// Followed by suppported rollup CLI options
for (let i = 0; i < args.length; i++) {
const arg = args[i];

// Non-option is assumed to be an input file
if (!arg.startsWith('--')) {
inputs.push(arg);
continue;
}

const option = arg.slice(2);
switch (option) {
case 'config':
configFile = args[++i];
break;

case 'silent':
inputOptions.onwarn = () => {};
break;

case 'output.dir':
case 'output.file':
case 'format':
case 'sourcemap':
outputOptions[option.replace('output.', '')] = args[++i];
break;

case 'preserveSymlinks':
inputOptions[option] = true;
break;

// Common rollup CLI args, but not required for use
case 'environment':
extractEnvironmentVariables(args[++i]);
break;

default:
throw new Error(`${MNEMONIC}: invalid or unsupported argument ${arg}`);
}
}

// If outputting a directory then rollup_bundle.bzl passed a series
// of name=path files as the input.
// TODO: do some not have the =?
if (outputOptions.dir) {
inputs = inputs.reduce((m, nameInput) => {
const [name, input] = nameInput.split('=', 2);
m[name] = input;
return m;
}, {});
}

// Additional options passed via config file
if (configFile) {
const config = await loadConfigFile(path.resolve(configFile));

if (config.output) {
outputOptions = {...config.output, ...outputOptions};
delete config.output;
}

inputOptions = {...config, ...inputOptions};
}

// The inputs are the rule entry_point[s]
inputOptions.input = inputs;

return {inputOptions, outputOptions};
}

async function main(args) {
// Bazel will pass a special argument to the program when it's running us as a worker
if (worker.runAsWorker(args)) {
worker.log(`Running ${MNEMONIC} as a Bazel worker`);

worker.runWorkerLoop(runRollupBundler);
} else {
// Running standalone so stdout is available as usual
console.log(`Running ${MNEMONIC} as a standalone process`);
console.error(
`Started a new process to perform this action. Your build might be misconfigured, try
--strategy=${MNEMONIC}=worker`);

// Parse the options from the bazel-supplied options file.
// The first argument to the program is prefixed with '@'
// because Bazel does that for param files. Strip it first.
const paramFile = process.argv[2].replace(/^@/, '');
const args = require('fs').readFileSync(paramFile, 'utf-8').trim().split('\n');

return (await runRollupBundler(args)) ? 0 : 1;
}
}

if (require.main === module) {
main(process.argv.slice(2)).then(r => (process.exitCode = r));
}
6 changes: 6 additions & 0 deletions packages/rollup/src/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
"url" : "https://github.com/bazelbuild/rules_nodejs.git",
"directory": "packages/rollup"
},
"bin": {
"rollup-worker": "index.js"
},
"bugs": {
"url": "https://github.com/bazelbuild/rules_nodejs/issues"
},
Expand All @@ -22,5 +25,8 @@
"npm_bazel_rollup": {
"rootPath": "."
}
},
"dependencies": {
"@bazel/worker": "0.0.0-PLACEHOLDER"
}
}
Loading

0 comments on commit 7cc42d6

Please sign in to comment.