Add --maxWorkers flag and allow transformers to run in-band.

Summary:
This diff cleans up some cruft and adds some features:

* It removes the usage of an env variable to control workers.
* It removes the lazy and handwavy calculation on how many workers to use for jest-haste-map. Jest itself uses the maximum amount of workers available and it has never been reported as an issue – especially since it is a one-time startup cost of about 3 seconds on a cold cache only.
* It adds a `--max-workers` flag to replace the env variable. This one is able to control both the number of workers for `jest-haste-map` as well as the transformers.
* It makes the transformers run in the parent process if 1 or fewer workers are are specified. This should help with debugging.

Once you approve this diff, I will publish a new version of metro to npm and update the version used in RN and remove the use of the env variable altogether: https://our.intern.facebook.com/intern/biggrep/?corpus=xplat&filename=&case=false&view=default&extre=&s=REACT_NATIVE_MAX_WORKERS&engine=apr_strmatch&context=false&filter[uninteresting]=false&filter[intern]=false&filter[test]=false&grep_regex=

Note: the process of adding a CLI option is really broken. Commander also has a weird API. We should consider building a better public API for Metro and then consider how to build a new CLI on top of it and simplify our internal integration. I really don't like how Metro is integrated across pieces of the RN cli in ways that is hard to manage. But that is a larger task for another time :)

Reviewed By: jeanlauliac

Differential Revision: D5217726

fbshipit-source-id: 74efddbb87755a9e744c816fbc62efa21f6a79bf

ContainerShip/Dockerfile.android

@@ -3,7 +3,6 @@ FROM containership/android-base:latest
 # set default environment variables
 ENV GRADLE_OPTS="-Dorg.gradle.jvmargs=\"-Xmx512m -XX:+HeapDumpOnOutOfMemoryError\""
 ENV JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8"
-ENV REACT_NATIVE_MAX_WORKERS=1
 
 # add ReactAndroid directory
 ADD .buckconfig /app/.buckconfig

ContainerShip/scripts/run-ci-e2e-tests.sh

@@ -17,7 +17,6 @@ AVD_UUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 8 | head -n 1)
 ANDROID_NPM_DEPS="[email protected] [email protected] [email protected] [email protected] [email protected]"
 CLI_PACKAGE=$ROOT/react-native-cli/react-native-cli-*.tgz
 PACKAGE=$ROOT/react-native-*.tgz
-REACT_NATIVE_MAX_WORKERS=1
 
 # solve issue with max user watches limit
 echo 65536 | tee -a /proc/sys/fs/inotify/max_user_watches
@@ -231,13 +230,13 @@ function e2e_suite() {
     # js tests
     if [ $RUN_JS -ne 0 ]; then
       # Check the packager produces a bundle (doesn't throw an error)
-      REACT_NATIVE_MAX_WORKERS=1 react-native bundle --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js
+      react-native bundle --max-workers 1 --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js
       if [ $? -ne 0 ]; then
         echo "Could not build android bundle"
         return 1
       fi
 
-      REACT_NATIVE_MAX_WORKERS=1 react-native bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js
+      react-native bundle --max-workers 1 --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js
       if [ $? -ne 0 ]; then
         echo "Could not build iOS bundle"
         return 1

circle.yml

@@ -69,7 +69,7 @@ test:
 
     # integration tests
     # build JS bundle for instrumentation tests
-    - REACT_NATIVE_MAX_WORKERS=1 node local-cli/cli.js bundle --platform android --dev true --entry-file ReactAndroid/src/androidTest/js/TestBundle.js --bundle-output ReactAndroid/src/androidTest/assets/AndroidTestBundle.js
+    - node local-cli/cli.js bundle --max-workers 1 --platform android --dev true --entry-file ReactAndroid/src/androidTest/js/TestBundle.js --bundle-output ReactAndroid/src/androidTest/assets/AndroidTestBundle.js
     # build test APK
     - buck install ReactAndroid/src/androidTest/buck-runner:instrumentation-tests --config build.threads=1
     # run installed apk with tests

local-cli/bundle/buildBundle.js

@@ -13,7 +13,7 @@
 
 const log = require('../util/log').out('bundle');
 const Server = require('metro-bundler/build/Server');
-const Terminal = require('metro-bundler/build/lib/TerminalClass');
+const Terminal = require('metro-bundler/build/lib/Terminal');
 const TerminalReporter = require('metro-bundler/build/lib/TerminalReporter');
 const TransformCaching = require('metro-bundler/build/lib/TransformCaching');
 
@@ -38,6 +38,7 @@ function buildBundle(
   args: OutputOptions & {
     assetsDest: mixed,
     entryFile: string,
+    maxWorkers: number,
     resetCache: boolean,
     transformer: string,
   },
@@ -89,6 +90,7 @@ function buildBundle(
       getTransformOptions: config.getTransformOptions,
       globalTransformCache: null,
       hasteImpl: config.hasteImpl,
+      maxWorkers: args.maxWorkers,
       platforms: defaultPlatforms.concat(platforms),
       postMinifyProcess: config.postMinifyProcess,
       postProcessModules: config.postProcessModules,

local-cli/bundle/bundleCommandLineArgs.js

@@ -31,6 +31,12 @@ module.exports = [
     command: '--bundle-encoding [string]',
     description: 'Encoding the bundle should be written in (https://nodejs.org/api/buffer.html#buffer_buffer).',
     default: 'utf8',
+  }, {
+    command: '--max-workers [number]',
+    description: 'Specifies the maximum number of workers the worker-pool ' +
+      'will spawn for transforming files. This defaults to the number of the ' +
+      'cores available on your machine.',
+    parse: (workers: string) => Number(workers),
   }, {
     command: '--sourcemap-output [string]',
     description: 'File name where to store the sourcemap file for resulting bundle, ex. /tmp/groups.map',

local-cli/dependencies/dependencies.js

@@ -96,6 +96,12 @@ module.exports = {
     }, {
       command: '--transformer [path]',
       description: 'Specify a custom transformer to be used'
+    }, {
+      command: '--max-workers [number]',
+      description: 'Specifies the maximum number of workers the worker-pool ' +
+        'will spawn for transforming files. This defaults to the number of the ' +
+        'cores available on your machine.',
+      parse: (workers: string) => Number(workers),
     }, {
       command: '--dev [boolean]',
       description: 'If false, skip all dev-only code path',

local-cli/server/runServer.js

@@ -15,7 +15,7 @@
 require('../../setupBabel')();
 const InspectorProxy = require('./util/inspectorProxy.js');
 const ReactPackager = require('metro-bundler');
-const Terminal = require('metro-bundler/build/lib/TerminalClass');
+const Terminal = require('metro-bundler/build/lib/Terminal');
 
 const attachHMRServer = require('./util/attachHMRServer');
 const connect = require('connect');
@@ -46,6 +46,7 @@ import type {Reporter} from 'metro-bundler/build/lib/reporting';
 export type Args = {|
   +assetExts: $ReadOnlyArray<string>,
   +host: string,
+  +maxWorkers: number,
   +nonPersistent: boolean,
   +platforms: $ReadOnlyArray<string>,
   +port: number,
@@ -160,10 +161,11 @@ function getPackagerServer(args, config) {
     extraNodeModules: config.extraNodeModules,
     getTransformOptions: config.getTransformOptions,
     hasteImpl: config.hasteImpl,
+    maxWorkers: args.maxWorkers,
     platforms: defaultPlatforms.concat(args.platforms),
     polyfillModuleNames: config.getPolyfillModuleNames(),
-    postProcessModules: config.postProcessModules,
     postMinifyProcess: config.postMinifyProcess,
+    postProcessModules: config.postProcessModules,
     projectRoots: args.projectRoots,
     providesModuleNodeModules: providesModuleNodeModules,
     reporter: new LogReporter(terminal),

local-cli/server/server.js

@@ -103,6 +103,12 @@ module.exports = {
       }
       return null;
     },
+  }, {
+    command: '--max-workers [number]',
+    description: 'Specifies the maximum number of workers the worker-pool ' +
+      'will spawn for transforming files. This defaults to the number of the ' +
+      'cores available on your machine.',
+    parse: (workers: string) => Number(workers),
   }, {
     command: '--skipflow',
     description: 'Disable flow checks'

local-cli/util/Config.js

@@ -10,9 +10,8 @@
  */
 'use strict';
 
-const findSymlinksPaths = require('./findSymlinksPaths');
-
 const blacklist = require('metro-bundler/build/blacklist');
+const findSymlinksPaths = require('./findSymlinksPaths');
 const fs = require('fs');
 const invariant = require('fbjs/lib/invariant');
 const path = require('path');

package.json

@@ -182,7 +182,7 @@
     "left-pad": "^1.1.3",
     "lodash": "^4.16.6",
     "merge-stream": "^1.0.1",
-    "metro-bundler": "^0.7.4",
+    "metro-bundler": "^0.8.1",
     "mime": "^1.3.4",
     "mime-types": "2.1.11",
     "minimist": "^1.2.0",

scripts/run-ci-e2e-tests.js

@@ -125,12 +125,9 @@ try {
     }
 
     echo(`Starting packager server, ${SERVER_PID}`);
-    const packagerEnv = Object.create(process.env);
-    packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
     // shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
-    const packagerProcess = spawn('npm', ['start'], {
-      // stdio: 'inherit',
-      env: packagerEnv
+    const packagerProcess = spawn('npm', ['start', '--', '--max-workers 1'], {
+      env: process.env
     });
     SERVER_PID = packagerProcess.pid;
     // wait a bit to allow packager to startup
@@ -193,12 +190,12 @@ try {
 
   if (argv.js) {
     // Check the packager produces a bundle (doesn't throw an error)
-    if (exec('REACT_NATIVE_MAX_WORKERS=1 react-native bundle --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js').code) {
+    if (exec('react-native bundle --max-workers 1 --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js').code) {
       echo('Could not build Android bundle');
       exitCode = 1;
       throw Error(exitCode);
     }
-    if (exec('REACT_NATIVE_MAX_WORKERS=1 react-native bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js').code) {
+    if (exec('react-native --max-workers 1 bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js').code) {
       echo('Could not build iOS bundle');
       exitCode = 1;
       throw Error(exitCode);