packager: add GlobalTransformCache

Reviewed By: davidaurelio

Differential Revision: D4175938

fbshipit-source-id: 1f57d594b4c8c8189feb2ea6d4d4011870ffd85f

local-cli/bundle/bundleCommandLineArgs.js

@@ -45,5 +45,9 @@ module.exports = [
     command: '--reset-cache',
     description: 'Removes cached files',
     default: false,
+  }, {
+    command: '--read-global-cache',
+    description: 'Try to fetch transformed JS code from the global cache, if configured.',
+    default: false,
   },
 ];

package.json

@@ -175,6 +175,7 @@
     "joi": "^6.6.1",
     "json-stable-stringify": "^1.0.1",
     "json5": "^0.4.0",
+    "left-pad": "^1.1.3",
     "lodash": "^4.16.6",
     "mime": "^1.3.4",
     "mime-types": "2.1.11",
@@ -192,6 +193,7 @@
     "react-transform-hmr": "^1.0.4",
     "rebound": "^0.0.13",
     "regenerator-runtime": "^0.9.5",
+    "request": "^2.79.0",
     "rimraf": "^2.5.4",
     "sane": "~1.4.1",
     "semver": "^5.0.3",

packager/react-packager/src/Server/__tests__/Server-test.js

@@ -19,7 +19,8 @@ jest.setMock('worker-farm', function() { return () => {}; })
     .mock('../../AssetServer')
     .mock('../../lib/declareOpts')
     .mock('../../node-haste')
-    .mock('../../Logger');
+    .mock('../../Logger')
+    .mock('../../lib/GlobalTransformCache');
 
 describe('processRequest', () => {
   let SourceMapConsumer, Bundler, Server, AssetServer, Promise;

packager/react-packager/src/lib/GlobalTransformCache.js

@@ -0,0 +1,198 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+
+'use strict';
+
+const debounce = require('lodash/debounce');
+const imurmurhash = require('imurmurhash');
+const jsonStableStringify = require('json-stable-stringify');
+const path = require('path');
+const request = require('request');
+const toFixedHex = require('./toFixedHex');
+
+import type {CachedResult} from './TransformCache';
+
+const SINGLE_REQUEST_MAX_KEYS = 100;
+const AGGREGATION_DELAY_MS = 100;
+
+type FetchResultURIs = (
+  keys: Array<string>,
+  callback: (error?: Error, results?: Map<string, string>) => void,
+) => mixed;
+
+type FetchProps = {
+  filePath: string,
+  sourceCode: string,
+  transformCacheKey: string,
+  transformOptions: mixed,
+};
+
+type FetchCallback = (error?: Error, resultURI?: ?CachedResult) => mixed;
+type FetchURICallback = (error?: Error, resultURI?: ?string) => mixed;
+
+/**
+ * We aggregate the requests to do a single request for many keys. It also
+ * ensures we do a single request at a time to avoid pressuring the I/O.
+ */
+class KeyURIFetcher {
+
+  _fetchResultURIs: FetchResultURIs;
+  _pendingQueries: Array<{key: string, callback: FetchURICallback}>;
+  _isProcessing: boolean;
+  _processQueriesDebounced: () => void;
+  _processQueries: () => void;
+
+  /**
+   * Fetch the pending keys right now, if any and if we're not already doing
+   * so in parallel. At the end of the fetch, we trigger a new batch fetching
+   * recursively.
+   */
+  _processQueries() {
+    const {_pendingQueries} = this;
+    if (_pendingQueries.length === 0 || this._isProcessing) {
+      return;
+    }
+    this._isProcessing = true;
+    const queries = _pendingQueries.splice(0, SINGLE_REQUEST_MAX_KEYS);
+    const keys = queries.map(query => query.key);
+    this._fetchResultURIs(keys, (error, results) => {
+      queries.forEach(query => {
+        query.callback(error, results && results.get(query.key));
+      });
+      this._isProcessing = false;
+      process.nextTick(this._processQueries);
+    });
+  }
+
+  /**
+   * Enqueue the fetching of a particular key.
+   */
+  fetch(key: string, callback: FetchURICallback) {
+    this._pendingQueries.push({key, callback});
+    this._processQueriesDebounced();
+  }
+
+  constructor(fetchResultURIs: FetchResultURIs) {
+    this._fetchResultURIs = fetchResultURIs;
+    this._pendingQueries = [];
+    this._isProcessing = false;
+    this._processQueries = this._processQueries.bind(this);
+    this._processQueriesDebounced =
+      debounce(this._processQueries, AGGREGATION_DELAY_MS);
+  }
+
+}
+
+function validateCachedResult(cachedResult: mixed): ?CachedResult {
+  if (
+    cachedResult != null &&
+    typeof cachedResult === 'object' &&
+    typeof cachedResult.code === 'string' &&
+    Array.isArray(cachedResult.dependencies) &&
+    cachedResult.dependencies.every(dep => typeof dep === 'string') &&
+    Array.isArray(cachedResult.dependencyOffsets) &&
+    cachedResult.dependencyOffsets.every(offset => typeof offset === 'number')
+  ) {
+    return (cachedResult: any);
+  }
+  return undefined;
+}
+
+/**
+ * One can enable the global cache by calling configure() from a custom CLI
+ * script. Eventually we may make it more flexible.
+ */
+class GlobalTransformCache {
+
+  _fetcher: KeyURIFetcher;
+  static _global: ?GlobalTransformCache;
+
+  constructor(fetchResultURIs: FetchResultURIs) {
+    this._fetcher = new KeyURIFetcher(fetchResultURIs);
+  }
+
+  /**
+   * Return a key for identifying uniquely a source file.
+   */
+  static keyOf(props: FetchProps) {
+    const sourceDigest = toFixedHex(8, imurmurhash(props.sourceCode).result());
+    const optionsHash = imurmurhash()
+      .hash(jsonStableStringify(props.transformOptions) || '')
+      .hash(props.transformCacheKey)
+      .result();
+    const optionsDigest = toFixedHex(8, optionsHash);
+    return (
+      `${optionsDigest}${sourceDigest}` +
+      `${path.basename(props.filePath)}`
+    );
+  }
+
+  /**
+   * We may want to improve that logic to return a stream instead of the whole
+   * blob of transformed results. However the results are generally only a few
+   * megabytes each.
+   */
+  _fetchFromURI(uri: string, callback: FetchCallback) {
+    request.get({uri, json: true}, (error, response, unvalidatedResult) => {
+      if (error != null) {
+        callback(error);
+        return;
+      }
+      if (response.statusCode !== 200) {
+        callback(new Error(
+          `Unexpected HTTP status code: ${response.statusCode}`,
+        ));
+        return;
+      }
+      const result = validateCachedResult(unvalidatedResult);
+      if (result == null) {
+        callback(new Error('Invalid result returned by server.'));
+        return;
+      }
+      callback(undefined, result);
+    });
+  }
+
+  fetch(props: FetchProps, callback: FetchCallback) {
+    this._fetcher.fetch(GlobalTransformCache.keyOf(props), (error, uri) => {
+      if (error != null) {
+        callback(error);
+      } else {
+        if (uri == null) {
+          callback();
+          return;
+        }
+        this._fetchFromURI(uri, callback);
+      }
+    });
+  }
+
+  /**
+   * For using the global cache one needs to have some kind of central key-value
+   * store that gets prefilled using keyOf() and the transformed results. The
+   * fetching function should provide a mapping of keys to URIs. The files
+   * referred by these URIs contains the transform results. Using URIs instead
+   * of returning the content directly allows for independent fetching of each
+   * result.
+   */
+  static configure(fetchResultURIs: FetchResultURIs) {
+    GlobalTransformCache._global = new GlobalTransformCache(fetchResultURIs);
+  }
+
+  static get() {
+    return GlobalTransformCache._global;
+  }
+
+}
+
+GlobalTransformCache._global = null;
+
+module.exports = GlobalTransformCache;

packager/react-packager/src/lib/TransformCache.js

@@ -22,6 +22,7 @@ const jsonStableStringify = require('json-stable-stringify');
 const mkdirp = require('mkdirp');
 const path = require('path');
 const rimraf = require('rimraf');
+const toFixedHex = require('./toFixedHex');
 const writeFileAtomicSync = require('write-file-atomic').sync;
 
 const CACHE_NAME = 'react-native-packager-cache';
@@ -66,15 +67,14 @@ function getCacheFilePaths(props: {
   const hasher = imurmurhash()
     .hash(props.filePath)
     .hash(jsonStableStringify(props.transformOptions) || '');
-  let hash = hasher.result().toString(16);
-  hash = Array(8 - hash.length + 1).join('0') + hash;
+  const hash = toFixedHex(8, hasher.result());
   const prefix = hash.substr(0, 2);
   const fileName = `${hash.substr(2)}${path.basename(props.filePath)}`;
   const base = path.join(getCacheDirPath(), prefix, fileName);
   return {transformedCode: base, metadata: base + '.meta'};
 }
 
-type CachedResult = {
+export type CachedResult = {
   code: string,
   dependencies: Array<string>,
   dependencyOffsets: Array<number>,
@@ -135,7 +135,7 @@ function writeSync(props: {
   ]));
 }
 
-type CacheOptions = {resetCache?: boolean};
+export type CacheOptions = {resetCache?: boolean};
 
 /* 1 day */
 const GARBAGE_COLLECTION_PERIOD = 24 * 60 * 60 * 1000;
@@ -272,6 +272,14 @@ function readMetadataFileSync(
   };
 }
 
+export type ReadTransformProps = {
+  filePath: string,
+  sourceCode: string,
+  transformOptions: mixed,
+  transformCacheKey: string,
+  cacheOptions: CacheOptions,
+};
+
 /**
  * We verify the source hash matches to ensure we always favor rebuilding when
  * source change (rather than just using fs.mtime(), a bit less robust).
@@ -285,13 +293,7 @@ function readMetadataFileSync(
  * Meanwhile we store transforms with different options in different files so
  * that it is fast to switch between ex. minified, or not.
  */
-function readSync(props: {
-  filePath: string,
-  sourceCode: string,
-  transformOptions: mixed,
-  transformCacheKey: string,
-  cacheOptions: CacheOptions,
-}): ?CachedResult {
+function readSync(props: ReadTransformProps): ?CachedResult {
   GARBAGE_COLLECTOR.collectIfNecessarySync(props.cacheOptions);
   const cacheFilePaths = getCacheFilePaths(props);
   let metadata, transformedCode;

packager/react-packager/src/lib/__mocks__/GlobalTransformCache.js

@@ -0,0 +1,18 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+
+'use strict';
+
+function get() {
+  return null;
+}
+
+module.exports = {get};

packager/react-packager/src/lib/__tests__/TransformCache-test.js

@@ -12,7 +12,9 @@
 jest
   .dontMock('imurmurhash')
   .dontMock('json-stable-stringify')
-  .dontMock('../TransformCache');
+  .dontMock('../TransformCache')
+  .dontMock('../toFixedHex')
+  .dontMock('left-pad');
 
 const imurmurhash = require('imurmurhash');
 

packager/react-packager/src/lib/toFixedHex.js

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+
+'use strict';
+
+const leftPad = require('left-pad');
+
+function toFixedHex(length: number, number: number): string {
+  return leftPad(number.toString(16), length, '0');
+}
+
+module.exports = toFixedHex;