feat(broccoli): add incremental MergeTrees plugin

tools/broccoli/broccoli-merge-trees.spec.ts

@@ -0,0 +1,68 @@
+/// <reference path="../typings/node/node.d.ts" />
+/// <reference path="../typings/jasmine/jasmine.d.ts" />
+
+let mockfs = require('mock-fs');
+import fs = require('fs');
+import {TreeDiffer} from './tree-differ';
+import {MergeTrees} from './broccoli-merge-trees';
+
+describe('MergeTrees', () => {
+  afterEach(() => mockfs.restore());
+
+  function mergeTrees(inputPaths, cachePath, options) {
+    return new MergeTrees(inputPaths, cachePath, options);
+  }
+
+  function MakeTreeDiffers(rootDirs) {
+    let treeDiffers = rootDirs.map((rootDir) => new TreeDiffer('MergeTrees', rootDir));
+    treeDiffers.diffTrees = () => { return treeDiffers.map(tree => tree.diffTree()); };
+    return treeDiffers;
+  }
+
+  function read(path) { return fs.readFileSync(path, "utf-8"); }
+
+  it('should copy the file from the right-most inputTree with overwrite=true', () => {
+    let testDir: any = {
+      'tree1': {'foo.js': mockfs.file({content: 'tree1/foo.js content', mtime: new Date(1000)})},
+      'tree2': {'foo.js': mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)})},
+      'tree3': {'foo.js': mockfs.file({content: 'tree3/foo.js content', mtime: new Date(1000)})}
+    };
+    mockfs(testDir);
+    let treeDiffer = MakeTreeDiffers(['tree1', 'tree2', 'tree3']);
+    let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {overwrite: true});
+    treeMerger.rebuild(treeDiffer.diffTrees());
+    expect(read('dest/foo.js')).toBe('tree3/foo.js content');
+
+    delete testDir.tree2['foo.js'];
+    delete testDir.tree3['foo.js'];
+    mockfs(testDir);
+    treeMerger.rebuild(treeDiffer.diffTrees());
+    expect(read('dest/foo.js')).toBe('tree1/foo.js content');
+
+    testDir.tree2['foo.js'] = mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)});
+    mockfs(testDir);
+    treeMerger.rebuild(treeDiffer.diffTrees());
+    expect(read('dest/foo.js')).toBe('tree2/foo.js content');
+  });
+
+  it('should throw if duplicates are used by default', () => {
+    let testDir: any = {
+      'tree1': {'foo.js': mockfs.file({content: 'tree1/foo.js content', mtime: new Date(1000)})},
+      'tree2': {'foo.js': mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)})},
+      'tree3': {'foo.js': mockfs.file({content: 'tree3/foo.js content', mtime: new Date(1000)})}
+    };
+    mockfs(testDir);
+    let treeDiffer = MakeTreeDiffers(['tree1', 'tree2', 'tree3']);
+    let treeMerger = mergeTrees(['tree1', 'tree2', 'tree3'], 'dest', {});
+    expect(() => treeMerger.rebuild(treeDiffer.diffTrees())).toThrow();
+
+    delete testDir.tree2['foo.js'];
+    delete testDir.tree3['foo.js'];
+    mockfs(testDir);
+    expect(() => treeMerger.rebuild(treeDiffer.diffTrees())).not.toThrow();
+
+    testDir.tree2['foo.js'] = mockfs.file({content: 'tree2/foo.js content', mtime: new Date(1000)});
+    mockfs(testDir);
+    expect(() => treeMerger.rebuild(treeDiffer.diffTrees())).toThrow();
+  });
+});

tools/broccoli/broccoli-merge-trees.ts

@@ -0,0 +1,119 @@
+import fs = require('fs');
+import fse = require('fs-extra');
+import path = require('path');
+var symlinkOrCopySync = require('symlink-or-copy').sync;
+import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin';
+
+interface MergeTreesOptions {
+  overwrite?: boolean;
+}
+
+function outputFileSync(sourcePath, destPath) {
+  let dirname = path.dirname(destPath);
+  fse.mkdirsSync(dirname, {fs: fs});
+  symlinkOrCopySync(sourcePath, destPath);
+}
+
+export class MergeTrees implements DiffingBroccoliPlugin {
+  private pathCache: {[key: string]: number[]} = Object.create(null);
+  public options: MergeTreesOptions;
+  private firstBuild: boolean = true;
+
+  constructor(public inputPaths: string[], public cachePath: string,
+              options: MergeTreesOptions = {}) {
+    this.options = options || {};
+  }
+
+  rebuild(treeDiffs: DiffResult[]) {
+    let overwrite = this.options.overwrite;
+    let pathsToEmit: string[] = [];
+    let pathsToRemove: string[] = [];
+    let emitted: {[key: string]: boolean} = Object.create(null);
+    let contains = (cache, val) => {
+      for (let i = 0, ii = cache.length; i < ii; ++i) {
+        if (cache[i] === val) return true;
+      }
+      return false;
+    };
+
+    let emit = (relativePath) => {
+      // ASSERT(!emitted[relativePath]);
+      pathsToEmit.push(relativePath);
+      emitted[relativePath] = true;
+    };
+
+    if (this.firstBuild) {
+      // Build initial cache
+      treeDiffs.reverse().forEach((treeDiff: DiffResult, index) => {
+        index = treeDiffs.length - 1 - index;
+        treeDiff.changedPaths.forEach((changedPath) => {
+          let cache = this.pathCache[changedPath];
+          if (cache === undefined) {
+            this.pathCache[changedPath] = [index];
+            pathsToEmit.push(changedPath);
+          } else if (overwrite) {
+            // ASSERT(contains(pathsToEmit, changedPath));
+            cache.unshift(index);
+          } else {
+            throw new Error("`overwrite` option is required for handling duplicates.");
+          }
+        });
+      });
+      this.firstBuild = false;
+    } else {
+      // Update cache
+      treeDiffs.reverse().forEach((treeDiff: DiffResult, index) => {
+        index = treeDiffs.length - 1 - index;
+        treeDiff.removedPaths.forEach((removedPath) => {
+          let cache = this.pathCache[removedPath];
+          // ASSERT(cache !== undefined);
+          // ASSERT(contains(cache, index));
+          if (cache[cache.length - 1] === index) {
+            pathsToRemove.push(path.join(this.cachePath, removedPath));
+            cache.pop();
+            if (cache.length === 0) {
+              this.pathCache[removedPath] = undefined;
+            } else if (!emitted[removedPath]) {
+              if (cache.length === 1 && !overwrite) {
+                throw new Error("`overwrite` option is required for handling duplicates.");
+              }
+              emit(removedPath);
+            }
+          }
+        });
+        treeDiff.changedPaths.forEach((changedPath) => {
+          let cache = this.pathCache[changedPath];
+          if (cache === undefined) {
+            // File was added
+            this.pathCache[changedPath] = [index];
+            emit(changedPath);
+          } else if (!contains(cache, index)) {
+            cache.push(index);
+            cache.sort((a, b) => a - b);
+            if (cache.length > 1 && !overwrite) {
+              throw new Error("`overwrite` option is required for handling duplicates.");
+            }
+            if (cache[cache.length - 1] === index && !emitted[changedPath]) {
+              emit(changedPath);
+            }
+          }
+        });
+      });
+    }
+
+    pathsToRemove.forEach((destPath) => fse.removeSync(destPath));
+    pathsToEmit.forEach((emittedPath) => {
+      let cache = this.pathCache[emittedPath];
+      let destPath = path.join(this.cachePath, emittedPath);
+      let sourceIndex = cache[cache.length - 1];
+      let sourceInputPath = this.inputPaths[sourceIndex];
+      let sourcePath = path.join(sourceInputPath, emittedPath);
+      if (cache.length > 1) {
+        fse.removeSync(destPath);
+      }
+      outputFileSync(sourcePath, destPath);
+    });
+  }
+}
+
+export default wrapDiffingPlugin(MergeTrees);

tools/broccoli/diffing-broccoli-plugin.ts

@@ -28,7 +28,7 @@ export function wrapDiffingPlugin(pluginClass): DiffingPluginWrapperFactory {
 
 
 export interface DiffingBroccoliPlugin {
-  rebuild(diff: DiffResult): (Promise<any>| void);
+  rebuild(diff: (DiffResult | DiffResult[])): (Promise<any>| void);
   cleanup ? () : void;
 }
 
@@ -38,7 +38,8 @@ type DiffingPluginWrapperFactory = (inputTrees: (BroccoliTree | BroccoliTree[]),
 
 
 class DiffingPluginWrapper implements BroccoliTree {
-  treeDiffer: TreeDiffer;
+  treeDiffer: TreeDiffer = null;
+  treeDiffers: TreeDiffer[] = null;
   initialized = false;
   wrappedPlugin: DiffingBroccoliPlugin = null;
   inputTree = null;
@@ -47,6 +48,7 @@ class DiffingPluginWrapper implements BroccoliTree {
 
   // props monkey-patched by broccoli builder:
   inputPath = null;
+  inputPaths = null;
   cachePath = null;
   outputPath = null;
 
@@ -61,13 +63,29 @@ class DiffingPluginWrapper implements BroccoliTree {
   }
 
 
+  private calculateDiff(firstRun: boolean): (DiffResult | DiffResult[]) {
+    if (this.treeDiffer) {
+      let diffResult = this.treeDiffer.diffTree();
+      diffResult.log(!firstRun);
+      return diffResult;
+    } else if (this.treeDiffers) {
+      return this.treeDiffers.map((treeDiffer) => treeDiffer.diffTree())
+          .map((diffResult) => {
+            diffResult.log(!firstRun);
+            return diffResult;
+          });
+    } else {
+      throw new Error("Missing TreeDiffer");
+    }
+  }
+
+
   rebuild() {
     try {
       let firstRun = !this.initialized;
       this.init();
 
-      let diffResult = this.treeDiffer.diffTree();
-      diffResult.log(!firstRun);
+      let diffResult = this.calculateDiff(firstRun);
 
       var rebuildPromise = this.wrappedPlugin.rebuild(diffResult);
 
@@ -101,17 +119,29 @@ class DiffingPluginWrapper implements BroccoliTree {
     if (!this.initialized) {
       let includeExtensions = this.pluginClass.includeExtensions || [];
       let excludeExtensions = this.pluginClass.excludeExtensions || [];
+      let description = this.description;
       this.initialized = true;
-      this.treeDiffer =
-          new TreeDiffer(this.description, this.inputPath, includeExtensions, excludeExtensions);
-      this.wrappedPlugin =
-          new this.pluginClass(this.inputPath, this.cachePath, this.wrappedPluginArguments[1]);
+      if (this.inputPaths) {
+        this.treeDiffers =
+            this.inputPaths.map((inputPath) => new TreeDiffer(
+                                    description, inputPath, includeExtensions, excludeExtensions));
+      } else if (this.inputPath) {
+        this.treeDiffer =
+            new TreeDiffer(description, this.inputPath, includeExtensions, excludeExtensions);
+      }
+      this.wrappedPlugin = new this.pluginClass(this.inputPaths || this.inputPath, this.cachePath,
+                                                this.wrappedPluginArguments[1]);
     }
   }
 
 
   private stabilizeTrees(trees: BroccoliTree[]) {
-    return trees.map((tree) => this.stabilizeTree(tree));
+    // Prevent extensions to prevent array from being mutated from the outside.
+    // For-loop used to avoid re-allocating a new array.
+    for (let i = 0; i < trees.length; ++i) {
+      trees[i] = this.stabilizeTree(trees[i]);
+    }
+    return Object.freeze(trees);
   }
 
 

tools/broccoli/trees/browser_tree.ts

@@ -3,7 +3,7 @@
 var Funnel = require('broccoli-funnel');
 var flatten = require('broccoli-flatten');
 var htmlReplace = require('../html-replace');
-var mergeTrees = require('broccoli-merge-trees');
+import mergeTrees from '../broccoli-merge-trees';
 var path = require('path');
 var replace = require('broccoli-replace');
 var stew = require('broccoli-stew');
@@ -16,6 +16,45 @@ import {default as transpileWithTraceur, TRACEUR_RUNTIME_PATH} from '../traceur/
 var projectRootDir = path.normalize(path.join(__dirname, '..', '..', '..', '..'));
 
 
+const kServedPaths = [
+  // Relative (to /modules) paths to benchmark directories
+  'benchmarks/src',
+  'benchmarks/src/change_detection',
+  'benchmarks/src/compiler',
+  'benchmarks/src/costs',
+  'benchmarks/src/di',
+  'benchmarks/src/element_injector',
+  'benchmarks/src/largetable',
+  'benchmarks/src/naive_infinite_scroll',
+  'benchmarks/src/tree',
+
+  // Relative (to /modules) paths to external benchmark directories
+  'benchmarks_external/src',
+  'benchmarks_external/src/compiler',
+  'benchmarks_external/src/largetable',
+  'benchmarks_external/src/naive_infinite_scroll',
+  'benchmarks_external/src/tree',
+  'benchmarks_external/src/tree/react',
+
+  // Relative (to /modules) paths to example directories
+  'examples/src/benchpress',
+  'examples/src/forms',
+  'examples/src/gestures',
+  'examples/src/hello_world',
+  'examples/src/key_events',
+  'examples/src/sourcemap',
+  'examples/src/todo',
+  'examples/src/material/button',
+  'examples/src/material/checkbox',
+  'examples/src/material/dialog',
+  'examples/src/material/grid_list',
+  'examples/src/material/input',
+  'examples/src/material/progress-linear',
+  'examples/src/material/radio',
+  'examples/src/material/switcher'
+];
+
+
 module.exports = function makeBrowserTree(options, destinationPath) {
   var modulesTree = new Funnel(
       'modules',
@@ -84,26 +123,30 @@ module.exports = function makeBrowserTree(options, destinationPath) {
       path.relative(projectRootDir, TRACEUR_RUNTIME_PATH)
     ]
   }));
+
   var vendorScripts_benchmark =
       new Funnel('tools/build/snippets', {files: ['url_params_to_form.js'], destDir: '/'});
   var vendorScripts_benchmarks_external =
       new Funnel('node_modules/angular', {files: ['angular.js'], destDir: '/'});
 
-  var servingTrees = [];
-
-  function copyVendorScriptsTo(destDir) {
-    servingTrees.push(new Funnel(vendorScriptsTree, {srcDir: '/', destDir: destDir}));
+  // Get scripts for each benchmark or example
+  let servingTrees = kServedPaths.reduce(getServedFunnels, []);
+  function getServedFunnels(funnels, destDir) {
+    let options = {
+      srcDir: '/',
+      destDir: destDir
+    };
+    funnels.push(new Funnel(vendorScriptsTree, options));
     if (destDir.indexOf('benchmarks') > -1) {
-      servingTrees.push(new Funnel(vendorScripts_benchmark, {srcDir: '/', destDir: destDir}));
+      funnels.push(new Funnel(vendorScripts_benchmark, options));
     }
     if (destDir.indexOf('benchmarks_external') > -1) {
-      servingTrees.push(
-          new Funnel(vendorScripts_benchmarks_external, {srcDir: '/', destDir: destDir}));
+      funnels.push(new Funnel(vendorScripts_benchmarks_external, options));
     }
+    return funnels;
   }
 
   function writeScriptsForPath(relativePath, result) {
-    copyVendorScriptsTo(path.dirname(relativePath));
     return result.replace('@@FILENAME_NO_EXT', relativePath.replace(/\.\w+$/, ''));
   }
 
@@ -124,11 +167,7 @@ module.exports = function makeBrowserTree(options, destinationPath) {
     replaceWithPath: writeScriptsForPath
   });
 
-  // Copy all vendor scripts into all examples and benchmarks
-  ['benchmarks/src', 'benchmarks_external/src', 'examples/src/benchpress'].forEach(
-      copyVendorScriptsTo);
-
-  var scripts = mergeTrees(servingTrees, {overwrite: true});
+  var scripts = mergeTrees(servingTrees);
   var css = new Funnel(modulesTree, {include: ["**/*.css"]});
   var polymerFiles = new Funnel('.', {
     files: [
@@ -145,7 +184,8 @@ module.exports = function makeBrowserTree(options, destinationPath) {
 
   es5Tree = mergeTrees([es5Tree, htmlTree]);
 
-  var mergedTree = mergeTrees([stew.mv(es6Tree, '/es6'), stew.mv(es5Tree, '/es5')]);
+  var mergedTree =
+      stew.log(mergeTrees([stew.mv(es6Tree, '/es6'), stew.mv(es5Tree, '/es5')]), {output: 'tree'});
 
   return destCopy(mergedTree, destinationPath);
 };