feat: Add 'changedSince' option (#80)

* Remove unneeded variable from the Prefixer class.

* Add jest-changed-files package.

* Remove duplicate setup for prettier.

* Add new option called changedSince. Add implementation for it.

* Update the help for the new command.

* Separate the changedSince functionality in a separate function.

* The filter function has been separated in a separate file. Added tests for it.

* Added documentation for changedSince.

.vscode/settings.json

@@ -1,8 +1,5 @@
 {
   "editor.formatOnSave": true,
-  "prettier.printWidth": 100,
-  "prettier.semi": false,
-  "prettier.singleQuote": true,
   "search.exclude": {
     "**/node_modules": true,
     "**/bower_components": true,

README.md

@@ -15,6 +15,8 @@ Mode (choose one):
 Package Options:
   --recursive, -r  Execute the same script on all of its dependencies, too                    [boolean]
   --package, -p    Run only for packages matching this glob. Can be used multiple times.        [array]
+  --changedSince   Runs commands in packages that have changed since the provided source control
+                   branch.                                                                     [string]
 
 Misc Options:
   --fast-exit, e        If at least one script exits with code > 0, abort                     [boolean]
@@ -70,6 +72,9 @@ in order, continuing when command outputs a line containing "Compilation complet
 
 `yarn wsrun --exclude-missing test` will run the test script only on packages that have it
 
+`yarn wsrun --changedSince --exclude-missing test` will run the test script only on packages that have c
+hanged since master branch and have `test` command
+
 #### Additional arguments to scripts
 
 If you want to pass additional arguments to the command you can do that by adding them after the

package.json

@@ -70,6 +70,7 @@
     "minimatch": "^3.0.4",
     "split": "^1.0.1",
     "throat": "^4.1.0",
-    "yargs": "^11.1.1"
+    "yargs": "^11.1.1",
+    "jest-changed-files": "^24.9.0"
   }
 }

src/filter-changed-packages.spec.ts

@@ -0,0 +1,19 @@
+import { filterChangedPackages } from './filter-changed-packages'
+
+describe('filterChangedPackages', () => {
+  it('should filter only the right package', async () => {
+    const res = filterChangedPackages(
+      ['packages/a/e.ts', 'packages/c/e.ts'],
+      { a: 'a', b: 'b' },
+      'packages'
+    )
+
+    expect(res).toEqual(['a'])
+  })
+
+  it('should filter out all packages if no package match is found', async () => {
+    const res = filterChangedPackages(['packages2/c/example.ts'], { a: 'a', b: 'b' }, 'packages2')
+
+    expect(res).toEqual([])
+  })
+})

src/filter-changed-packages.ts

@@ -0,0 +1,20 @@
+import { Dict } from './workspace'
+import * as path from 'path'
+
+/**
+ * filter the packages by checking if they have any changed files. This way is quicker
+ * (mapping over packages) because package count is usually lower than changed files count
+ * and we only need to check once per package.
+ */
+export const filterChangedPackages = (
+  files: string[],
+  pkgPaths: Dict<string>,
+  workspacePath: string
+) => {
+  return Object.keys(pkgPaths).filter(pkg => {
+    const pkgPath = pkgPaths[pkg]
+    const p = path.join(workspacePath, pkgPath)
+
+    return files.some(f => f.startsWith(p))
+  })
+}

src/index.ts

@@ -47,7 +47,7 @@ let yargsParser = yargs
       describe: 'Same as "stages" but with no parallelism at the stage level'
     }
   })
-  .group(['recursive', 'package'], 'Package Options:')
+  .group(['recursive', 'package', 'changedSince'], 'Package Options:')
   .options({
     package: {
       alias: 'p',
@@ -64,10 +64,18 @@ let yargsParser = yargs
       default: false,
       boolean: true,
       describe: 'Execute the same script on all of its dependencies, too'
+    },
+    changedSince: {
+      type: 'string',
+      nargs: 1,
+      describe:
+        'Runs commands in packages that have changed since the provided source control branch.'
     }
   })
   .group(
     [
+      'if',
+      'ifDependency',
       'fast-exit',
       'collect-logs',
       'no-prefix',
@@ -76,9 +84,7 @@ let yargsParser = yargs
       'done-criteria',
       'exclude',
       'exclude-missing',
-      'report',
-      'if',
-      'ifDependency'
+      'report'
     ],
     'Misc Options:'
   )
@@ -212,6 +218,7 @@ let runner = new RunGraph(
     mode: mode as any,
     recursive: argv.recursive,
     doneCriteria: argv.doneCriteria,
+    changedSince: argv.changedSince,
     exclude,
     excludeMissing: argv.excludeMissing,
     showReport: argv.report,

src/run-graph.ts

@@ -3,11 +3,13 @@ import chalk from 'chalk'
 
 import { PkgJson, Dict } from './workspace'
 import { ResultSpecialValues, Result, ProcResolution } from './enums'
-import { uniq } from 'lodash'
+import { uniq, values } from 'lodash'
 import { CmdProcess } from './cmd-process'
 import minimatch = require('minimatch')
 import { fixPaths } from './fix-paths'
 import { ConsoleFactory, SerializedConsole, DefaultConsole } from './console'
+import { getChangedFilesForRoots } from 'jest-changed-files'
+import { filterChangedPackages } from './filter-changed-packages'
 
 type PromiseFn<T> = () => Bromise<T>
 type PromiseFnRunner = <T>(f: PromiseFn<T>) => Bromise<T>
@@ -17,7 +19,7 @@ let mkThroat = require('throat')(Bromise) as (limit: number) => PromiseFnRunner
 let passThrough: PromiseFnRunner = f => f()
 
 class Prefixer {
-  constructor(private wspath: string) {}
+  constructor() {}
   private currentName = ''
   prefixer = (basePath: string, pkg: string, line: string) => {
     let l = ''
@@ -36,6 +38,7 @@ export interface GraphOptions {
   mode: 'parallel' | 'serial' | 'stages'
   recursive: boolean
   doneCriteria: string | undefined
+  changedSince: string | undefined
   workspacePath: string
   exclude: string[]
   excludeMissing: boolean
@@ -54,7 +57,7 @@ export class RunGraph {
   private resultMap = new Map<string, Result>()
   private throat: PromiseFnRunner = passThrough
   private consoles: ConsoleFactory
-  prefixer = new Prefixer(this.opts.workspacePath).prefixer
+  private prefixer = new Prefixer().prefixer
   pathRewriter = (pkgPath: string, line: string) => fixPaths(this.opts.workspacePath, pkgPath, line)
 
   constructor(
@@ -305,12 +308,30 @@ export class RunGraph {
     return pkgsInError.length > 0
   }
 
-  expandGlobs(globs: string[]) {
-    return this.pkgJsons.map(p => p.name).filter(name => globs.some(glob => minimatch(name, glob)))
+  async expandGlobs(globs: string[]) {
+    let pkgs = this.pkgJsons
+      .map(p => p.name)
+      .filter(name => globs.some(glob => minimatch(name, glob)))
+
+    // if changedSince is defined, filter the packages to contain only changed packages (according to git)
+    if (this.opts.changedSince) {
+      return getChangedFilesForRoots([this.opts.workspacePath], {
+        changedSince: this.opts.changedSince
+      }).then(data => {
+        if (!data.repos || (data.repos.git.size === 0 && data.repos.hg.size === 0)) {
+          throw new Error(
+            "The workspace is not a git/hg repo and it cannot work with 'changedSince'"
+          )
+        }
+
+        return filterChangedPackages([...data.changedFiles], this.pkgPaths, this.opts.workspacePath)
+      })
+    }
+    return Promise.resolve(pkgs)
   }
 
-  run(cmd: string[], globs: string[] = ['**/*']) {
-    let pkgs = this.expandGlobs(globs)
+  async run(cmd: string[], globs: string[] = ['**/*']) {
+    let pkgs = await this.expandGlobs(globs)
     this.runList = new Set(pkgs)
     return (
       Bromise.all(pkgs.map(pkg => this.lookupOrRun(cmd, pkg)))

yarn.lock

@@ -18,6 +18,15 @@
     esutils "^2.0.2"
     js-tokens "^4.0.0"
 
+"@jest/types@^24.9.0":
+  version "24.9.0"
+  resolved "https://registry.yarnpkg.com/@jest/types/-/types-24.9.0.tgz#63cb26cb7500d069e5a389441a7c6ab5e909fc59"
+  integrity sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==
+  dependencies:
+    "@types/istanbul-lib-coverage" "^2.0.0"
+    "@types/istanbul-reports" "^1.1.1"
+    "@types/yargs" "^13.0.0"
+
 "@nodelib/[email protected]":
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.1.tgz#7fa8fed654939e1a39753d286b48b4836d00e0eb"
@@ -190,6 +199,26 @@
     "@types/minimatch" "*"
     "@types/node" "*"
 
+"@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0":
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.1.tgz#42995b446db9a48a11a07ec083499a860e9138ff"
+  integrity sha512-hRJD2ahnnpLgsj6KWMYSrmXkM3rm2Dl1qkx6IOFD5FnuNPXJIG5L0dhgKXCYTRMGzU4n0wImQ/xfmRc4POUFlg==
+
+"@types/istanbul-lib-report@*":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@types/istanbul-lib-report/-/istanbul-lib-report-1.1.1.tgz#e5471e7fa33c61358dd38426189c037a58433b8c"
+  integrity sha512-3BUTyMzbZa2DtDI2BkERNC6jJw2Mr2Y0oGI7mRxYNBPxppbtEK1F66u3bKwU2g+wxwWI7PAoRpJnOY1grJqzHg==
+  dependencies:
+    "@types/istanbul-lib-coverage" "*"
+
+"@types/istanbul-reports@^1.1.1":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@types/istanbul-reports/-/istanbul-reports-1.1.1.tgz#7a8cbf6a406f36c8add871625b278eaf0b0d255a"
+  integrity sha512-UpYjBi8xefVChsCoBpKShdxTllC9pwISirfoZsUa2AAdQg/Jd2KQGtSbw+ya7GPo7x/wAPlH6JBhKhAsXUEZNA==
+  dependencies:
+    "@types/istanbul-lib-coverage" "*"
+    "@types/istanbul-lib-report" "*"
+
 "@types/jest@^21.1.6":
   version "21.1.10"
   resolved "https://registry.yarnpkg.com/@types/jest/-/jest-21.1.10.tgz#dcacb5217ddf997a090cc822bba219b4b2fd7984"
@@ -254,11 +283,23 @@
   dependencies:
     "@types/node" "*"
 
+"@types/yargs-parser@*":
+  version "13.1.0"
+  resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-13.1.0.tgz#c563aa192f39350a1d18da36c5a8da382bbd8228"
+  integrity sha512-gCubfBUZ6KxzoibJ+SCUc/57Ms1jz5NjHe4+dI2krNmU5zCPAphyLJYyTOg06ueIyfj+SaCUqmzun7ImlxDcKg==
+
 "@types/yargs@^11.1.3":
   version "11.1.3"
   resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-11.1.3.tgz#33c8ebf05f78f1edeb249c1cde1a42ae57f5664e"
   integrity sha512-moBUF6X8JsK5MbLZGP3vCfG/TVHZHsaePj3EimlLKp8+ESUjGjpXalxyn90a2L9fTM2ZGtW4swb6Am1DvVRNGA==
 
+"@types/yargs@^13.0.0":
+  version "13.0.3"
+  resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-13.0.3.tgz#76482af3981d4412d65371a318f992d33464a380"
+  integrity sha512-K8/LfZq2duW33XW/tFwEAfnZlqIfVsoyRB3kfXdPXYhl0nfM8mmh7GS0jg7WrX2Dgq/0Ha/pR1PaR+BvmWwjiQ==
+  dependencies:
+    "@types/yargs-parser" "*"
+
 JSONStream@^1.0.4, JSONStream@^1.3.4, JSONStream@^1.3.5:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0"
@@ -3106,6 +3147,15 @@ jest-changed-files@^23.4.2:
   dependencies:
     throat "^4.0.0"
 
+jest-changed-files@^24.9.0:
+  version "24.9.0"
+  resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-24.9.0.tgz#08d8c15eb79a7fa3fc98269bc14b451ee82f8039"
+  integrity sha512-6aTWpe2mHF0DhL28WjdkO8LyGjs3zItPET4bMSeXU6T3ub4FPMw+mcOcbdGXQOAfmLcxofD23/5Bl9Z4AkFwqg==
+  dependencies:
+    "@jest/types" "^24.9.0"
+    execa "^1.0.0"
+    throat "^4.0.0"
+
 jest-cli@^23.6.0:
   version "23.6.0"
   resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-23.6.0.tgz#61ab917744338f443ef2baa282ddffdd658a5da4"