diff --git a/lib/cli/collect-files.js b/lib/cli/collect-files.js index cc04559443..52e3e64db6 100644 --- a/lib/cli/collect-files.js +++ b/lib/cli/collect-files.js @@ -28,7 +28,8 @@ module.exports = ({ file: fileArgs, recursive, sort, - spec + spec, + shard } = {}) => { const unmatched = []; const specFiles = spec.reduce((specFiles, arg) => { @@ -79,6 +80,15 @@ module.exports = ({ }); } + // Filter out files that don't match the shard + if (shard && shard.totalShards > 1) { + const desiredShardIndex = shard.desiredShard - 1; + return files.filter( + (filename, fileNumber) => + fileNumber % shard.totalShards === desiredShardIndex + ); + } + return files; }; @@ -92,4 +102,7 @@ module.exports = ({ * @property {string[]} file - List of additional files to include * @property {boolean} recursive - Find files recursively * @property {boolean} sort - Sort test files + * @property {Object} shard - Shard configuration + * @property {number} shard.desiredShard - Shard to run + * @property {number} shard.totalShards - Total shards */ diff --git a/lib/cli/run-helpers.js b/lib/cli/run-helpers.js index 078ca7e434..151a5ceb9a 100644 --- a/lib/cli/run-helpers.js +++ b/lib/cli/run-helpers.js @@ -15,6 +15,7 @@ const collectFiles = require('./collect-files'); const {format} = require('util'); const {createInvalidLegacyPluginError} = require('../errors'); const {requireOrImport} = require('../nodejs/esm-utils'); +const {parseShardString} = require('../utils'); const PluginLoader = require('../plugin-loader'); /** @@ -177,7 +178,9 @@ exports.runMocha = async (mocha, options) => { file, recursive, sort, - spec + spec, + // Use the processed --shard argument, not what is directly passed in through the argv. + shard: mocha.options.shard }; let run; @@ -190,6 +193,20 @@ exports.runMocha = async (mocha, options) => { return run(mocha, options, fileCollectParams); }; +/** + * Returns true if the given shard string is valid. Also returns true if no + * shard string is given at all. + * @param shardString + * @returns {boolean} + */ +exports.validateShardString = shardString => { + if (!shardString) { + return true; + } + + return Boolean(parseShardString(shardString)); +}; + /** * Used for `--reporter` and `--ui`. Ensures there's only one, and asserts that * it actually exists. This must be run _after_ requires are processed (see diff --git a/lib/cli/run-option-metadata.js b/lib/cli/run-option-metadata.js index 492608fbdd..8f5695fa2b 100644 --- a/lib/cli/run-option-metadata.js +++ b/lib/cli/run-option-metadata.js @@ -57,6 +57,7 @@ const TYPES = (exports.types = { 'package', 'reporter', 'ui', + 'shard', 'slow', 'timeout' ] diff --git a/lib/cli/run.js b/lib/cli/run.js index fbbe510e94..c7e677e3e3 100644 --- a/lib/cli/run.js +++ b/lib/cli/run.js @@ -20,6 +20,7 @@ const { list, handleRequires, validateLegacyPlugin, + validateShardString, runMocha } = require('./run-helpers'); const {ONE_AND_DONES, ONE_AND_DONE_ARGS} = require('./one-and-dones'); @@ -231,6 +232,11 @@ exports.builder = yargs => description: 'Retry failed tests this many times', group: GROUPS.RULES }, + shard: { + description: + 'Shard tests and execute only the desired shard, specify in the form "desired/total", starting with 1, for example "2/3".', + group: GROUPS.RULES + }, slow: { default: defaults.slow, description: 'Specify "slow" test threshold (in milliseconds)', @@ -293,6 +299,12 @@ exports.builder = yargs => } if (argv.parallel) { + if (argv.shard) { + throw createUnsupportedError( + '--parallel runs test files in a non-deterministic order, and is mutually exclusive with --shard' + ); + } + // yargs.conflicts() can't deal with `--file foo.js --no-parallel`, either if (argv.file) { throw createUnsupportedError( @@ -349,6 +361,7 @@ exports.builder = yargs => const plugins = await handleRequires(argv.require); validateLegacyPlugin(argv, 'reporter', Mocha.reporters); validateLegacyPlugin(argv, 'ui', Mocha.interfaces); + validateShardString(argv.shard); Object.assign(argv, plugins); } catch (err) { // this could be a bad --require, bad reporter, ui, etc. diff --git a/lib/errors.js b/lib/errors.js index bcc7291c99..97ace0c770 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -102,6 +102,13 @@ var constants = { */ INVALID_REPORTER: 'ERR_MOCHA_INVALID_REPORTER', + /** + * A a shard value is invalid + * @constant + * @default + */ + INVALID_SHARD: 'ERR_MOCHA_INVALID_SHARD', + /** * `done()` was called twice in a `Test` or `Hook` callback * @constant @@ -210,6 +217,21 @@ function createInvalidReporterError(message, reporter) { return err; } +/** + * Creates an error object to be thrown when the shard string is not properly formatted. + * + * @public + * @param {string} message - Error message to be displayed. + * @param {string} shardString - User-specified shard string. + * @returns {Error} instance detailing the error condition + */ +function createInvalidShardError(message, shardString) { + var err = new TypeError(message); + err.code = constants.INVALID_SHARD; + err.shardString = shardString; + return err; +} + /** * Creates an error object to be thrown when the interface specified in the options was not found. * @@ -540,6 +562,7 @@ module.exports = { createInvalidPluginError, createInvalidPluginImplementationError, createInvalidReporterError, + createInvalidShardError, createMissingArgumentError, createMochaInstanceAlreadyDisposedError, createMochaInstanceAlreadyRunningError, diff --git a/lib/mocha.js b/lib/mocha.js index f93865df7e..339d6ffd29 100644 --- a/lib/mocha.js +++ b/lib/mocha.js @@ -21,6 +21,7 @@ const { createMochaInstanceAlreadyRunningError, createUnsupportedError } = require('./errors'); +const {parseShardString} = require('./utils'); const {EVENT_FILE_PRE_REQUIRE, EVENT_FILE_POST_REQUIRE, EVENT_FILE_REQUIRE} = Suite.constants; var debug = require('debug')('mocha:mocha'); @@ -171,6 +172,8 @@ exports.run = function (...args) { * @param {Object} [options.reporterOption] - Reporter settings object. * @param {number} [options.retries] - Number of times to retry failed tests. * @param {number} [options.slow] - Slow threshold value. + * @param {number} [options.slow] - Slow threshold value. + * @param {string} [options.shard] - The shard config represented as a string "desired"/"total". * @param {number|string} [options.timeout] - Timeout threshold value. * @param {string} [options.ui] - Interface name. * @param {boolean} [options.parallel] - Run jobs in parallel. @@ -196,7 +199,8 @@ function Mocha(options = {}) { options.reporterOption || options.reporterOptions // for backwards compatibility ) .slow(options.slow) - .global(options.global); + .global(options.global) + .shard(options.shard); // this guard exists because Suite#timeout does not consider `undefined` to be valid input if (typeof options.timeout !== 'undefined') { @@ -785,6 +789,28 @@ Mocha.prototype.slow = function (msecs) { return this; }; +/** + * Converts and validates the shard string into a shard object. + * + * @public + * @see [CLI option](../#-shard) + * @param {string} shardString - The shard config represented as a string "desired"/"total". + * @return {Mocha} this + * @chainable + */ +Mocha.prototype.shard = function (shardString) { + if (!shardString) { + return this; + } + + const [desiredShard, totalShards] = parseShardString(shardString); + this.options.shard = { + desiredShard, + totalShards + }; + return this; +}; + /** * Forces all tests to either accept a `done` callback or return a promise. * diff --git a/lib/runner.js b/lib/runner.js index 12807725fb..bddcf286e3 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -1053,6 +1053,7 @@ Runner.prototype.run = function (fn, opts = {}) { rootSuite.filterOnly(); debug('run(): filtered exclusive Runnables'); } + this.state = constants.STATE_RUNNING; if (this._opts.delay) { this.emit(constants.EVENT_DELAY_END); diff --git a/lib/utils.js b/lib/utils.js index fc7271d019..462776b460 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -11,6 +11,7 @@ var path = require('path'); var util = require('util'); var he = require('he'); +const {createInvalidShardError} = require('./errors'); const MOCHA_ID_PROP_NAME = '__mocha_id__'; @@ -647,3 +648,24 @@ exports.assignNewMochaID = obj => { */ exports.getMochaID = obj => obj && typeof obj === 'object' ? obj[MOCHA_ID_PROP_NAME] : undefined; + +/** + * Returns the desired shard and total shards as numbers given a shard string + * @param shardString + * @returns {[number,number]} + */ +exports.parseShardString = shardString => { + const shardParts = shardString.split('/'); + const desiredShard = parseInt(shardParts[0]); + const totalShards = parseInt(shardParts[1]); + if (isNaN(desiredShard) || isNaN(totalShards)) { + throw createInvalidShardError('Invalid shard values.', shardString); + } + if (desiredShard <= 0 || totalShards < desiredShard) { + throw createInvalidShardError( + 'Desired shard must be greater than 0 and less than total shards.', + shardString + ); + } + return [desiredShard, totalShards]; +}; diff --git a/test/assertions.js b/test/assertions.js index b6ed7b9cc9..24211bd1c6 100644 --- a/test/assertions.js +++ b/test/assertions.js @@ -146,6 +146,12 @@ module.exports = { expect(result.stats.tests, '[not] to be', count); } ) + .addAssertion( + ' [not] to have suite count ', + (expect, result, count) => { + expect(result.stats.suites, '[not] to be', count); + } + ) .addAssertion( ' [not] to have failed [test] count ', (expect, result, count) => { diff --git a/test/integration/fixtures/options/shard-file-alpha.fixture.js b/test/integration/fixtures/options/shard-file-alpha.fixture.js new file mode 100644 index 0000000000..4698bc09c2 --- /dev/null +++ b/test/integration/fixtures/options/shard-file-alpha.fixture.js @@ -0,0 +1,5 @@ +'use strict'; + +describe('suite 1', function () { + it('test 1', function () {}); +}); diff --git a/test/integration/fixtures/options/shard-file-beta.fixture.js b/test/integration/fixtures/options/shard-file-beta.fixture.js new file mode 100644 index 0000000000..775a4fdcbe --- /dev/null +++ b/test/integration/fixtures/options/shard-file-beta.fixture.js @@ -0,0 +1,17 @@ +'use strict'; + +describe('suite 2', function () { + it('test 1', function () {}); +}); + +describe('suite 3', function () { + it('test 1', function () {}); +}); + +describe('suite 4', function () { + it('test 1', function () {}); +}); + +describe('suite 5', function () { + it('test 1', function () {}); +}); diff --git a/test/integration/fixtures/options/shard-file-theta.fixture.js b/test/integration/fixtures/options/shard-file-theta.fixture.js new file mode 100644 index 0000000000..d8307d2548 --- /dev/null +++ b/test/integration/fixtures/options/shard-file-theta.fixture.js @@ -0,0 +1,21 @@ +'use strict'; + +describe('suite 6', function () { + it('test 1', function () {}); +}); + +describe('suite 7', function () { + it('test 1', function () {}); +}); + +describe('suite 8', function () { + it('test 1', function () {}); +}); + +describe('suite 9', function () { + it('test 1', function () {}); +}); + +describe('suite 10', function () { + it('test 1', function () {}); +}); diff --git a/test/integration/options/shard.spec.js b/test/integration/options/shard.spec.js new file mode 100644 index 0000000000..97efaca445 --- /dev/null +++ b/test/integration/options/shard.spec.js @@ -0,0 +1,109 @@ +'use strict'; + +var helpers = require('../helpers'); +const {posix: path} = require('path'); +var runMocha = helpers.runMocha; +var runMochaJSON = helpers.runMochaJSON; +var resolvePath = helpers.resolveFixturePath; + +describe('--shard', function () { + var fixtures = { + alpha: { + name: 'alpha', + suiteCount: 1, + path: path.join('options', 'shard-file-alpha') + }, + beta: { + name: 'beta', + suiteCount: 4, + path: path.join('options', 'shard-file-beta') + }, + theta: { + name: 'theta', + suiteCount: 5, + path: path.join('options', 'shard-file-theta') + } + }; + + const combinations = [ + // Each combination with distinct files + [fixtures.alpha, fixtures.beta, fixtures.theta], + [fixtures.alpha, fixtures.theta, fixtures.beta], + [fixtures.beta, fixtures.alpha, fixtures.theta], + [fixtures.beta, fixtures.theta, fixtures.alpha], + [fixtures.theta, fixtures.alpha, fixtures.beta], + [fixtures.theta, fixtures.beta, fixtures.alpha] + ]; + + const shards = ['1/2', '2/2']; + + for (const [fixture1, fixture2, fixture3] of combinations) { + for (const shard of shards) { + const testName = `should run specs for combination of ${fixture1.name}, ${fixture2.name}, and ${fixture3.name} on shard ${shard}`; + let expectedCount; + + if (shard === shards[0]) { + expectedCount = fixture1.suiteCount + fixture3.suiteCount; + } else { + expectedCount = fixture2.suiteCount; + } + + it(testName, function (done) { + const args = [ + '--file', + resolvePath(fixture1.path), + '--file', + resolvePath(fixture2.path), + '--shard', + shard + ]; + // Test that come in through --file are always run first + runMochaJSON(fixture3.path, args, function (err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed') + .and('to have suite count', expectedCount) + .and('not to have pending tests'); + done(); + }); + }); + } + } + + it('should fail if the parameter has non numeric value', function (done) { + var spawnOpts = {stdio: 'pipe'}; + runMocha( + fixtures.alpha.path, + ['--shard', '1/a'], + function (err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with output', /Invalid shard values/); + done(); + }, + spawnOpts + ); + }); + + it('should fail if the parameter has invalid numeric value', function (done) { + var spawnOpts = {stdio: 'pipe'}; + runMocha( + fixtures.alpha.path, + ['--shard', '2/1'], + function (err, res) { + if (err) { + return done(err); + } + expect( + res, + 'to have failed with output', + /Desired shard must be greater than 0 and less than total shards./ + ); + done(); + }, + spawnOpts + ); + }); +});