Print failures for pending/only forbidden (#2874)

lib/runner.js

@@ -429,6 +429,10 @@ Runner.prototype.runTest = function (fn) {
   if (!test) {
     return;
   }
+  if (this.forbidOnly && this.hasOnly) {
+    fn(new Error('`.only` forbidden'));
+    return;
+  }
   if (this.asyncOnly) {
     test.asyncOnly = true;
   }
@@ -529,7 +533,13 @@ Runner.prototype.runTests = function (suite, fn) {
     }
 
     if (test.isPending()) {
-      self.emit('pending', test);
+      if (self.forbidPending) {
+        test.isPending = alwaysFalse;
+        self.fail(test, new Error('Pending test forbidden'));
+        delete test.isPending;
+      } else {
+        self.emit('pending', test);
+      }
       self.emit('test end', test);
       return next();
     }
@@ -538,7 +548,13 @@ Runner.prototype.runTests = function (suite, fn) {
     self.emit('test', self.test = test);
     self.hookDown('beforeEach', function (err, errSuite) {
       if (test.isPending()) {
-        self.emit('pending', test);
+        if (self.forbidPending) {
+          test.isPending = alwaysFalse;
+          self.fail(test, new Error('Pending test forbidden'));
+          delete test.isPending;
+        } else {
+          self.emit('pending', test);
+        }
         self.emit('test end', test);
         return next();
       }
@@ -550,7 +566,9 @@ Runner.prototype.runTests = function (suite, fn) {
         test = self.test;
         if (err) {
           var retry = test.currentRetry();
-          if (err instanceof Pending) {
+          if (err instanceof Pending && self.forbidPending) {
+            self.fail(test, new Error('Pending test forbidden'));
+          } else if (err instanceof Pending) {
             test.pending = true;
             self.emit('pending', test);
           } else if (retry < test.retries()) {
@@ -586,6 +604,10 @@ Runner.prototype.runTests = function (suite, fn) {
   next();
 };
 
+function alwaysFalse () {
+  return false;
+}
+
 /**
  * Run the given `suite` and invoke the callback `fn()` when complete.
  *
@@ -820,12 +842,6 @@ Runner.prototype.run = function (fn) {
 
   // callback
   this.on('end', function () {
-    if (self.forbidOnly && self.hasOnly) {
-      self.failures += self.stats.tests;
-    }
-    if (self.forbidPending) {
-      self.failures += self.stats.pending;
-    }
     debug('end');
     process.removeListener('uncaughtException', uncaught);
     fn(self.failures);

test/integration/fixtures/options/forbid-only/only-suite.js

@@ -0,0 +1,5 @@
+'use strict';
+
+describe.only('forbid only - suite marked with only', function () {
+  it('test1', function () {});
+});

test/integration/fixtures/options/forbid-pending/before-this.skip.js

@@ -0,0 +1,6 @@
+'use strict';
+
+describe('forbid pending - before calls `skip()`', function () {
+  it('test', function () {});
+  before(function () { this.skip(); });
+});

test/integration/fixtures/options/forbid-pending/beforeEach-this.skip.js

@@ -0,0 +1,6 @@
+'use strict';
+
+describe('forbid pending - beforeEach calls `skip()`', function () {
+  it('test', function () {});
+  beforeEach(function () { this.skip(); });
+});

test/integration/fixtures/options/forbid-pending/skip-suite.js

@@ -0,0 +1,5 @@
+'use strict';
+
+describe.skip('forbid pending - suite marked with skip', function () {
+  it('test1', function () {});
+});

test/integration/fixtures/options/forbid-pending/this.skip.js

@@ -0,0 +1,7 @@
+'use strict';
+
+describe('forbid pending - test calls `skip()`', function () {
+  it('test1', function () {});
+  it('test2', function () { this.skip(); });
+  it('test3', function () {});
+});

test/integration/options.spec.js

@@ -215,6 +215,8 @@ describe('options', function () {
   });
 
   describe('--forbid-only', function () {
+    var onlyErrorMessage = '`.only` forbidden';
+
     before(function () {
       args = ['--forbid-only'];
     });
@@ -237,12 +239,24 @@ describe('options', function () {
           return;
         }
         assert.equal(res.code, 1);
+        assert.equal(res.failures[0].err.message, onlyErrorMessage);
+        done();
+      });
+    });
+
+    it('fails if there are tests in suites marked only', function (done) {
+      run('options/forbid-only/only-suite.js', args, function (err, res) {
+        assert(!err);
+        assert.equal(res.code, 1);
+        assert.equal(res.failures[0].err.message, onlyErrorMessage);
         done();
       });
     });
   });
 
   describe('--forbid-pending', function () {
+    var pendingErrorMessage = 'Pending test forbidden';
+
     before(function () {
       args = ['--forbid-pending'];
     });
@@ -265,6 +279,7 @@ describe('options', function () {
           return;
         }
         assert.equal(res.code, 1);
+        assert.equal(res.failures[0].err.message, pendingErrorMessage);
         done();
       });
     });
@@ -276,6 +291,43 @@ describe('options', function () {
           return;
         }
         assert.equal(res.code, 1);
+        assert.equal(res.failures[0].err.message, pendingErrorMessage);
+        done();
+      });
+    });
+
+    it('fails if tests call `skip()`', function (done) {
+      run('options/forbid-pending/this.skip.js', args, function (err, res) {
+        assert(!err);
+        assert.equal(res.code, 1);
+        assert.equal(res.failures[0].err.message, pendingErrorMessage);
+        done();
+      });
+    });
+
+    it('fails if beforeEach calls `skip()`', function (done) {
+      run('options/forbid-pending/beforeEach-this.skip.js', args, function (err, res) {
+        assert(!err);
+        assert.equal(res.code, 1);
+        assert.equal(res.failures[0].err.message, pendingErrorMessage);
+        done();
+      });
+    });
+
+    it('fails if before calls `skip()`', function (done) {
+      run('options/forbid-pending/before-this.skip.js', args, function (err, res) {
+        assert(!err);
+        assert.equal(res.code, 1);
+        assert.equal(res.failures[0].err.message, pendingErrorMessage);
+        done();
+      });
+    });
+
+    it('fails if there are tests in suites marked skip', function (done) {
+      run('options/forbid-pending/skip-suite.js', args, function (err, res) {
+        assert(!err);
+        assert.equal(res.code, 1);
+        assert.equal(res.failures[0].err.message, pendingErrorMessage);
         done();
       });
     });