feat: add new option 'fail-zero' (#4716)

Author: juergba

docs/index.md

@@ -876,7 +876,7 @@ Use this option to have Mocha check for global variables that are leaked while r
 
 ### `--dry-run`
 
-> _New in v9.0.0._ Report tests without executing any of them, neither tests nor hooks.
+> _New in v9.0.0_ Report tests without executing any of them, neither tests nor hooks.
 
 ### `--exit`
 
@@ -899,6 +899,10 @@ To ensure your tests aren't leaving messes around, here are some ideas to get st
 - Try something like [wtfnode][npm-wtfnode]
 - Use [`.only`](#exclusive-tests) until you find the test that causes Mocha to hang
 
+### `--fail-zero`
+
+> _New in v9.1.0_ Fail test run if no tests are encountered with `exit-code: 1`.
+
 ### `--forbid-only`
 
 Enforce a rule that tests may not be exclusive (use of e.g., `describe.only()` or `it.only()` is disallowed).
@@ -1007,15 +1011,15 @@ Can be specified as a comma-delimited list.
 
 ### `--config <path>`
 
-> _New in v6.0.0._
+> _New in v6.0.0_
 
 Specify an explicit path to a [configuration file](#configuring-mocha-nodejs).
 
 By default, Mocha will search for a config file if `--config` is not specified; use `--no-config` to suppress this behavior.
 
 ### `--node-option <name>, -n <name>`
 
-> _New in v9.1.0._
+> _New in v9.1.0_
 
 For Node.js and V8 options. Mocha forwards these options to Node.js by spawning a new child-process.<br>
 The options are set without leading dashes `--`, e.g. `-n require=foo -n unhandled-rejections=strict`
@@ -1028,7 +1032,7 @@ Can also be specified as a comma-delimited list: `-n require=foo,unhandled-rejec
 
 ### `--package <path>`
 
-> _New in v6.0.0._
+> _New in v6.0.0_
 
 Specify an explicit path to a [`package.json` file](#configuring-mocha-nodejs) (ostensibly containing configuration in a `mocha` property).
 
@@ -1042,7 +1046,7 @@ Specifying `--extension` will _remove_ `.js` as a test file extension; use `--ex
 
 The option can be given multiple times. The option accepts a comma-delimited list: `--extension a,b` is equivalent to `--extension a --extension b`.
 
-> _New in v8.2.0._
+> _New in v8.2.0_
 
 `--extension` now supports multipart extensions (e.g., `spec.js`), leading dots (`.js`) and combinations thereof (`.spec.js`);
 
@@ -1217,7 +1221,7 @@ These flags vary depending on your version of Node.js.
 
 `node` flags can be defined in Mocha's [configuration](#configuring-mocha-nodejs).
 
-> _New in v9.1.0._ You can also pass `node` flags to Node.js using [`--node-option`](#-node-option-name-n-name).
+> _New in v9.1.0_ You can also pass `node` flags to Node.js using [`--node-option`](#-node-option-name-n-name).
 
 ### `--enable-source-maps`
 
@@ -1238,7 +1242,7 @@ Prepend `--v8-` to any flag listed in the output of `node --v8-options` (excludi
 
 V8 flags can be defined in Mocha's [configuration](#configuring-mocha-nodejs).
 
-> _New in v9.1.0._ You can also pass V8 flags (without `--v8-`) to Node.js using [`--node-option`](#-node-option-name-n-name).
+> _New in v9.1.0_ You can also pass V8 flags (without `--v8-`) to Node.js using [`--node-option`](#-node-option-name-n-name).
 
 ## Parallel Tests
 
@@ -1369,7 +1373,7 @@ It's unlikely (but not impossible) to see a performance gain from a [job count](
 
 ## Root Hook Plugins
 
-> _New in v8.0.0._
+> _New in v8.0.0_
 
 In some cases, you may want a [hook](#hooks) before (or after) every test in every file. These are called _root hooks_. Previous to v8.0.0, the way to accomplish this was to use `--file` combined with root hooks (see [example above](#root-hooks-are-not-global)). This still works in v8.0.0, but _not_ when running tests in parallel mode! For that reason, running root hooks using this method is _strongly discouraged_, and may be deprecated in the future.
 
@@ -1593,7 +1597,7 @@ If you're a library maintainer, and your library uses root hooks, you can migrat
 
 ## Global Fixtures
 
-> New in v8.2.0
+> _New in v8.2.0_
 
 At first glance, _global fixtures_ seem similar to [root hooks](#root-hook-plugins). However, unlike root hooks, global fixtures:
 
@@ -2121,6 +2125,7 @@ mocha.setup({
   bail: true,
   checkLeaks: true,
   dryRun: true,
+  failZero: true,
   forbidOnly: true,
   forbidPending: true,
   global: ['MyLib'],

example/config/.mocharc.js

@@ -15,6 +15,7 @@ module.exports = {
   diff: true,
   exit: false, // could be expressed as "'no-exit': true"
   extension: ['js', 'cjs', 'mjs'],
+  'fail-zero': true,
   fgrep: 'something', // fgrep and grep are mutually exclusive
   file: ['/path/to/some/file', '/path/to/some/other/file'],
   'forbid-only': false,

example/config/.mocharc.yml

@@ -8,6 +8,7 @@ delay: false
 diff: true
 exit: false # could be expressed as "no-exit: true"
 extension: ['js', 'cjs', 'mjs']
+fail-zero: true
 # fgrep and grep are mutually exclusive
 fgrep: 'something'
 file:

lib/cli/run-option-metadata.js

@@ -35,6 +35,7 @@ const TYPES = (exports.types = {
     'diff',
     'dry-run',
     'exit',
+    'fail-zero',
     'forbid-only',
     'forbid-pending',
     'full-trace',

lib/cli/run.js

@@ -98,6 +98,10 @@ exports.builder = yargs =>
         requiresArg: true,
         coerce: list
       },
+      'fail-zero': {
+        description: 'Fail test run if no test(s) encountered',
+        group: GROUPS.RULES
+      },
       fgrep: {
         conflicts: 'grep',
         description: 'Only run tests containing this string',

lib/mocha.js

@@ -164,6 +164,7 @@ exports.run = function(...args) {
  * @param {boolean} [options.delay] - Delay root suite execution?
  * @param {boolean} [options.diff] - Show diff on failure?
  * @param {boolean} [options.dryRun] - Report tests without running them?
+ * @param {boolean} [options.failZero] - Fail test run if zero tests?
  * @param {string} [options.fgrep] - Test filter given string.
  * @param {boolean} [options.forbidOnly] - Tests marked `only` fail the suite?
  * @param {boolean} [options.forbidPending] - Pending tests fail the suite?
@@ -223,6 +224,7 @@ function Mocha(options = {}) {
     'delay',
     'diff',
     'dryRun',
+    'failZero',
     'forbidOnly',
     'forbidPending',
     'fullTrace',
@@ -778,20 +780,6 @@ Mocha.prototype.diff = function(diff) {
   return this;
 };
 
-/**
- * Enables or disables running tests in dry-run mode.
- *
- * @public
- * @see [CLI option](../#-dry-run)
- * @param {boolean} [dryRun=true] - Whether to activate dry-run mode.
- * @return {Mocha} this
- * @chainable
- */
-Mocha.prototype.dryRun = function(dryRun) {
-  this.options.dryRun = dryRun !== false;
-  return this;
-};
-
 /**
  * @summary
  * Sets timeout threshold value.
@@ -918,6 +906,34 @@ Mocha.prototype.delay = function delay() {
   return this;
 };
 
+/**
+ * Enables or disables running tests in dry-run mode.
+ *
+ * @public
+ * @see [CLI option](../#-dry-run)
+ * @param {boolean} [dryRun=true] - Whether to activate dry-run mode.
+ * @return {Mocha} this
+ * @chainable
+ */
+Mocha.prototype.dryRun = function(dryRun) {
+  this.options.dryRun = dryRun !== false;
+  return this;
+};
+
+/**
+ * Fails test run if no tests encountered with exit-code 1.
+ *
+ * @public
+ * @see [CLI option](../#-fail-zero)
+ * @param {boolean} [failZero=true] - Whether to fail test run.
+ * @return {Mocha} this
+ * @chainable
+ */
+Mocha.prototype.failZero = function(failZero) {
+  this.options.failZero = failZero !== false;
+  return this;
+};
+
 /**
  * Causes tests marked `only` to fail the suite.
  *
@@ -1023,9 +1039,10 @@ Mocha.prototype.run = function(fn) {
   var options = this.options;
   options.files = this.files;
   const runner = new this._runnerClass(suite, {
+    cleanReferencesAfterRun: this._cleanReferencesAfterRun,
     delay: options.delay,
     dryRun: options.dryRun,
-    cleanReferencesAfterRun: this._cleanReferencesAfterRun
+    failZero: options.failZero
   });
   createStatsCollector(runner);
   var reporter = new this._reporter(runner, options);

lib/runner.js

@@ -135,10 +135,11 @@ class Runner extends EventEmitter {
    * @public
    * @class
    * @param {Suite} suite - Root suite
-   * @param {Object|boolean} [opts] - Options. If `boolean` (deprecated), whether or not to delay execution of root suite until ready.
+   * @param {Object|boolean} [opts] - Options. If `boolean` (deprecated), whether to delay execution of root suite until ready.
+   * @param {boolean} [opts.cleanReferencesAfterRun] - Whether to clean references to test fns and hooks when a suite is done.
    * @param {boolean} [opts.delay] - Whether to delay execution of root suite until ready.
    * @param {boolean} [opts.dryRun] - Whether to report tests without running them.
-   * @param {boolean} [opts.cleanReferencesAfterRun] - Whether to clean references to test fns and hooks when a suite is done.
+   * @param {boolean} [options.failZero] - Whether to fail test run if zero tests encountered.
    */
   constructor(suite, opts) {
     super();
@@ -1044,6 +1045,8 @@ Runner.prototype.run = function(fn, opts = {}) {
   fn = fn || function() {};
 
   const end = () => {
+    if (!this.total && this._opts.failZero) this.failures = 1;
+
     debug('run(): root suite completed; emitting %s', constants.EVENT_RUN_END);
     this.emit(constants.EVENT_RUN_END);
   };

test/integration/options/failZero.spec.js

@@ -0,0 +1,22 @@
+'use strict';
+
+var helpers = require('../helpers');
+var runMochaJSON = helpers.runMochaJSON;
+
+describe('--fail-zero', function() {
+  var args = ['--fail-zero', '--grep', 'yyyyyy'];
+
+  it('should fail since no tests are encountered', function(done) {
+    var fixture = '__default__.fixture.js';
+    runMochaJSON(fixture, args, function(err, res) {
+      if (err) {
+        return done(err);
+      }
+
+      expect(res, 'to have passed test count', 0)
+        .and('to have test count', 0)
+        .and('to have exit code', 1);
+      done();
+    });
+  });
+});

test/unit/mocha.spec.js

@@ -265,22 +265,18 @@ describe('Mocha', function() {
     });
 
     describe('bail()', function() {
-      describe('when provided no arguments', function() {
-        it('should set the "bail" flag on the root suite', function() {
-          mocha.bail();
-          expect(suite.bail, 'to have a call satisfying', [true]).and(
-            'was called once'
-          );
-        });
+      it('should set the "bail" flag on the root suite', function() {
+        mocha.bail();
+        expect(suite.bail, 'to have a call satisfying', [true]).and(
+          'was called once'
+        );
       });
 
-      describe('when provided a falsy argument', function() {
-        it('should unset the "bail" flag on the root suite', function() {
-          mocha.bail(false);
-          expect(suite.bail, 'to have a call satisfying', [false]).and(
-            'was called once'
-          );
-        });
+      it('should unset the "bail" flag on the root suite', function() {
+        mocha.bail(false);
+        expect(suite.bail, 'to have a call satisfying', [false]).and(
+          'was called once'
+        );
       });
 
       it('should be chainable', function() {
@@ -344,25 +340,9 @@ describe('Mocha', function() {
         expect(mocha.options, 'to have property', 'diff', true);
       });
 
-      describe('when provided `false` argument', function() {
-        it('should set the diff option to false', function() {
-          mocha.diff(false);
-          expect(mocha.options, 'to have property', 'diff', false);
-        });
-      });
-    });
-
-    describe('dryRun()', function() {
-      it('should set the dryRun option to true', function() {
-        mocha.dryRun();
-        expect(mocha.options, 'to have property', 'dryRun', true);
-      });
-
-      describe('when provided `false` argument', function() {
-        it('should set the dryRun option to false', function() {
-          mocha.dryRun(false);
-          expect(mocha.options, 'to have property', 'dryRun', false);
-        });
+      it('should set the diff option to false', function() {
+        mocha.diff(false);
+        expect(mocha.options, 'to have property', 'diff', false);
       });
     });
 
@@ -385,6 +365,30 @@ describe('Mocha', function() {
       });
     });
 
+    describe('dryRun()', function() {
+      it('should set the dryRun option to true', function() {
+        mocha.dryRun();
+        expect(mocha.options, 'to have property', 'dryRun', true);
+      });
+
+      it('should set the dryRun option to false', function() {
+        mocha.dryRun(false);
+        expect(mocha.options, 'to have property', 'dryRun', false);
+      });
+    });
+
+    describe('failZero()', function() {
+      it('should set the failZero option to true', function() {
+        mocha.failZero();
+        expect(mocha.options, 'to have property', 'failZero', true);
+      });
+
+      it('should set the failZero option to false', function() {
+        mocha.failZero(false);
+        expect(mocha.options, 'to have property', 'failZero', false);
+      });
+    });
+
     describe('forbidOnly()', function() {
       it('should set the forbidOnly option to true', function() {
         mocha.forbidOnly();