HTML Reporter: Add support for displaying early errors

qunitjs · Jul 23, 2024 · 722acf2 · 722acf2
1 parent 05e15ba
commit 722acf2
Show file tree

Hide file tree

Showing 6 changed files with 225 additions and 21 deletions.
diff --git a/demos/qunit-onerror-early.html b/demos/qunit-onerror-early.html
@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<meta charset="UTF-8">
+	<title>window-onerror-early</title>
+  <link rel="stylesheet" href="../src/core/qunit.css">
+  <script src="../qunit/qunit.js"></script>
+  <script>
+    QUnit.begin(function () {
+      // eslint-disable-next-line no-undef
+      beginBoom();
+    });
+
+    // eslint-disable-next-line no-undef
+    outerBoom();
+  </script>
+  <script>
+    QUnit.test('example', function (assert) {
+      assert.true(true);
+    });
+  </script>
+</head>
+<body>
+	<div id="qunit"></div>
+</body>
+</html>
diff --git a/docs/api/callbacks/QUnit.on.md b/docs/api/callbacks/QUnit.on.md
@@ -128,7 +128,7 @@ The `runEnd` event indicates the end of a test run. It is emitted exactly once.
 
 <p class="note" markdown="1">
 
-Unlike other events, the `runEnd` event has **memory** (since QUnit 3.0). This means listening for the event is possible, even if the event already fired. For example, if you build an integration system that automates running tests in a browser, and are unable to reliably inject a listener before tests have finished executing. You can attach a late event listeners for the `runEnd` event. These will be invoked immediately in that case. This removes the need for HTML scraping.
+The `runEnd` event has **memory** (since QUnit 3.0). This means listening for this event is possible, even if the event already fired. For example, if you build an integration system that automates running tests in a browser, and are unable to reliably inject a listener before tests have finished executing. You can attach a late event listeners for the `runEnd` event. These will be invoked immediately in that case. This removes the need for HTML scraping.
 
 </p>
 
@@ -159,6 +159,12 @@ The `error` event notifies plugins of uncaught global errors during a test run.
 
 See also [QUnit.onUncaughtException()](../extension/QUnit.onUncaughtException.md) which is where you can report your own uncaught errors.
 
+<p class="note" markdown="1">
+
+The `error` event has **memory** (since QUnit 3.0). This means listening for this event is possible, even if the event already fired. This improves reliability of reporters in browser automations, where it might be difficult to reliably inject a listener between qunit.js and anything else.
+
+</p>
+
 | `Error|any` | `error`
 
 ```js

diff --git a/src/core/callbacks.js b/src/core/callbacks.js
@@ -1,4 +1,5 @@
 import config from './config.js';
+import { prioritySymbol } from './events.js';
 import Promise from './promise.js';
 
 export function createRegisterCallbackFunction (key) {
@@ -7,11 +8,19 @@ export function createRegisterCallbackFunction (key) {
     config.callbacks[key] = [];
   }
 
-  return function registerCallback (callback) {
+  return function registerCallback (callback, priority = null) {
     if (typeof callback !== 'function') {
       throw new TypeError('Callback parameter must be a function');
     }
-    config.callbacks[key].push(callback);
+    /* istanbul ignore if: internal argument */
+    if (priority && priority !== prioritySymbol) {
+      throw new TypeError('invalid priority parameter');
+    }
+    if (priority === prioritySymbol) {
+      config.callbacks[key].unshift(callback);
+    } else {
+      config.callbacks[key].push(callback);
+    }
   };
 }
 

diff --git a/src/core/events.js b/src/core/events.js
@@ -12,6 +12,7 @@ const SUPPORTED_EVENTS = [
   'runEnd'
 ];
 const MEMORY_EVENTS = [
+  'error',
   'runEnd'
 ];
 
@@ -41,7 +42,7 @@ export function emit (eventName, data) {
     callbacks[i](data);
   }
 
-  if (inArray(MEMORY_EVENTS, eventName)) {
+  if (inArray(eventName, MEMORY_EVENTS)) {
     config._event_memory[eventName] = data;
   }
 }
@@ -69,6 +70,7 @@ export function on (eventName, callback, priority = null) {
   if (typeof callback !== 'function') {
     throw new TypeError('callback must be a function when registering a listener');
   }
+  /* istanbul ignore if: internal argument */
   if (priority && priority !== prioritySymbol) {
     throw new TypeError('invalid priority parameter');
   }

diff --git a/src/core/reporters/HtmlReporter.js b/src/core/reporters/HtmlReporter.js
@@ -187,18 +187,25 @@ export default class HtmlReporter {
     });
     this.dropdownData = null;
 
+    // We must not fallback to creating `<div id="qunit">` ourselves if it
+    // does not exist, because not having id="qunit" is how projects indicate
+    // that they wish to run QUnit headless, with their own reporters.
     this.element = options.element || undefined;
     this.elementBanner = null;
     this.elementDisplay = null;
     this.elementTests = null;
-    // TODO: Consider rendering the UI early when possible for improved UX.
 
-    // NOTE: Only listen for "error" and "runStart" now.
+    // NOTE: Only listen for "runStart" now.
     // Other event handlers are added via listen() from onRunStart,
     // after we know that the element exists. This reduces overhead and avoids
     // potential internal errors when the HTML Reporter is disabled.
     this.listen = function () {
       this.listen = null;
+      // Use prioritySignal for begin() to ensure the UI shows up
+      // reliably to render errors from onError.
+      // Without this, user-defined "QUnit.begin()" callbacks will end
+      // up in the queue before ours, and if those throw an error,
+      // then this handler will never run, thus leaving the page blank.
       QUnit.begin(this.onBegin.bind(this), prioritySymbol);
       // Use prioritySignal for testStart() to increase availability
       // of the HTML API for TESTID elements toward other event listeners.
@@ -207,7 +214,10 @@ export default class HtmlReporter {
       QUnit.testDone(this.onTestDone.bind(this));
       QUnit.on('runEnd', this.onRunEnd.bind(this));
     };
-    QUnit.on('error', this.onError.bind(this), prioritySymbol);
+    this.listenError = function () {
+      this.listenError = null;
+      QUnit.on('error', this.onError.bind(this), prioritySymbol);
+    };
     QUnit.on('runStart', this.onRunStart.bind(this), prioritySymbol);
   }
 
@@ -712,6 +722,16 @@ export default class HtmlReporter {
   onBegin (beginDetails) {
     this.appendInterface(beginDetails);
     this.elementDisplay.className = 'running';
+
+    // TODO: We render the UI late here from onBegin because the toolbar and module
+    // dropdown rely on user-defined information. If we refactor the UI to render
+    // most of it earlier (from the constructor if possible, with last retry during
+    // runStart), then we can move this listener to the listen() function. We separate it
+    // so that onError can safely call appendTest() and rely on `this.element` and
+    // `this.elementTests` being set. We could instead add the "error" listener in
+    // the constructor, and buffer "early error" event inside this class until the
+    // UI is ready, but we instead rely on QUnit.on's built-in memory for "error".
+    this.listenError();
   }
 
   getRerunFailedHtml (failedTests) {
@@ -999,15 +1019,7 @@ export default class HtmlReporter {
   }
 
   onError (error) {
-    const testItem = this.elementTests && this.appendTest('global failure');
-    if (!testItem) {
-      // HTML Reporter is probably disabled or not yet initialized.
-      // This kind of early error will be visible in the browser console
-      // and via window.onerror, but we can't show it in the UI.
-      //
-      // TODO: Consider stashing early error here and replay in UI during onRunStart.
-      return;
-    }
+    const testItem = this.appendTest('global failure');
 
     // Render similar to a failed assertion (see above QUnit.log callback)
     let message = escapeText(errorString(error));
@@ -1025,7 +1037,8 @@ export default class HtmlReporter {
     assertList.appendChild(assertLi);
 
     // Make it visible
-    testItem.className = 'fail';
+    DOM.removeClass(testItem, 'running');
+    DOM.addClass(testItem, 'fail');
   }
 }
 

diff --git a/test/main/HtmlReporter.js b/test/main/HtmlReporter.js
@@ -38,18 +38,25 @@ QUnit.module.if('HtmlReporter', typeof document !== 'undefined', {
         this.emit('runStart', { testCounts: { total: 0 } });
         this.emit('begin', { modules: [] });
       },
-      // The first 1.5 test.
-      _do_mixed_run_half: function () {
+      // The first 1 test.
+      _do_start_with_one: function () {
         this.emit('runStart', { testCounts: { total: 4 } });
         this.emit('begin', { modules: [] });
 
+        this._do_one_test();
+      },
+      _do_one_test: function () {
         this.emit('testStart', { testId: '00A', name: 'A' });
         this.emit('log', {
           testId: '00A',
           result: true,
           runtime: 0
         });
         this.emit('testDone', { testId: '00A', name: 'A', total: 1, passed: 1, failed: 0, runtime: 0 });
+      },
+      // The first 1.5 test.
+      _do_mixed_run_half: function () {
+        this._do_start_with_one();
 
         this.emit('testStart', { testId: '00B', name: 'B' });
         this.emit('log', {
@@ -92,6 +99,11 @@ QUnit.module.if('HtmlReporter', typeof document !== 'undefined', {
         });
       }
     };
+  },
+  afterEach: function () {
+    if (this.restoreQUnitElement) {
+      this.restoreQUnitElement.id = 'qunit';
+    }
   }
 });
 
@@ -223,6 +235,75 @@ QUnit.test('test-output [trace]', function (assert) {
   );
 });
 
+QUnit.test('onError [early]', function (assert) {
+  var element = document.createElement('div');
+  new QUnit.reporters.html(this.MockQUnit, {
+    element: element,
+    config: {
+      urlConfig: []
+    }
+  });
+  var err = new Error('boo');
+  err.stack = '[email protected]\[email protected]\[email protected]';
+
+  this.MockQUnit._do_start_empty();
+  this.MockQUnit.emit('error', err);
+  this.MockQUnit._do_one_test();
+
+  var testItem = element.querySelector('#qunit-test-output-00A');
+  assert.strictEqual(
+    testItem.textContent,
+    'A (1)' + 'Rerun' + '0 ms' +
+      'okay' + '@ 0 ms',
+    'test item (normal)'
+  );
+
+  // first child
+  var errorItem = element.querySelector('#qunit-tests > li:first-child');
+  assert.strictEqual(errorItem.id, '', 'error item, ID');
+  assert.strictEqual(
+    errorItem.textContent,
+    'global failure' +
+      'Error: boo' +
+      'Source: [email protected]\[email protected]\[email protected]',
+    'error item, text'
+  );
+});
+
+QUnit.test('onError [mid-run]', function (assert) {
+  var element = document.createElement('div');
+  new QUnit.reporters.html(this.MockQUnit, {
+    element: element,
+    config: {
+      urlConfig: []
+    }
+  });
+  var err = new Error('boo');
+  err.stack = '[email protected]\[email protected]\[email protected]';
+
+  this.MockQUnit._do_start_with_one();
+  this.MockQUnit.emit('error', err);
+
+  var testItem = element.querySelector('#qunit-test-output-00A');
+  assert.strictEqual(
+    testItem.textContent,
+    'A (1)' + 'Rerun' + '0 ms' +
+      'okay' + '@ 0 ms',
+    'test item (normal)'
+  );
+
+  // last child
+  var errorItem = element.querySelector('#qunit-tests > li:last-child');
+  assert.strictEqual(errorItem.id, '', 'error item, ID');
+  assert.strictEqual(
+    errorItem.textContent,
+    'global failure' +
+      'Error: boo' +
+      'Source: [email protected]\[email protected]\[email protected]',
+    'error item, text'
+  );
+});
+
 QUnit.test('appendFilteredTest()', function (assert) {
   var element = document.createElement('div');
   new QUnit.reporters.html(this.MockQUnit, {
@@ -322,7 +403,7 @@ QUnit.test('options [urlConfig]', function (assert) {
   ));
 });
 
-QUnit.test('disable [via options.element=null]', function (assert) {
+QUnit.test('listen [disable via options.element=null]', function (assert) {
   this.MockQUnit.on = function (type) {
     assert.step('listen on-' + type);
   };
@@ -336,7 +417,74 @@ QUnit.test('disable [via options.element=null]', function (assert) {
   });
   this.MockQUnit._do_mixed_run_full();
 
-  assert.verifySteps([], 'zero listeners when disabled');
+  assert.verifySteps([], 'listeners');
+});
+
+QUnit.test('listen [disable via no qunit element]', function (assert) {
+  // Temporarily hide the global #qunit element
+  var globalElement = document.querySelector('#qunit');
+  if (globalElement) {
+    globalElement.id = 'not-qunit';
+    this.restoreQUnitElement = globalElement;
+  }
+
+  this.MockQUnit.on = function (type) {
+    assert.step('listen on-' + type);
+  };
+  this.MockQUnit.emit = function () {};
+
+  new QUnit.reporters.html(this.MockQUnit, {
+    config: {
+      urlConfig: []
+    }
+  });
+  this.MockQUnit._do_mixed_run_full();
+
+  var newElement = document.querySelector('#qunit');
+  assert.strictEqual(newElement, null, 'no automatic element created');
+
+  assert.verifySteps(
+    [
+      'listen on-runStart'
+    ],
+    'listeners'
+  );
+});
+
+QUnit.test('listen [enable via options.element]', function (assert) {
+  this.MockQUnit.on = function (event, fn) {
+    assert.step('listen on-' + event);
+    this.on[event] = fn;
+  };
+
+  var element = document.createElement('div');
+  new QUnit.reporters.html(this.MockQUnit, {
+    element: element,
+    config: {
+      urlConfig: []
+    }
+  });
+
+  assert.verifySteps(
+    [
+      'listen on-runStart'
+    ],
+    'listeners after construction'
+  );
+
+  this.MockQUnit._do_start_empty();
+
+  assert.verifySteps(
+    [
+      'listen on-begin',
+      'listen on-testStart',
+      'listen on-log',
+      'listen on-testDone',
+      'listen on-runEnd',
+      'listen on-error'
+    ],
+    'listeners after start'
+  );
 });
 
 QUnit.test('module selector', function (assert) {