From 4847e211ac1b9c152512fcb4a63ca569f857eab2 Mon Sep 17 00:00:00 2001 From: Timo Tijhof Date: Tue, 9 Apr 2024 13:17:32 +0100 Subject: [PATCH] Tests: Split up fixture output files * Move tap output to txt files and move the commands to there as well. This way, the files are easier to discover from each other, and there is now only one place to edit stuff instead of updating things in three different places. * Improve concurrency algorithm. --- .gitattributes | 1 + test/cli/cli-main.js | 315 ++++-- test/cli/custom-reporter.js | 22 +- test/cli/fixtures/_load-default.tap.txt | 11 + test/cli/fixtures/_load-dir-file-glob.tap.txt | 14 + test/cli/fixtures/_load-dir.tap.txt | 11 + test/cli/fixtures/_load-filenotfound.tap.txt | 7 + test/cli/fixtures/_load-glob.tap.txt | 11 + .../cli/fixtures/_load-multiple-files.tap.txt | 12 + test/cli/fixtures/_load-single-file.tap.txt | 10 + ...ing-expect.js => assert-expect-failure.js} | 0 .../fixtures/assert-expect-failure.tap.txt | 21 + ...ions.js => assert-expect-no-assertions.js} | 0 .../assert-expect-no-assertions.tap.txt | 21 + ...rows-match.js => assert-throws-failure.js} | 0 ...est.js => async-module-warning-promise.js} | 0 .../async-module-warning-promise.tap.txt | 13 + .../test.js => async-module-warning.js} | 0 .../cli/fixtures/async-module-warning.tap.txt | 13 + test/cli/fixtures/{single.js => basic-one.js} | 0 test/cli/fixtures/{double.js => basic-two.js} | 0 test/cli/fixtures/callbacks-rejected.tap.txt | 20 + .../config-filter-regex-exclude.tap.txt | 11 + test/cli/fixtures/config-filter-regex.tap.txt | 12 + .../cli/fixtures/config-filter-string.tap.txt | 11 + test/cli/fixtures/config-module.tap.txt | 10 + test/cli/fixtures/config-moduleId.tap.txt | 14 + .../add-global.js => config-noglobals-add.js} | 0 .../cli/fixtures/config-noglobals-add.tap.txt | 20 + ...ignored.js => config-noglobals-ignored.js} | 0 .../fixtures/config-noglobals-ignored.tap.txt | 10 + ...e-global.js => config-noglobals-remove.js} | 0 .../fixtures/config-noglobals-remove.tap.txt | 20 + ...js => config-notrycatch-hook-rejection.js} | 0 ...js => config-notrycatch-test-rejection.js} | 0 ...re-expects.js => config-requireExpects.js} | 0 .../fixtures/config-requireExpects.tap.txt | 21 + test/cli/fixtures/config-testId.tap.txt | 13 + test/cli/fixtures/config-testTimeout.tap.txt | 20 + test/cli/fixtures/done-after-timeout.tap.txt | 20 + .../drooling-extra-done-outside.tap.txt | 24 + test/cli/fixtures/expected/tap-outputs.js | 976 ------------------ test/cli/fixtures/{fail => }/failure.js | 0 test/cli/fixtures/failure.tap.txt | 29 + .../filter-modulename-insensitive.tap.txt | 10 + test/cli/fixtures/filter-modulename.tap.txt | 10 + .../index.js => hanging-test.js} | 0 test/cli/fixtures/hanging-test.tap.txt | 10 + .../cli/fixtures/hooks-global-context.tap.txt | 12 + .../test.js => hooks-other-module-warning.js} | 0 .../hooks-other-module-warning.tap.txt | 13 + test/cli/fixtures/module-nested.tap.txt | 11 + ...o-tests.js => no-tests-failOnZeroTests.js} | 0 .../fixtures/no-tests-failOnZeroTests.tap.txt | 9 + .../{no-tests/index.js => no-tests.js} | 0 test/cli/fixtures/no-tests.tap.txt | 22 + .../module-flat.js => only-module-flat.js} | 0 ...-then-test.js => only-module-then-test.js} | 0 .../fixtures/only-module-then-test.tap.txt | 10 + .../{only/module.js => only-module.js} | 0 .../fixtures/{only/test.js => only-test.js} | 0 test/cli/fixtures/only-test.tap.txt | 11 + .../pending-async-after-timeout.tap.txt | 20 + .../require-module-and-script.tap.txt | 12 + test/cli/fixtures/seed.tap.txt | 15 + .../{syntax-error/test.js => syntax-error.js} | 0 test/cli/fixtures/syntax-error.tap.txt | 34 + .../fixtures/{timeout/index.js => timeout.js} | 0 test/cli/fixtures/timeout.tap.txt | 21 + ...s => uncaught-error-after-assert-async.js} | 0 .../uncaught-error-after-assert-async.tap.txt | 24 + ... => uncaught-error-callback-moduleDone.js} | 0 ...uncaught-error-callback-moduleDone.tap.txt | 10 + ...s => uncaught-error-callback-testStart.js} | 0 .../uncaught-error-callback-testStart.tap.txt | 9 + ...w.js => uncaught-error-callbacks-begin.js} | 0 .../uncaught-error-callbacks-begin.tap.txt | 20 + ...ow.js => uncaught-error-callbacks-done.js} | 0 .../uncaught-error-callbacks-done.tap.txt | 22 + ...r-in-hook.js => uncaught-error-in-hook.js} | 0 .../fixtures/uncaught-error-in-hook.tap.txt | 21 + test/cli/fixtures/unhandled-rejection.tap.txt | 34 + test/cli/fixtures/zero-assertions.tap.txt | 10 + test/cli/helpers/execute.js | 33 +- test/cli/helpers/fixtures.js | 48 + test/cli/utils.js | 4 +- 86 files changed, 1074 insertions(+), 1094 deletions(-) create mode 100644 test/cli/fixtures/_load-default.tap.txt create mode 100644 test/cli/fixtures/_load-dir-file-glob.tap.txt create mode 100644 test/cli/fixtures/_load-dir.tap.txt create mode 100644 test/cli/fixtures/_load-filenotfound.tap.txt create mode 100644 test/cli/fixtures/_load-glob.tap.txt create mode 100644 test/cli/fixtures/_load-multiple-files.tap.txt create mode 100644 test/cli/fixtures/_load-single-file.tap.txt rename test/cli/fixtures/{assert-expect/failing-expect.js => assert-expect-failure.js} (100%) create mode 100644 test/cli/fixtures/assert-expect-failure.tap.txt rename test/cli/fixtures/{assert-expect/no-assertions.js => assert-expect-no-assertions.js} (100%) create mode 100644 test/cli/fixtures/assert-expect-no-assertions.tap.txt rename test/cli/fixtures/{fail/throws-match.js => assert-throws-failure.js} (100%) rename test/cli/fixtures/{async-module-warning/promise-test.js => async-module-warning-promise.js} (100%) create mode 100644 test/cli/fixtures/async-module-warning-promise.tap.txt rename test/cli/fixtures/{async-module-warning/test.js => async-module-warning.js} (100%) create mode 100644 test/cli/fixtures/async-module-warning.tap.txt rename test/cli/fixtures/{single.js => basic-one.js} (100%) rename test/cli/fixtures/{double.js => basic-two.js} (100%) create mode 100644 test/cli/fixtures/callbacks-rejected.tap.txt create mode 100644 test/cli/fixtures/config-filter-regex-exclude.tap.txt create mode 100644 test/cli/fixtures/config-filter-regex.tap.txt create mode 100644 test/cli/fixtures/config-filter-string.tap.txt create mode 100644 test/cli/fixtures/config-module.tap.txt create mode 100644 test/cli/fixtures/config-moduleId.tap.txt rename test/cli/fixtures/{noglobals/add-global.js => config-noglobals-add.js} (100%) create mode 100644 test/cli/fixtures/config-noglobals-add.tap.txt rename test/cli/fixtures/{noglobals/ignored.js => config-noglobals-ignored.js} (100%) create mode 100644 test/cli/fixtures/config-noglobals-ignored.tap.txt rename test/cli/fixtures/{noglobals/remove-global.js => config-noglobals-remove.js} (100%) create mode 100644 test/cli/fixtures/config-noglobals-remove.tap.txt rename test/cli/fixtures/{notrycatch/returns-rejection-in-hook.js => config-notrycatch-hook-rejection.js} (100%) rename test/cli/fixtures/{notrycatch/returns-rejection.js => config-notrycatch-test-rejection.js} (100%) rename test/cli/fixtures/{assert-expect/require-expects.js => config-requireExpects.js} (100%) create mode 100644 test/cli/fixtures/config-requireExpects.tap.txt create mode 100644 test/cli/fixtures/config-testId.tap.txt create mode 100644 test/cli/fixtures/config-testTimeout.tap.txt create mode 100644 test/cli/fixtures/done-after-timeout.tap.txt create mode 100644 test/cli/fixtures/drooling-extra-done-outside.tap.txt delete mode 100644 test/cli/fixtures/expected/tap-outputs.js rename test/cli/fixtures/{fail => }/failure.js (100%) create mode 100644 test/cli/fixtures/failure.tap.txt create mode 100644 test/cli/fixtures/filter-modulename-insensitive.tap.txt create mode 100644 test/cli/fixtures/filter-modulename.tap.txt rename test/cli/fixtures/{hanging-test/index.js => hanging-test.js} (100%) create mode 100644 test/cli/fixtures/hanging-test.tap.txt create mode 100644 test/cli/fixtures/hooks-global-context.tap.txt rename test/cli/fixtures/{incorrect-hooks-warning/test.js => hooks-other-module-warning.js} (100%) create mode 100644 test/cli/fixtures/hooks-other-module-warning.tap.txt create mode 100644 test/cli/fixtures/module-nested.tap.txt rename test/cli/fixtures/{assert-expect/no-tests.js => no-tests-failOnZeroTests.js} (100%) create mode 100644 test/cli/fixtures/no-tests-failOnZeroTests.tap.txt rename test/cli/fixtures/{no-tests/index.js => no-tests.js} (100%) create mode 100644 test/cli/fixtures/no-tests.tap.txt rename test/cli/fixtures/{only/module-flat.js => only-module-flat.js} (100%) rename test/cli/fixtures/{only/module-then-test.js => only-module-then-test.js} (100%) create mode 100644 test/cli/fixtures/only-module-then-test.tap.txt rename test/cli/fixtures/{only/module.js => only-module.js} (100%) rename test/cli/fixtures/{only/test.js => only-test.js} (100%) create mode 100644 test/cli/fixtures/only-test.tap.txt create mode 100644 test/cli/fixtures/pending-async-after-timeout.tap.txt create mode 100644 test/cli/fixtures/require-module-and-script.tap.txt create mode 100644 test/cli/fixtures/seed.tap.txt rename test/cli/fixtures/{syntax-error/test.js => syntax-error.js} (100%) create mode 100644 test/cli/fixtures/syntax-error.tap.txt rename test/cli/fixtures/{timeout/index.js => timeout.js} (100%) create mode 100644 test/cli/fixtures/timeout.tap.txt rename test/cli/fixtures/{hard-error-in-test-with-no-async-handler.js => uncaught-error-after-assert-async.js} (100%) create mode 100644 test/cli/fixtures/uncaught-error-after-assert-async.tap.txt rename test/cli/fixtures/{bad-callbacks/moduleDone-throw.js => uncaught-error-callback-moduleDone.js} (100%) create mode 100644 test/cli/fixtures/uncaught-error-callback-moduleDone.tap.txt rename test/cli/fixtures/{bad-callbacks/testStart-throw.js => uncaught-error-callback-testStart.js} (100%) create mode 100644 test/cli/fixtures/uncaught-error-callback-testStart.tap.txt rename test/cli/fixtures/{bad-callbacks/begin-throw.js => uncaught-error-callbacks-begin.js} (100%) create mode 100644 test/cli/fixtures/uncaught-error-callbacks-begin.tap.txt rename test/cli/fixtures/{bad-callbacks/done-throw.js => uncaught-error-callbacks-done.js} (100%) create mode 100644 test/cli/fixtures/uncaught-error-callbacks-done.tap.txt rename test/cli/fixtures/{hard-error-in-hook.js => uncaught-error-in-hook.js} (100%) create mode 100644 test/cli/fixtures/uncaught-error-in-hook.tap.txt create mode 100644 test/cli/fixtures/unhandled-rejection.tap.txt create mode 100644 test/cli/fixtures/zero-assertions.tap.txt create mode 100644 test/cli/helpers/fixtures.js diff --git a/.gitattributes b/.gitattributes index 6f59647ad..37826b22a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,5 +3,6 @@ # JS and HTML files must always use LF for tools to work *.js eol=lf +*.tap.txt eol=lf *.json eol=lf *.html eol=lf diff --git a/test/cli/cli-main.js b/test/cli/cli-main.js index be4079322..f7151f05d 100644 --- a/test/cli/cli-main.js +++ b/test/cli/cli-main.js @@ -1,94 +1,47 @@ 'use strict'; -const expectedOutput = require('./fixtures/expected/tap-outputs'); -const { execute, prettyPrintCommand, concurrentMapKeys } = require('./helpers/execute'); +const path = require('path'); + const semver = require('semver'); -const skipOnWinTest = (process.platform === 'win32' ? 'skip' : 'test'); +const { execute, concurrentMapKeys } = require('./helpers/execute.js'); +const { readFixtures } = require('./helpers/fixtures.js'); -function getExpected (command) { - return expectedOutput[prettyPrintCommand(command)]; -} - -const fixtureCases = { - 'load "tests" directory by default': ['qunit'], - 'load file not found': ['qunit', 'does-not-exist.js'], - 'load glob pattern': ['qunit', 'glob/**/*-test.js'], - 'load single file': ['qunit', 'single.js'], - 'load multiple files': ['qunit', 'single.js', 'double.js'], - 'load a directory': ['qunit', 'test'], - 'load mixture of file, directory, and glob': ['qunit', 'test', 'single.js', 'glob/**/*-test.js'], - 'load file with syntax error': ['qunit', 'syntax-error/test.js'], - - 'no tests': ['qunit', 'no-tests'], - 'no tests and config.failOnZeroTests=false': ['qunit', 'assert-expect/no-tests.js'], - - 'test with failing assertion': ['qunit', 'fail/failure.js'], - 'test that hangs': ['qunit', 'hanging-test'], - 'test with pending async after timeout': ['qunit', 'pending-async-after-timeout.js'], - 'two tests with one timeout': ['qunit', 'timeout'], - 'test with zero assertions': ['qunit', 'zero-assertions.js'], - - 'unhandled rejection': ['qunit', 'unhandled-rejection.js'], - 'uncaught error after assert.async()': ['qunit', 'hard-error-in-test-with-no-async-handler.js'], - 'uncaught error in hook': ['qunit', 'hard-error-in-hook.js'], - 'uncaught error in "begin" callback': ['qunit', 'bad-callbacks/begin-throw.js'], - 'uncaught error in "done" callback': ['qunit', 'bad-callbacks/done-throw.js'], - // FIXME: Details of moduleDone() error are swallowed - 'uncaught error in "moduleDone" callback"': ['qunit', 'bad-callbacks/moduleDone-throw.js'], - // FIXME: Details of testStart() error are swallowed - 'uncaught error in "testStart" callback"': ['qunit', 'bad-callbacks/testStart-throw.js'], - 'rejection from callbacks': ['qunit', 'callbacks-rejected.js'], - - 'QUnit.hooks context': ['qunit', 'hooks-global-context.js'], - - '--filter matches module': ['qunit', '--filter', 'single', 'test', 'single.js', 'glob/**/*-test.js'], - '--module selects a module (case-insensitive)': ['qunit', '--module', 'seconD', 'test/'], - '--require loads dependency and script': ['qunit', 'single.js', '--require', 'require-dep', '--require', './node_modules/require-dep/module.js'], - '--seed value': ['qunit', '--seed', 's33d', 'test', 'single.js', 'glob/**/*-test.js'], - - 'config.filter with a string': ['qunit', 'config-filter-string.js'], - 'config.filter with a regex': ['qunit', 'config-filter-regex.js'], - 'config.filter with inverted regex': ['qunit', 'config-filter-regex-exclude.js'], - - 'config.module': ['qunit', 'config-module.js'], - 'config.moduleId': ['qunit', 'config-moduleId.js'], - 'config.testId': ['qunit', 'config-testId.js'], - 'config.testTimeout': ['qunit', 'config-testTimeout.js'], - 'config.noglobals and add a global': ['qunit', 'noglobals/add-global.js'], - 'config.noglobals and remove a global': ['qunit', 'noglobals/remove-global.js'], - 'config.noglobals and add ignored DOM global': ['qunit', 'noglobals/ignored.js'], - - 'assert.async() handled after timeout': ['qunit', 'done-after-timeout.js'], - 'assert.async() handled outside test': ['qunit', 'drooling-extra-done-outside.js'], - 'assert.expect() different count': ['qunit', 'assert-expect/failing-expect.js'], - 'assert.expect() no assertions': ['qunit', 'assert-expect/no-assertions.js'], - 'assert.expect() missing and config.requireExpects=true': ['qunit', 'assert-expect/require-expects.js'], - 'test.only()': ['qunit', 'only/test.js'], - 'module.only() followed by test': ['qunit', 'only/module-then-test.js'], - 'module() nested with interrupted executeNow': ['qunit', 'module-nested.js'], - 'module() with async function': ['qunit', 'async-module-warning/test.js'], - 'module() with promise': ['qunit', 'async-module-warning/promise-test.js'], - - 'hooks.beforeEach() during other module': ['qunit', 'incorrect-hooks-warning/test.js'] -}; +const FIXTURES_DIR = path.join(__dirname, 'fixtures'); +const skipOnWinTest = (process.platform === 'win32' ? 'skip' : 'test'); QUnit.module('CLI Main', () => { QUnit.test.each('fixtures', // Faster testing: Let the commands run in the background with concurrency, // and only await/assert the already-started command. - concurrentMapKeys(fixtureCases, 0, (command) => execute(command)), - async (assert, execution) => { - const result = await execution; - assert.equal(result.snapshot, getExpected(result.command)); + concurrentMapKeys(readFixtures(FIXTURES_DIR), 0, (runFixture) => runFixture()), + async (assert, fixture) => { + const result = await fixture; + assert.equal(result.snapshot, result.expected); } ); // TODO: Figure out why trace isn't trimmed on Windows. https://github.com/qunitjs/qunit/issues/1359 QUnit[skipOnWinTest]('report assert.throws() failures properly', async assert => { - const command = ['qunit', 'fail/throws-match.js']; + const command = ['qunit', 'assert-throws-failure.js']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +not ok 1 Throws match > bad + --- + message: match error + severity: failed + actual : Error: Match me with a pattern + expected: "/incorrect pattern/" + stack: | + at /qunit/test/cli/fixtures/assert-throws-failure.js:3:12 + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1`); }); QUnit.test('callbacks', async assert => { @@ -272,7 +225,13 @@ HOOK: BCD1 @ B after`; const stderr = semver.gte(process.versions.node, '14.0.0') ? execution.stderr : ''; assert.equal(execution.code, 0); assert.equal(stderr, ''); - assert.equal(execution.stdout, getExpected(command)); + assert.equal(execution.stdout, `TAP version 13 +ok 1 ESM test suite > sum() +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0`); }); } @@ -283,7 +242,24 @@ HOOK: BCD1 @ B after`; const command = ['qunit', 'sourcemap/source.js']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +ok 1 Example > good +not ok 2 Example > bad + --- + message: failed + severity: failed + actual : false + expected: true + stack: | + at /qunit/test/cli/fixtures/sourcemap/source.js:7:16 + ... +1..2 +# pass 1 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1`); }); // skip if running in code coverage mode, @@ -297,7 +273,24 @@ HOOK: BCD1 @ B after`; env: { NODE_OPTIONS: '--enable-source-maps' } }); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +ok 1 Example > good +not ok 2 Example > bad + --- + message: failed + severity: failed + actual : false + expected: true + stack: | + at /qunit/test/cli/fixtures/sourcemap/sourcemap/source.js:7:10 + ... +1..2 +# pass 1 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1`); }); } @@ -307,19 +300,40 @@ HOOK: BCD1 @ B after`; QUnit.test('memory-leak/module-closure [unfiltered]', async assert => { const command = ['node', '--expose-gc', '../../../bin/qunit.js', 'memory-leak/module-closure.js']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +ok 1 module-closure > example test +ok 2 module-closure > example child module > example child module test +ok 3 module-closure check > memory release +1..3 +# pass 3 +# skip 0 +# todo 0 +# fail 0`); }); QUnit.test('memory-leak/module-closure [filtered module]', async assert => { const command = ['node', '--expose-gc', '../../../bin/qunit.js', '--filter', '!child', 'memory-leak/module-closure.js']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +ok 1 module-closure > example test +ok 2 module-closure check > memory release +1..2 +# pass 2 +# skip 0 +# todo 0 +# fail 0`); }); QUnit.test('memory-leak/test-object', async assert => { const command = ['node', '--expose-gc', '../../../bin/qunit.js', 'memory-leak/test-object.js']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +ok 1 test-object > example test +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0`); }); } @@ -331,11 +345,29 @@ HOOK: BCD1 @ B after`; const command = ['qunit', '--filter', 'no matches', 'test']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +not ok 1 global failure + --- + message: "No tests matched the filter \\"no matches\\"." + severity: failed + actual : undefined + expected: undefined + stack: | + Error: No tests matched the filter "no matches". + at qunit.js + at internal + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1`); }); QUnit.test('--require loads unknown module', async assert => { - const command = ['qunit', 'single.js', '--require', 'does-not-exist-at-all']; + const command = ['qunit', 'basic-one.js', '--require', 'does-not-exist-at-all']; const execution = await execute(command); // TODO: Change to a generic tap-outputs.js // https://github.com/qunitjs/qunit/issues/1688 @@ -345,7 +377,7 @@ HOOK: BCD1 @ B after`; }); QUnit.test('config.notrycatch with rejected test', async assert => { - const command = ['qunit', 'notrycatch/returns-rejection.js']; + const command = ['qunit', 'config-notrycatch-test-rejection.js']; const execution = await execute(command); assert.pushResult({ @@ -356,7 +388,7 @@ HOOK: BCD1 @ B after`; }); QUnit.test('config.notrycatch with rejected hook', async assert => { - const command = ['qunit', 'notrycatch/returns-rejection-in-hook.js']; + const command = ['qunit', 'config-notrycatch-hook-rejection.js']; const execution = await execute(command); assert.pushResult({ @@ -371,7 +403,28 @@ HOOK: BCD1 @ B after`; const command = ['qunit', 'drooling-done.js']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +not ok 1 Test A + --- + message: |+ + Died on test #2: this is an intentional error + at /qunit/test/cli/fixtures/drooling-done.js:5:7 + at internal + severity: failed + actual : null + expected: undefined + stack: | + Error: this is an intentional error + at /qunit/test/cli/fixtures/drooling-done.js:8:9 + ... +ok 2 Test B +1..2 +# pass 1 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1`); }); // TODO: Figure out why trace isn't trimmed on Windows. https://github.com/qunitjs/qunit/issues/1359 @@ -379,7 +432,29 @@ HOOK: BCD1 @ B after`; const command = ['qunit', 'drooling-extra-done.js']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +ok 1 Test A +not ok 2 Test B + --- + message: |+ + Died on test #2: Unexpected release of async pause during a different test. + > Test: Test A [async #1] + at /qunit/test/cli/fixtures/drooling-extra-done.js:13:7 + at internal + severity: failed + actual : null + expected: undefined + stack: | + Error: Unexpected release of async pause during a different test. + > Test: Test A [async #1] + ... +1..2 +# pass 1 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1`); }); // TODO: Figure out why trace isn't trimmed on Windows. https://github.com/qunitjs/qunit/issues/1359 @@ -387,22 +462,78 @@ HOOK: BCD1 @ B after`; const command = ['qunit', 'too-many-done-calls.js']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +not ok 1 Test A + --- + message: |+ + Died on test #2: Tried to release async pause that was already released. + > Test: Test A [async #1] + at /qunit/test/cli/fixtures/too-many-done-calls.js:1:7 + at internal + severity: failed + actual : null + expected: undefined + stack: | + Error: Tried to release async pause that was already released. + > Test: Test A [async #1] + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1`); }); // TODO: Figure out why trace isn't trimmed on Windows. https://github.com/qunitjs/qunit/issues/1359 QUnit[skipOnWinTest]('module.only() nested', async assert => { - const command = ['qunit', 'only/module.js']; + const command = ['qunit', 'only-module.js']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +not ok 1 # TODO module B > Only this module should run > a todo test + --- + message: not implemented yet + severity: todo + actual : false + expected: true + stack: | + at /qunit/test/cli/fixtures/only-module.js:17:18 + ... +ok 2 # SKIP module B > Only this module should run > implicitly skipped test +ok 3 module B > Only this module should run > normal test +ok 4 module D > test D +ok 5 module E > module F > test F +ok 6 module E > test E +1..8 +# pass 6 +# skip 1 +# todo 1 +# fail 0`); }); // TODO: Figure out why trace isn't trimmed on Windows. https://github.com/qunitjs/qunit/issues/1359 QUnit[skipOnWinTest]('module.only() flat', async assert => { - const command = ['qunit', 'only/module-flat.js']; + const command = ['qunit', 'only-module-flat.js']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `TAP version 13 +not ok 1 # TODO module B > test B + --- + message: not implemented yet + severity: todo + actual : false + expected: true + stack: | + at /qunit/test/cli/fixtures/only-module-flat.js:8:14 + ... +ok 2 # SKIP module B > test C +ok 3 module B > test D +1..4 +# pass 2 +# skip 1 +# todo 1 +# fail 0`); }); }); diff --git a/test/cli/custom-reporter.js b/test/cli/custom-reporter.js index 7ae4744b9..0728b081c 100644 --- a/test/cli/custom-reporter.js +++ b/test/cli/custom-reporter.js @@ -2,12 +2,7 @@ const NPMReporter = require('npm-reporter'); const findReporter = require('../../src/cli/find-reporter').findReporter; -const expectedOutput = require('./fixtures/expected/tap-outputs'); -const { execute, prettyPrintCommand } = require('./helpers/execute'); - -function getExpected (command) { - return expectedOutput[prettyPrintCommand(command)]; -} +const { execute } = require('./helpers/execute'); QUnit.module('find-reporter', function () { QUnit.test('tap reporter is bundled', function (assert) { @@ -40,20 +35,29 @@ QUnit.module('CLI Reporter', function () { const command = ['qunit', '--reporter', 'npm-reporter']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, 'Run ended!'); }); QUnit.test('exits early and lists available reporters if reporter is not found', async function (assert) { const command = ['qunit', '--reporter', 'does-not-exist']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `# stderr +No reporter found matching "does-not-exist". +Built-in reporters: console, tap +Extra reporters found among package dependencies: npm-reporter + +# exit code: 1`); }); QUnit.test('exits early and lists available reporters if reporter option is used with no value', async function (assert) { const command = ['qunit', '--reporter']; const execution = await execute(command); - assert.equal(execution.snapshot, getExpected(command)); + assert.equal(execution.snapshot, `# stderr +Built-in reporters: console, tap +Extra reporters found among package dependencies: npm-reporter + +# exit code: 1`); }); }); diff --git a/test/cli/fixtures/_load-default.tap.txt b/test/cli/fixtures/_load-default.tap.txt new file mode 100644 index 000000000..c437e2b78 --- /dev/null +++ b/test/cli/fixtures/_load-default.tap.txt @@ -0,0 +1,11 @@ +# name: load "test" directory by default +# command: ["qunit"] + +TAP version 13 +ok 1 First > 1 +ok 2 Second > 1 +1..2 +# pass 2 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/_load-dir-file-glob.tap.txt b/test/cli/fixtures/_load-dir-file-glob.tap.txt new file mode 100644 index 000000000..c6f9cd5cc --- /dev/null +++ b/test/cli/fixtures/_load-dir-file-glob.tap.txt @@ -0,0 +1,14 @@ +# name: load mixture of file, directory, and glob +# command: ["qunit","test","basic-one.js","glob/**/*-test.js"] + +TAP version 13 +ok 1 Single > has a test +ok 2 A-Test > derp +ok 3 Nested-Test > herp +ok 4 First > 1 +ok 5 Second > 1 +1..5 +# pass 5 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/_load-dir.tap.txt b/test/cli/fixtures/_load-dir.tap.txt new file mode 100644 index 000000000..2714e3c70 --- /dev/null +++ b/test/cli/fixtures/_load-dir.tap.txt @@ -0,0 +1,11 @@ +# name: load a directory +# command: ["qunit","test"] + +TAP version 13 +ok 1 First > 1 +ok 2 Second > 1 +1..2 +# pass 2 +# skip 0 +# todo 0 +# fail 0 \ No newline at end of file diff --git a/test/cli/fixtures/_load-filenotfound.tap.txt b/test/cli/fixtures/_load-filenotfound.tap.txt new file mode 100644 index 000000000..a49c75452 --- /dev/null +++ b/test/cli/fixtures/_load-filenotfound.tap.txt @@ -0,0 +1,7 @@ +# name: load file not found +# command: ["qunit","does-not-exist.js"] + +# stderr +No files were found matching: does-not-exist.js + +# exit code: 1 \ No newline at end of file diff --git a/test/cli/fixtures/_load-glob.tap.txt b/test/cli/fixtures/_load-glob.tap.txt new file mode 100644 index 000000000..f76c4f43c --- /dev/null +++ b/test/cli/fixtures/_load-glob.tap.txt @@ -0,0 +1,11 @@ +# name: load glob pattern +# command: ["qunit","glob/**/*-test.js"] + +TAP version 13 +ok 1 A-Test > derp +ok 2 Nested-Test > herp +1..2 +# pass 2 +# skip 0 +# todo 0 +# fail 0 \ No newline at end of file diff --git a/test/cli/fixtures/_load-multiple-files.tap.txt b/test/cli/fixtures/_load-multiple-files.tap.txt new file mode 100644 index 000000000..153cfddee --- /dev/null +++ b/test/cli/fixtures/_load-multiple-files.tap.txt @@ -0,0 +1,12 @@ +# name: load multiple files +# command: ["qunit","basic-one.js","basic-two.js"] + +TAP version 13 +ok 1 Single > has a test +ok 2 Double > has a test +ok 3 Double > has another test +1..3 +# pass 3 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/_load-single-file.tap.txt b/test/cli/fixtures/_load-single-file.tap.txt new file mode 100644 index 000000000..506927d05 --- /dev/null +++ b/test/cli/fixtures/_load-single-file.tap.txt @@ -0,0 +1,10 @@ +# name: load single file +# command: ["qunit","basic-one.js"] + +TAP version 13 +ok 1 Single > has a test +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/assert-expect/failing-expect.js b/test/cli/fixtures/assert-expect-failure.js similarity index 100% rename from test/cli/fixtures/assert-expect/failing-expect.js rename to test/cli/fixtures/assert-expect-failure.js diff --git a/test/cli/fixtures/assert-expect-failure.tap.txt b/test/cli/fixtures/assert-expect-failure.tap.txt new file mode 100644 index 000000000..ca7938917 --- /dev/null +++ b/test/cli/fixtures/assert-expect-failure.tap.txt @@ -0,0 +1,21 @@ +# name: assert.expect() different count +# command: ["qunit", "assert-expect-failure.js"] + +TAP version 13 +not ok 1 failing test + --- + message: Expected 2 assertions, but 1 were run + severity: failed + actual : null + expected: undefined + stack: | + at /qunit/test/cli/fixtures/assert-expect-failure.js:1:7 + at internal + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 diff --git a/test/cli/fixtures/assert-expect/no-assertions.js b/test/cli/fixtures/assert-expect-no-assertions.js similarity index 100% rename from test/cli/fixtures/assert-expect/no-assertions.js rename to test/cli/fixtures/assert-expect-no-assertions.js diff --git a/test/cli/fixtures/assert-expect-no-assertions.tap.txt b/test/cli/fixtures/assert-expect-no-assertions.tap.txt new file mode 100644 index 000000000..a30494830 --- /dev/null +++ b/test/cli/fixtures/assert-expect-no-assertions.tap.txt @@ -0,0 +1,21 @@ +# name: assert.expect() no assertions +# command: ["qunit", "assert-expect-no-assertions.js"] + +TAP version 13 +not ok 1 no assertions + --- + message: Expected at least one assertion, but none were run - call expect(0) to accept zero assertions. + severity: failed + actual : null + expected: undefined + stack: | + at /qunit/test/cli/fixtures/assert-expect-no-assertions.js:1:7 + at internal + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 diff --git a/test/cli/fixtures/fail/throws-match.js b/test/cli/fixtures/assert-throws-failure.js similarity index 100% rename from test/cli/fixtures/fail/throws-match.js rename to test/cli/fixtures/assert-throws-failure.js diff --git a/test/cli/fixtures/async-module-warning/promise-test.js b/test/cli/fixtures/async-module-warning-promise.js similarity index 100% rename from test/cli/fixtures/async-module-warning/promise-test.js rename to test/cli/fixtures/async-module-warning-promise.js diff --git a/test/cli/fixtures/async-module-warning-promise.tap.txt b/test/cli/fixtures/async-module-warning-promise.tap.txt new file mode 100644 index 000000000..2df40d80c --- /dev/null +++ b/test/cli/fixtures/async-module-warning-promise.tap.txt @@ -0,0 +1,13 @@ +# name: module() with promise +# command: ["qunit","async-module-warning-promise.js"] + +TAP version 13 +ok 1 module manually returning a promise > has a test +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 + +# stderr +Returning a promise from a module callback is not supported. Instead, use hooks for async behavior. This will become an error in QUnit 3.0. diff --git a/test/cli/fixtures/async-module-warning/test.js b/test/cli/fixtures/async-module-warning.js similarity index 100% rename from test/cli/fixtures/async-module-warning/test.js rename to test/cli/fixtures/async-module-warning.js diff --git a/test/cli/fixtures/async-module-warning.tap.txt b/test/cli/fixtures/async-module-warning.tap.txt new file mode 100644 index 000000000..1a6acb31b --- /dev/null +++ b/test/cli/fixtures/async-module-warning.tap.txt @@ -0,0 +1,13 @@ +# name: module() with async function +# command: ["qunit","async-module-warning.js"] + +TAP version 13 +ok 1 resulting parent module > has a test +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 + +# stderr +Returning a promise from a module callback is not supported. Instead, use hooks for async behavior. This will become an error in QUnit 3.0. diff --git a/test/cli/fixtures/single.js b/test/cli/fixtures/basic-one.js similarity index 100% rename from test/cli/fixtures/single.js rename to test/cli/fixtures/basic-one.js diff --git a/test/cli/fixtures/double.js b/test/cli/fixtures/basic-two.js similarity index 100% rename from test/cli/fixtures/double.js rename to test/cli/fixtures/basic-two.js diff --git a/test/cli/fixtures/callbacks-rejected.tap.txt b/test/cli/fixtures/callbacks-rejected.tap.txt new file mode 100644 index 000000000..43961091b --- /dev/null +++ b/test/cli/fixtures/callbacks-rejected.tap.txt @@ -0,0 +1,20 @@ +# name: rejection from callbacks +# command: ["qunit", "callbacks-rejected.js"] + +TAP version 13 +not ok 1 global failure + --- + message: Error: begin + severity: failed + stack: | + Error: begin + at /qunit/test/cli/fixtures/callbacks-rejected.js:8:25 + at qunit.js + at internal + ... +Bail out! Error: begin + +# stderr +Error: Process exited before tests finished running + +# exit code: 1 diff --git a/test/cli/fixtures/config-filter-regex-exclude.tap.txt b/test/cli/fixtures/config-filter-regex-exclude.tap.txt new file mode 100644 index 000000000..12b733b96 --- /dev/null +++ b/test/cli/fixtures/config-filter-regex-exclude.tap.txt @@ -0,0 +1,11 @@ +# name: config.filter with inverted regex +# command: ["qunit","config-filter-regex-exclude.js"] + +TAP version 13 +ok 1 filter > foo test +ok 2 filter > Bar test +1..2 +# pass 2 +# skip 0 +# todo 0 +# fail 0 \ No newline at end of file diff --git a/test/cli/fixtures/config-filter-regex.tap.txt b/test/cli/fixtures/config-filter-regex.tap.txt new file mode 100644 index 000000000..6327cc80b --- /dev/null +++ b/test/cli/fixtures/config-filter-regex.tap.txt @@ -0,0 +1,12 @@ +# name: config.filter with a regex +# command: ["qunit","config-filter-regex.js"] + +TAP version 13 +ok 1 filter > foo test +ok 2 filter > FOO test +ok 3 filter > bar test +1..3 +# pass 3 +# skip 0 +# todo 0 +# fail 0 \ No newline at end of file diff --git a/test/cli/fixtures/config-filter-string.tap.txt b/test/cli/fixtures/config-filter-string.tap.txt new file mode 100644 index 000000000..d6b42c00a --- /dev/null +++ b/test/cli/fixtures/config-filter-string.tap.txt @@ -0,0 +1,11 @@ +# name: config.filter with a string +# command: ["qunit","config-filter-string.js"] + +TAP version 13 +ok 1 filter > foo test +ok 2 filter > bar test +1..2 +# pass 2 +# skip 0 +# todo 0 +# fail 0 \ No newline at end of file diff --git a/test/cli/fixtures/config-module.tap.txt b/test/cli/fixtures/config-module.tap.txt new file mode 100644 index 000000000..6e6f61c95 --- /dev/null +++ b/test/cli/fixtures/config-module.tap.txt @@ -0,0 +1,10 @@ +# name: config.module +# command: ["qunit","config-module.js"] + +TAP version 13 +ok 1 Module B > Test B +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 \ No newline at end of file diff --git a/test/cli/fixtures/config-moduleId.tap.txt b/test/cli/fixtures/config-moduleId.tap.txt new file mode 100644 index 000000000..93cdaf9db --- /dev/null +++ b/test/cli/fixtures/config-moduleId.tap.txt @@ -0,0 +1,14 @@ +# name: config.moduleId +# command: ["qunit", "config-moduleId.js"] + +TAP version 13 +ok 1 module A scoped > module C nested > test C1 +ok 2 module D scoped > test D1 +ok 3 module D scoped > module E nested > test E1 +ok 4 module D scoped > test D2 +ok 5 module F flat > test F1 +1..5 +# pass 5 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/noglobals/add-global.js b/test/cli/fixtures/config-noglobals-add.js similarity index 100% rename from test/cli/fixtures/noglobals/add-global.js rename to test/cli/fixtures/config-noglobals-add.js diff --git a/test/cli/fixtures/config-noglobals-add.tap.txt b/test/cli/fixtures/config-noglobals-add.tap.txt new file mode 100644 index 000000000..5f502247a --- /dev/null +++ b/test/cli/fixtures/config-noglobals-add.tap.txt @@ -0,0 +1,20 @@ +# name: config.noglobals and add a global +# command: ["qunit", "config-noglobals-add.js"] + +TAP version 13 +not ok 1 adds global var + --- + message: Introduced global variable(s): dummyGlobal + severity: failed + actual : null + expected: undefined + stack: | + at qunit.js + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 diff --git a/test/cli/fixtures/noglobals/ignored.js b/test/cli/fixtures/config-noglobals-ignored.js similarity index 100% rename from test/cli/fixtures/noglobals/ignored.js rename to test/cli/fixtures/config-noglobals-ignored.js diff --git a/test/cli/fixtures/config-noglobals-ignored.tap.txt b/test/cli/fixtures/config-noglobals-ignored.tap.txt new file mode 100644 index 000000000..f092c372a --- /dev/null +++ b/test/cli/fixtures/config-noglobals-ignored.tap.txt @@ -0,0 +1,10 @@ +# name: config.noglobals and add ignored DOM global +# command: ["qunit", "config-noglobals-ignored.js"] + +TAP version 13 +ok 1 adds global var +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/noglobals/remove-global.js b/test/cli/fixtures/config-noglobals-remove.js similarity index 100% rename from test/cli/fixtures/noglobals/remove-global.js rename to test/cli/fixtures/config-noglobals-remove.js diff --git a/test/cli/fixtures/config-noglobals-remove.tap.txt b/test/cli/fixtures/config-noglobals-remove.tap.txt new file mode 100644 index 000000000..e02489100 --- /dev/null +++ b/test/cli/fixtures/config-noglobals-remove.tap.txt @@ -0,0 +1,20 @@ +# name: config.noglobals and remove a global +# command: ["qunit","config-noglobals-remove.js"] + +TAP version 13 +not ok 1 deletes global var + --- + message: Deleted global variable(s): dummyGlobal + severity: failed + actual : null + expected: undefined + stack: | + at qunit.js + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 \ No newline at end of file diff --git a/test/cli/fixtures/notrycatch/returns-rejection-in-hook.js b/test/cli/fixtures/config-notrycatch-hook-rejection.js similarity index 100% rename from test/cli/fixtures/notrycatch/returns-rejection-in-hook.js rename to test/cli/fixtures/config-notrycatch-hook-rejection.js diff --git a/test/cli/fixtures/notrycatch/returns-rejection.js b/test/cli/fixtures/config-notrycatch-test-rejection.js similarity index 100% rename from test/cli/fixtures/notrycatch/returns-rejection.js rename to test/cli/fixtures/config-notrycatch-test-rejection.js diff --git a/test/cli/fixtures/assert-expect/require-expects.js b/test/cli/fixtures/config-requireExpects.js similarity index 100% rename from test/cli/fixtures/assert-expect/require-expects.js rename to test/cli/fixtures/config-requireExpects.js diff --git a/test/cli/fixtures/config-requireExpects.tap.txt b/test/cli/fixtures/config-requireExpects.tap.txt new file mode 100644 index 000000000..964327a06 --- /dev/null +++ b/test/cli/fixtures/config-requireExpects.tap.txt @@ -0,0 +1,21 @@ +# name: assert.expect() missing and config.requireExpects=true +# command: ["qunit","config-requireExpects.js"] + +TAP version 13 +not ok 1 passing test + --- + message: Expected number of assertions to be defined, but expect() was not called. + severity: failed + actual : null + expected: undefined + stack: | + at /qunit/test/cli/fixtures/config-requireExpects.js:3:7 + at internal + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 \ No newline at end of file diff --git a/test/cli/fixtures/config-testId.tap.txt b/test/cli/fixtures/config-testId.tap.txt new file mode 100644 index 000000000..1ea40a9b8 --- /dev/null +++ b/test/cli/fixtures/config-testId.tap.txt @@ -0,0 +1,13 @@ +# name: config.testId +# command: ["qunit","config-testId.js"] + +TAP version 13 +ok 1 test 2 +ok 2 module A > module B > test 1 +ok 3 module A > module C > test 2 +ok 4 module D > test 1 +1..4 +# pass 4 +# skip 0 +# todo 0 +# fail 0 \ No newline at end of file diff --git a/test/cli/fixtures/config-testTimeout.tap.txt b/test/cli/fixtures/config-testTimeout.tap.txt new file mode 100644 index 000000000..cb4da3807 --- /dev/null +++ b/test/cli/fixtures/config-testTimeout.tap.txt @@ -0,0 +1,20 @@ +# name: config.testTimeout +# command: ["qunit", "config-testTimeout.js"] + +TAP version 13 +not ok 1 slow + --- + message: Test took longer than 10ms; test timed out. + severity: failed + actual : null + expected: undefined + stack: | + at internal + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 diff --git a/test/cli/fixtures/done-after-timeout.tap.txt b/test/cli/fixtures/done-after-timeout.tap.txt new file mode 100644 index 000000000..f0afcbc26 --- /dev/null +++ b/test/cli/fixtures/done-after-timeout.tap.txt @@ -0,0 +1,20 @@ +# name: assert.async() handled after timeout +# command: ["qunit","done-after-timeout.js"] + +TAP version 13 +not ok 1 times out before scheduled done is called + --- + message: Test took longer than 10ms; test timed out. + severity: failed + actual : null + expected: undefined + stack: | + at internal + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 \ No newline at end of file diff --git a/test/cli/fixtures/drooling-extra-done-outside.tap.txt b/test/cli/fixtures/drooling-extra-done-outside.tap.txt new file mode 100644 index 000000000..cb3b4e686 --- /dev/null +++ b/test/cli/fixtures/drooling-extra-done-outside.tap.txt @@ -0,0 +1,24 @@ +# name: assert.async() handled outside test +# command: ["qunit","drooling-extra-done-outside.js"] + +TAP version 13 +ok 1 extra done scheduled outside any test +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 +Bail out! Error: Unexpected release of async pause after tests finished. + --- + message: |+ + Error: Unexpected release of async pause after tests finished. + > Test: extra done scheduled outside any test [async #1] + severity: failed + stack: | + Error: Unexpected release of async pause after tests finished. + > Test: extra done scheduled outside any test [async #1] + at qunit.js + at internal + ... + +# exit code: 1 \ No newline at end of file diff --git a/test/cli/fixtures/expected/tap-outputs.js b/test/cli/fixtures/expected/tap-outputs.js deleted file mode 100644 index 95fccd98f..000000000 --- a/test/cli/fixtures/expected/tap-outputs.js +++ /dev/null @@ -1,976 +0,0 @@ -'use strict'; -/* eslint-disable quotes */ - -// Expected outputs from the TapReporter for the commands run in CLI tests -module.exports = { - qunit: -`TAP version 13 -ok 1 First > 1 -ok 2 Second > 1 -1..2 -# pass 2 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit does-not-exist.js': -`# stderr -No files were found matching: does-not-exist.js - -# exit code: 1`, - - "qunit 'glob/**/*-test.js'": -`TAP version 13 -ok 1 A-Test > derp -ok 2 Nested-Test > herp -1..2 -# pass 2 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit single.js': -`TAP version 13 -ok 1 Single > has a test -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit single.js double.js': -`TAP version 13 -ok 1 Double > has a test -ok 2 Double > has another test -ok 3 Single > has a test -1..3 -# pass 3 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit test': -`TAP version 13 -ok 1 First > 1 -ok 2 Second > 1 -1..2 -# pass 2 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit fail/throws-match.js': -`TAP version 13 -not ok 1 Throws match > bad - --- - message: match error - severity: failed - actual : Error: Match me with a pattern - expected: "/incorrect pattern/" - stack: | - at /qunit/test/cli/fixtures/fail/throws-match.js:3:12 - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - "qunit test single.js 'glob/**/*-test.js'": -`TAP version 13 -ok 1 A-Test > derp -ok 2 Nested-Test > herp -ok 3 Single > has a test -ok 4 First > 1 -ok 5 Second > 1 -1..5 -# pass 5 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit syntax-error/test.js': -`not ok 1 global failure - --- - message: |+ - Error: Failed to load file syntax-error/test.js - ReferenceError: varIsNotDefined is not defined - severity: failed - stack: | - ReferenceError: varIsNotDefined is not defined - at /qunit/test/cli/fixtures/syntax-error/test.js:1:1 - at internal - ... -Bail out! Error: Failed to load file syntax-error/test.js -TAP version 13 -not ok 2 global failure - --- - message: No tests were run. - severity: failed - actual : undefined - expected: undefined - stack: | - Error: No tests were run. - at qunit.js - at internal - ... -1..2 -# pass 0 -# skip 0 -# todo 0 -# fail 2 - -# exit code: 1`, - - 'qunit fail/failure.js': -`TAP version 13 -not ok 1 Failure > bad - --- - message: failed - severity: failed - actual : false - expected: true - stack: | - at /qunit/test/cli/fixtures/fail/failure.js:3:16 - ... -not ok 2 Failure > bad again - --- - message: failed - severity: failed - actual : false - expected: true - stack: | - at /qunit/test/cli/fixtures/fail/failure.js:7:16 - ... -1..2 -# pass 0 -# skip 0 -# todo 0 -# fail 2 - -# exit code: 1`, - - "qunit --seed s33d test single.js 'glob/**/*-test.js'": -`Running tests with seed: s33d -TAP version 13 -ok 1 Second > 1 -ok 2 Single > has a test -ok 3 First > 1 -ok 4 Nested-Test > herp -ok 5 A-Test > derp -1..5 -# pass 5 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit --reporter npm-reporter': -`Run ended!`, - - 'qunit --reporter does-not-exist': -`# stderr -No reporter found matching "does-not-exist". -Built-in reporters: console, tap -Extra reporters found among package dependencies: npm-reporter - -# exit code: 1`, - - 'qunit --reporter': -`# stderr -Built-in reporters: console, tap -Extra reporters found among package dependencies: npm-reporter - -# exit code: 1`, - - 'qunit hanging-test': -`TAP version 13 - -# stderr -Error: Process exited before tests finished running -Last test to run (hanging) has an async hold. Ensure all assert.async() callbacks are invoked and Promises resolve. You should also set a standard timeout via QUnit.config.testTimeout. - -# exit code: 1`, - - 'qunit pending-async-after-timeout.js': -`TAP version 13 -not ok 1 example - --- - message: Test took longer than 10ms; test timed out. - severity: failed - actual : null - expected: undefined - stack: | - at internal - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit unhandled-rejection.js': -`not ok 1 global failure - --- - message: Error: outside of a test context - severity: failed - stack: | - Error: outside of a test context - at /qunit/test/cli/fixtures/unhandled-rejection.js:17:18 - at qunit.js - at /qunit/test/cli/fixtures/unhandled-rejection.js:3:7 - at internal - ... -Bail out! Error: outside of a test context -TAP version 13 -not ok 2 Unhandled Rejections > test passes just fine, but has a rejected promise - --- - message: global failure: Error: Error thrown in non-returned promise! - severity: failed - actual : undefined - expected: undefined - stack: | - Error: Error thrown in non-returned promise! - at /qunit/test/cli/fixtures/unhandled-rejection.js:10:13 - at internal - ... -1..2 -# pass 0 -# skip 0 -# todo 0 -# fail 2 - -# exit code: 1`, - - 'qunit hard-error-in-test-with-no-async-handler.js': -`TAP version 13 -not ok 1 contains a hard error after using assert.async() - --- - message: |+ - Died on test #2: expected error thrown in test - at /qunit/test/cli/fixtures/hard-error-in-test-with-no-async-handler.js:1:7 - at internal - severity: failed - actual : null - expected: undefined - stack: | - Error: expected error thrown in test - at /qunit/test/cli/fixtures/hard-error-in-test-with-no-async-handler.js:4:9 - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit hard-error-in-hook.js': -`TAP version 13 -not ok 1 contains a hard error in hook > contains a hard error - --- - message: before failed on contains a hard error: expected error thrown in hook - severity: failed - actual : null - expected: undefined - stack: | - Error: expected error thrown in hook - at /qunit/test/cli/fixtures/hard-error-in-hook.js:3:11 - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit bad-callbacks/moduleDone-throw.js': -`TAP version 13 -ok 1 module1 > test1 - -# stderr -Error: Process exited before tests finished running - -# exit code: 1`, - - 'qunit bad-callbacks/testStart-throw.js': -`TAP version 13 - -# stderr -Error: Process exited before tests finished running - -# exit code: 1`, - - 'qunit callbacks-rejected.js': -`TAP version 13 -not ok 1 global failure - --- - message: Error: begin - severity: failed - stack: | - Error: begin - at /qunit/test/cli/fixtures/callbacks-rejected.js:8:25 - at qunit.js - at internal - ... -Bail out! Error: begin - -# stderr -Error: Process exited before tests finished running - -# exit code: 1`, - - 'qunit no-tests': -`TAP version 13 -not ok 1 global failure - --- - message: No tests were run. - severity: failed - actual : undefined - expected: undefined - stack: | - Error: No tests were run. - at qunit.js - at internal - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit sourcemap/source.js': -`TAP version 13 -ok 1 Example > good -not ok 2 Example > bad - --- - message: failed - severity: failed - actual : false - expected: true - stack: | - at /qunit/test/cli/fixtures/sourcemap/source.js:7:16 - ... -1..2 -# pass 1 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit sourcemap/source.min.js': -`TAP version 13 -ok 1 Example > good -not ok 2 Example > bad - --- - message: failed - severity: failed - actual : false - expected: true - stack: | - at /qunit/test/cli/fixtures/sourcemap/sourcemap/source.js:7:10 - ... -1..2 -# pass 1 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit ../../es2018/esm.mjs': -`TAP version 13 -ok 1 ESM test suite > sum() -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit timeout': -`TAP version 13 -not ok 1 timeout > first - --- - message: Test took longer than 10ms; test timed out. - severity: failed - actual : null - expected: undefined - stack: | - at internal - ... -ok 2 timeout > second -1..2 -# pass 1 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit hooks-global-context.js': -`TAP version 13 -ok 1 A > A1 -ok 2 A > AB > AB1 -ok 3 B -1..3 -# pass 3 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit zero-assertions.js': -`TAP version 13 -ok 1 Zero assertions > has a test -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0`, - - "qunit --filter single test single.js 'glob/**/*-test.js'": -`TAP version 13 -ok 1 Single > has a test -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0`, - - "qunit --filter 'no matches' test": -`TAP version 13 -not ok 1 global failure - --- - message: "No tests matched the filter \\"no matches\\"." - severity: failed - actual : undefined - expected: undefined - stack: | - Error: No tests matched the filter "no matches". - at qunit.js - at internal - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit --module seconD test/': -`TAP version 13 -ok 1 Second > 1 -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit single.js --require require-dep --require ./node_modules/require-dep/module.js': -`required require-dep/index.js -required require-dep/module.js -TAP version 13 -ok 1 Single > has a test -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0`, - - "node --expose-gc ../../../bin/qunit.js memory-leak/module-closure.js": -`TAP version 13 -ok 1 module-closure > example test -ok 2 module-closure > example child module > example child module test -ok 3 module-closure check > memory release -1..3 -# pass 3 -# skip 0 -# todo 0 -# fail 0`, - - "node --expose-gc ../../../bin/qunit.js --filter !child memory-leak/module-closure.js": -`TAP version 13 -ok 1 module-closure > example test -ok 2 module-closure check > memory release -1..2 -# pass 2 -# skip 0 -# todo 0 -# fail 0`, - - "node --expose-gc ../../../bin/qunit.js memory-leak/test-object.js": -`TAP version 13 -ok 1 test-object > example test -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit only/test.js': -`TAP version 13 -ok 1 run this test -ok 2 all tests with only run -1..3 -# pass 3 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit only/module.js': -`TAP version 13 -not ok 1 # TODO module B > Only this module should run > a todo test - --- - message: not implemented yet - severity: todo - actual : false - expected: true - stack: | - at /qunit/test/cli/fixtures/only/module.js:17:18 - ... -ok 2 # SKIP module B > Only this module should run > implicitly skipped test -ok 3 module B > Only this module should run > normal test -ok 4 module D > test D -ok 5 module E > module F > test F -ok 6 module E > test E -1..8 -# pass 6 -# skip 1 -# todo 1 -# fail 0`, - - 'qunit only/module-then-test.js': -`TAP version 13 -ok 1 module A > module B > test B -1..2 -# pass 2 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit only/module-flat.js': -`TAP version 13 -not ok 1 # TODO module B > test B - --- - message: not implemented yet - severity: todo - actual : false - expected: true - stack: | - at /qunit/test/cli/fixtures/only/module-flat.js:8:14 - ... -ok 2 # SKIP module B > test C -ok 3 module B > test D -1..4 -# pass 2 -# skip 1 -# todo 1 -# fail 0`, - - 'qunit module-nested.js': -`TAP version 13 -ok 1 module 1 > test in module 1 -ok 2 module 3 > test in module 3 -1..2 -# pass 2 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit incorrect-hooks-warning/test.js': -`TAP version 13 -ok 1 module providing hooks > module not providing hooks > has a test -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0 - -# stderr -The \`beforeEach\` hook was called inside the wrong module (\`module providing hooks > module not providing hooks\`). Instead, use hooks provided by the callback to the containing module (\`module providing hooks\`). This will become an error in QUnit 3.0.`, - - 'qunit async-module-warning/test.js': -`TAP version 13 -ok 1 resulting parent module > has a test -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0 - -# stderr -Returning a promise from a module callback is not supported. Instead, use hooks for async behavior. This will become an error in QUnit 3.0.`, - - 'qunit async-module-warning/promise-test.js': -`TAP version 13 -ok 1 module manually returning a promise > has a test -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0 - -# stderr -Returning a promise from a module callback is not supported. Instead, use hooks for async behavior. This will become an error in QUnit 3.0.`, - - 'qunit config-filter-string.js': -`TAP version 13 -ok 1 filter > foo test -ok 2 filter > bar test -1..2 -# pass 2 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit config-filter-regex.js': -`TAP version 13 -ok 1 filter > foo test -ok 2 filter > FOO test -ok 3 filter > bar test -1..3 -# pass 3 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit config-filter-regex-exclude.js': -`TAP version 13 -ok 1 filter > foo test -ok 2 filter > Bar test -1..2 -# pass 2 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit config-module.js': -`TAP version 13 -ok 1 Module B > Test B -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit config-moduleId.js': -`TAP version 13 -ok 1 module A scoped > module C nested > test C1 -ok 2 module D scoped > test D1 -ok 3 module D scoped > module E nested > test E1 -ok 4 module D scoped > test D2 -ok 5 module F flat > test F1 -1..5 -# pass 5 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit config-testId.js': -`TAP version 13 -ok 1 test 2 -ok 2 module A > module B > test 1 -ok 3 module A > module C > test 2 -ok 4 module D > test 1 -1..4 -# pass 4 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit config-testTimeout.js': -`TAP version 13 -not ok 1 slow - --- - message: Test took longer than 10ms; test timed out. - severity: failed - actual : null - expected: undefined - stack: | - at internal - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit noglobals/add-global.js': -`TAP version 13 -not ok 1 adds global var - --- - message: Introduced global variable(s): dummyGlobal - severity: failed - actual : null - expected: undefined - stack: | - at qunit.js - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit noglobals/remove-global.js': -`TAP version 13 -not ok 1 deletes global var - --- - message: Deleted global variable(s): dummyGlobal - severity: failed - actual : null - expected: undefined - stack: | - at qunit.js - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit noglobals/ignored.js': -`TAP version 13 -ok 1 adds global var -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit bad-callbacks/begin-throw.js': -`TAP version 13 -not ok 1 global failure - --- - message: Error: No dice - severity: failed - stack: | - Error: No dice - at /qunit/test/cli/fixtures/bad-callbacks/begin-throw.js:2:9 - at qunit.js - at internal - ... -Bail out! Error: No dice - -# stderr -Error: Process exited before tests finished running - -# exit code: 1`, - - 'qunit bad-callbacks/done-throw.js': -`TAP version 13 -ok 1 module1 > test1 -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0 -Bail out! Error: No dice - --- - message: Error: No dice - severity: failed - stack: | - Error: No dice - at /qunit/test/cli/fixtures/bad-callbacks/done-throw.js:2:9 - at qunit.js - at internal - ... - -# exit code: 1`, - - 'qunit done-after-timeout.js': -`TAP version 13 -not ok 1 times out before scheduled done is called - --- - message: Test took longer than 10ms; test timed out. - severity: failed - actual : null - expected: undefined - stack: | - at internal - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit drooling-done.js': -`TAP version 13 -not ok 1 Test A - --- - message: |+ - Died on test #2: this is an intentional error - at /qunit/test/cli/fixtures/drooling-done.js:5:7 - at internal - severity: failed - actual : null - expected: undefined - stack: | - Error: this is an intentional error - at /qunit/test/cli/fixtures/drooling-done.js:8:9 - ... -ok 2 Test B -1..2 -# pass 1 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit drooling-extra-done.js': -`TAP version 13 -ok 1 Test A -not ok 2 Test B - --- - message: |+ - Died on test #2: Unexpected release of async pause during a different test. - > Test: Test A [async #1] - at /qunit/test/cli/fixtures/drooling-extra-done.js:13:7 - at internal - severity: failed - actual : null - expected: undefined - stack: | - Error: Unexpected release of async pause during a different test. - > Test: Test A [async #1] - ... -1..2 -# pass 1 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit drooling-extra-done-outside.js': -`TAP version 13 -ok 1 extra done scheduled outside any test -1..1 -# pass 1 -# skip 0 -# todo 0 -# fail 0 -Bail out! Error: Unexpected release of async pause after tests finished. - --- - message: |+ - Error: Unexpected release of async pause after tests finished. - > Test: extra done scheduled outside any test [async #1] - severity: failed - stack: | - Error: Unexpected release of async pause after tests finished. - > Test: extra done scheduled outside any test [async #1] - at qunit.js - at internal - ... - -# exit code: 1`, - - 'qunit too-many-done-calls.js': -`TAP version 13 -not ok 1 Test A - --- - message: |+ - Died on test #2: Tried to release async pause that was already released. - > Test: Test A [async #1] - at /qunit/test/cli/fixtures/too-many-done-calls.js:1:7 - at internal - severity: failed - actual : null - expected: undefined - stack: | - Error: Tried to release async pause that was already released. - > Test: Test A [async #1] - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit assert-expect/no-tests.js': -`TAP version 13 -1..0 -# pass 0 -# skip 0 -# todo 0 -# fail 0`, - - 'qunit assert-expect/failing-expect.js': -`TAP version 13 -not ok 1 failing test - --- - message: Expected 2 assertions, but 1 were run - severity: failed - actual : null - expected: undefined - stack: | - at /qunit/test/cli/fixtures/assert-expect/failing-expect.js:1:7 - at internal - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit assert-expect/no-assertions.js': -`TAP version 13 -not ok 1 no assertions - --- - message: Expected at least one assertion, but none were run - call expect(0) to accept zero assertions. - severity: failed - actual : null - expected: undefined - stack: | - at /qunit/test/cli/fixtures/assert-expect/no-assertions.js:1:7 - at internal - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1`, - - 'qunit assert-expect/require-expects.js': -`TAP version 13 -not ok 1 passing test - --- - message: Expected number of assertions to be defined, but expect() was not called. - severity: failed - actual : null - expected: undefined - stack: | - at /qunit/test/cli/fixtures/assert-expect/require-expects.js:3:7 - at internal - ... -1..1 -# pass 0 -# skip 0 -# todo 0 -# fail 1 - -# exit code: 1` -}; diff --git a/test/cli/fixtures/fail/failure.js b/test/cli/fixtures/failure.js similarity index 100% rename from test/cli/fixtures/fail/failure.js rename to test/cli/fixtures/failure.js diff --git a/test/cli/fixtures/failure.tap.txt b/test/cli/fixtures/failure.tap.txt new file mode 100644 index 000000000..e2c5b0d7f --- /dev/null +++ b/test/cli/fixtures/failure.tap.txt @@ -0,0 +1,29 @@ +# name: test with failing assertion +# command: ["qunit","failure.js"] + +TAP version 13 +not ok 1 Failure > bad + --- + message: failed + severity: failed + actual : false + expected: true + stack: | + at /qunit/test/cli/fixtures/failure.js:3:16 + ... +not ok 2 Failure > bad again + --- + message: failed + severity: failed + actual : false + expected: true + stack: | + at /qunit/test/cli/fixtures/failure.js:7:16 + ... +1..2 +# pass 0 +# skip 0 +# todo 0 +# fail 2 + +# exit code: 1 \ No newline at end of file diff --git a/test/cli/fixtures/filter-modulename-insensitive.tap.txt b/test/cli/fixtures/filter-modulename-insensitive.tap.txt new file mode 100644 index 000000000..cb6188bce --- /dev/null +++ b/test/cli/fixtures/filter-modulename-insensitive.tap.txt @@ -0,0 +1,10 @@ +# name: --module selects a module (case-insensitive) +# command: ["qunit","--module","seconD","test/"] + +TAP version 13 +ok 1 Second > 1 +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 \ No newline at end of file diff --git a/test/cli/fixtures/filter-modulename.tap.txt b/test/cli/fixtures/filter-modulename.tap.txt new file mode 100644 index 000000000..9d4fb8b87 --- /dev/null +++ b/test/cli/fixtures/filter-modulename.tap.txt @@ -0,0 +1,10 @@ +# name: --filter matches module +# command: ["qunit", "--filter", "single", "test", "basic-one.js", "glob/**/*-test.js"] + +TAP version 13 +ok 1 Single > has a test +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/hanging-test/index.js b/test/cli/fixtures/hanging-test.js similarity index 100% rename from test/cli/fixtures/hanging-test/index.js rename to test/cli/fixtures/hanging-test.js diff --git a/test/cli/fixtures/hanging-test.tap.txt b/test/cli/fixtures/hanging-test.tap.txt new file mode 100644 index 000000000..449758e4a --- /dev/null +++ b/test/cli/fixtures/hanging-test.tap.txt @@ -0,0 +1,10 @@ +# name: test that hangs +# command: ["qunit","hanging-test.js"] + +TAP version 13 + +# stderr +Error: Process exited before tests finished running +Last test to run (hanging) has an async hold. Ensure all assert.async() callbacks are invoked and Promises resolve. You should also set a standard timeout via QUnit.config.testTimeout. + +# exit code: 1 \ No newline at end of file diff --git a/test/cli/fixtures/hooks-global-context.tap.txt b/test/cli/fixtures/hooks-global-context.tap.txt new file mode 100644 index 000000000..216df1cac --- /dev/null +++ b/test/cli/fixtures/hooks-global-context.tap.txt @@ -0,0 +1,12 @@ +# name: QUnit.hooks context +# command: ["qunit", "hooks-global-context.js"] + +TAP version 13 +ok 1 A > A1 +ok 2 A > AB > AB1 +ok 3 B +1..3 +# pass 3 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/incorrect-hooks-warning/test.js b/test/cli/fixtures/hooks-other-module-warning.js similarity index 100% rename from test/cli/fixtures/incorrect-hooks-warning/test.js rename to test/cli/fixtures/hooks-other-module-warning.js diff --git a/test/cli/fixtures/hooks-other-module-warning.tap.txt b/test/cli/fixtures/hooks-other-module-warning.tap.txt new file mode 100644 index 000000000..12ccd9317 --- /dev/null +++ b/test/cli/fixtures/hooks-other-module-warning.tap.txt @@ -0,0 +1,13 @@ +# name: hooks.beforeEach() during other module +# command: ["qunit", "hooks-other-module-warning.js"] + +TAP version 13 +ok 1 module providing hooks > module not providing hooks > has a test +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 + +# stderr +The `beforeEach` hook was called inside the wrong module (`module providing hooks > module not providing hooks`). Instead, use hooks provided by the callback to the containing module (`module providing hooks`). This will become an error in QUnit 3.0. diff --git a/test/cli/fixtures/module-nested.tap.txt b/test/cli/fixtures/module-nested.tap.txt new file mode 100644 index 000000000..dedf2a543 --- /dev/null +++ b/test/cli/fixtures/module-nested.tap.txt @@ -0,0 +1,11 @@ +# name: module() nested with interrupted executeNow +# command: ["qunit", "module-nested.js"] + +TAP version 13 +ok 1 module 1 > test in module 1 +ok 2 module 3 > test in module 3 +1..2 +# pass 2 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/assert-expect/no-tests.js b/test/cli/fixtures/no-tests-failOnZeroTests.js similarity index 100% rename from test/cli/fixtures/assert-expect/no-tests.js rename to test/cli/fixtures/no-tests-failOnZeroTests.js diff --git a/test/cli/fixtures/no-tests-failOnZeroTests.tap.txt b/test/cli/fixtures/no-tests-failOnZeroTests.tap.txt new file mode 100644 index 000000000..f5854f7dd --- /dev/null +++ b/test/cli/fixtures/no-tests-failOnZeroTests.tap.txt @@ -0,0 +1,9 @@ +# name: no tests and config.failOnZeroTests=false +# command: ["qunit", "no-tests-failOnZeroTests.js"] + +TAP version 13 +1..0 +# pass 0 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/no-tests/index.js b/test/cli/fixtures/no-tests.js similarity index 100% rename from test/cli/fixtures/no-tests/index.js rename to test/cli/fixtures/no-tests.js diff --git a/test/cli/fixtures/no-tests.tap.txt b/test/cli/fixtures/no-tests.tap.txt new file mode 100644 index 000000000..5dc50674f --- /dev/null +++ b/test/cli/fixtures/no-tests.tap.txt @@ -0,0 +1,22 @@ +# name: no tests +# command: ["qunit", "no-tests.js"] + +TAP version 13 +not ok 1 global failure + --- + message: No tests were run. + severity: failed + actual : undefined + expected: undefined + stack: | + Error: No tests were run. + at qunit.js + at internal + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 diff --git a/test/cli/fixtures/only/module-flat.js b/test/cli/fixtures/only-module-flat.js similarity index 100% rename from test/cli/fixtures/only/module-flat.js rename to test/cli/fixtures/only-module-flat.js diff --git a/test/cli/fixtures/only/module-then-test.js b/test/cli/fixtures/only-module-then-test.js similarity index 100% rename from test/cli/fixtures/only/module-then-test.js rename to test/cli/fixtures/only-module-then-test.js diff --git a/test/cli/fixtures/only-module-then-test.tap.txt b/test/cli/fixtures/only-module-then-test.tap.txt new file mode 100644 index 000000000..604e7af50 --- /dev/null +++ b/test/cli/fixtures/only-module-then-test.tap.txt @@ -0,0 +1,10 @@ +# name: module.only() followed by test +# command: ["qunit", "only-module-then-test.js"] + +TAP version 13 +ok 1 module A > module B > test B +1..2 +# pass 2 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/only/module.js b/test/cli/fixtures/only-module.js similarity index 100% rename from test/cli/fixtures/only/module.js rename to test/cli/fixtures/only-module.js diff --git a/test/cli/fixtures/only/test.js b/test/cli/fixtures/only-test.js similarity index 100% rename from test/cli/fixtures/only/test.js rename to test/cli/fixtures/only-test.js diff --git a/test/cli/fixtures/only-test.tap.txt b/test/cli/fixtures/only-test.tap.txt new file mode 100644 index 000000000..e6e19800e --- /dev/null +++ b/test/cli/fixtures/only-test.tap.txt @@ -0,0 +1,11 @@ +# name: test.only() +# command: ["qunit","only-test.js"] + +TAP version 13 +ok 1 run this test +ok 2 all tests with only run +1..3 +# pass 3 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/pending-async-after-timeout.tap.txt b/test/cli/fixtures/pending-async-after-timeout.tap.txt new file mode 100644 index 000000000..127374998 --- /dev/null +++ b/test/cli/fixtures/pending-async-after-timeout.tap.txt @@ -0,0 +1,20 @@ +# name: test with pending async after timeout +# command: ["qunit","pending-async-after-timeout.js"] + +TAP version 13 +not ok 1 example + --- + message: Test took longer than 10ms; test timed out. + severity: failed + actual : null + expected: undefined + stack: | + at internal + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 \ No newline at end of file diff --git a/test/cli/fixtures/require-module-and-script.tap.txt b/test/cli/fixtures/require-module-and-script.tap.txt new file mode 100644 index 000000000..15bbcdf2a --- /dev/null +++ b/test/cli/fixtures/require-module-and-script.tap.txt @@ -0,0 +1,12 @@ +# name: --require loads dependency and script +# command: ["qunit","basic-one.js","--require","require-dep","--require","./node_modules/require-dep/module.js"] + +required require-dep/index.js +required require-dep/module.js +TAP version 13 +ok 1 Single > has a test +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/seed.tap.txt b/test/cli/fixtures/seed.tap.txt new file mode 100644 index 000000000..170fa4ccd --- /dev/null +++ b/test/cli/fixtures/seed.tap.txt @@ -0,0 +1,15 @@ +# name: --seed value +# command: ["qunit", "--seed", "s33d", "test", "basic-one.js", "glob/**/*-test.js"] + +Running tests with seed: s33d +TAP version 13 +ok 1 Second > 1 +ok 2 Nested-Test > herp +ok 3 First > 1 +ok 4 A-Test > derp +ok 5 Single > has a test +1..5 +# pass 5 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/fixtures/syntax-error/test.js b/test/cli/fixtures/syntax-error.js similarity index 100% rename from test/cli/fixtures/syntax-error/test.js rename to test/cli/fixtures/syntax-error.js diff --git a/test/cli/fixtures/syntax-error.tap.txt b/test/cli/fixtures/syntax-error.tap.txt new file mode 100644 index 000000000..fa9f317cb --- /dev/null +++ b/test/cli/fixtures/syntax-error.tap.txt @@ -0,0 +1,34 @@ +# name: load file with syntax error +# command: ["qunit", "syntax-error.js"] + +not ok 1 global failure + --- + message: |+ + Error: Failed to load file syntax-error.js + ReferenceError: varIsNotDefined is not defined + severity: failed + stack: | + ReferenceError: varIsNotDefined is not defined + at /qunit/test/cli/fixtures/syntax-error.js:1:1 + at internal + ... +Bail out! Error: Failed to load file syntax-error.js +TAP version 13 +not ok 2 global failure + --- + message: No tests were run. + severity: failed + actual : undefined + expected: undefined + stack: | + Error: No tests were run. + at qunit.js + at internal + ... +1..2 +# pass 0 +# skip 0 +# todo 0 +# fail 2 + +# exit code: 1 diff --git a/test/cli/fixtures/timeout/index.js b/test/cli/fixtures/timeout.js similarity index 100% rename from test/cli/fixtures/timeout/index.js rename to test/cli/fixtures/timeout.js diff --git a/test/cli/fixtures/timeout.tap.txt b/test/cli/fixtures/timeout.tap.txt new file mode 100644 index 000000000..5b66bd991 --- /dev/null +++ b/test/cli/fixtures/timeout.tap.txt @@ -0,0 +1,21 @@ +# name: two tests with one timeout +# command: ["qunit", "timeout.js"] + +TAP version 13 +not ok 1 timeout > first + --- + message: Test took longer than 10ms; test timed out. + severity: failed + actual : null + expected: undefined + stack: | + at internal + ... +ok 2 timeout > second +1..2 +# pass 1 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 diff --git a/test/cli/fixtures/hard-error-in-test-with-no-async-handler.js b/test/cli/fixtures/uncaught-error-after-assert-async.js similarity index 100% rename from test/cli/fixtures/hard-error-in-test-with-no-async-handler.js rename to test/cli/fixtures/uncaught-error-after-assert-async.js diff --git a/test/cli/fixtures/uncaught-error-after-assert-async.tap.txt b/test/cli/fixtures/uncaught-error-after-assert-async.tap.txt new file mode 100644 index 000000000..9d6533222 --- /dev/null +++ b/test/cli/fixtures/uncaught-error-after-assert-async.tap.txt @@ -0,0 +1,24 @@ +# name: uncaught error after assert.async() +# command: ["qunit", "uncaught-error-after-assert-async.js"] + +TAP version 13 +not ok 1 contains a hard error after using assert.async() + --- + message: |+ + Died on test #2: expected error thrown in test + at /qunit/test/cli/fixtures/uncaught-error-after-assert-async.js:1:7 + at internal + severity: failed + actual : null + expected: undefined + stack: | + Error: expected error thrown in test + at /qunit/test/cli/fixtures/uncaught-error-after-assert-async.js:4:9 + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 diff --git a/test/cli/fixtures/bad-callbacks/moduleDone-throw.js b/test/cli/fixtures/uncaught-error-callback-moduleDone.js similarity index 100% rename from test/cli/fixtures/bad-callbacks/moduleDone-throw.js rename to test/cli/fixtures/uncaught-error-callback-moduleDone.js diff --git a/test/cli/fixtures/uncaught-error-callback-moduleDone.tap.txt b/test/cli/fixtures/uncaught-error-callback-moduleDone.tap.txt new file mode 100644 index 000000000..cf678e4be --- /dev/null +++ b/test/cli/fixtures/uncaught-error-callback-moduleDone.tap.txt @@ -0,0 +1,10 @@ +# name: uncaught error in "moduleDone" callback +# command: ["qunit","uncaught-error-callback-moduleDone.js"] + +TAP version 13 +ok 1 module1 > test1 + +# stderr +Error: Process exited before tests finished running + +# exit code: 1 diff --git a/test/cli/fixtures/bad-callbacks/testStart-throw.js b/test/cli/fixtures/uncaught-error-callback-testStart.js similarity index 100% rename from test/cli/fixtures/bad-callbacks/testStart-throw.js rename to test/cli/fixtures/uncaught-error-callback-testStart.js diff --git a/test/cli/fixtures/uncaught-error-callback-testStart.tap.txt b/test/cli/fixtures/uncaught-error-callback-testStart.tap.txt new file mode 100644 index 000000000..411217a77 --- /dev/null +++ b/test/cli/fixtures/uncaught-error-callback-testStart.tap.txt @@ -0,0 +1,9 @@ +# name: uncaught error in "testStart" callback +# command: ["qunit","uncaught-error-callback-testStart.js"] + +TAP version 13 + +# stderr +Error: Process exited before tests finished running + +# exit code: 1 diff --git a/test/cli/fixtures/bad-callbacks/begin-throw.js b/test/cli/fixtures/uncaught-error-callbacks-begin.js similarity index 100% rename from test/cli/fixtures/bad-callbacks/begin-throw.js rename to test/cli/fixtures/uncaught-error-callbacks-begin.js diff --git a/test/cli/fixtures/uncaught-error-callbacks-begin.tap.txt b/test/cli/fixtures/uncaught-error-callbacks-begin.tap.txt new file mode 100644 index 000000000..9d739d0c0 --- /dev/null +++ b/test/cli/fixtures/uncaught-error-callbacks-begin.tap.txt @@ -0,0 +1,20 @@ +# name: uncaught error in "begin" callback +# command: ["qunit", "uncaught-error-callbacks-begin.js"] + +TAP version 13 +not ok 1 global failure + --- + message: Error: No dice + severity: failed + stack: | + Error: No dice + at /qunit/test/cli/fixtures/uncaught-error-callbacks-begin.js:2:9 + at qunit.js + at internal + ... +Bail out! Error: No dice + +# stderr +Error: Process exited before tests finished running + +# exit code: 1 diff --git a/test/cli/fixtures/bad-callbacks/done-throw.js b/test/cli/fixtures/uncaught-error-callbacks-done.js similarity index 100% rename from test/cli/fixtures/bad-callbacks/done-throw.js rename to test/cli/fixtures/uncaught-error-callbacks-done.js diff --git a/test/cli/fixtures/uncaught-error-callbacks-done.tap.txt b/test/cli/fixtures/uncaught-error-callbacks-done.tap.txt new file mode 100644 index 000000000..ff4fc094c --- /dev/null +++ b/test/cli/fixtures/uncaught-error-callbacks-done.tap.txt @@ -0,0 +1,22 @@ +# name: uncaught error in "done" callback +# command: ["qunit", "uncaught-error-callbacks-done.js"] + +TAP version 13 +ok 1 module1 > test1 +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 +Bail out! Error: No dice + --- + message: Error: No dice + severity: failed + stack: | + Error: No dice + at /qunit/test/cli/fixtures/uncaught-error-callbacks-done.js:2:9 + at qunit.js + at internal + ... + +# exit code: 1 diff --git a/test/cli/fixtures/hard-error-in-hook.js b/test/cli/fixtures/uncaught-error-in-hook.js similarity index 100% rename from test/cli/fixtures/hard-error-in-hook.js rename to test/cli/fixtures/uncaught-error-in-hook.js diff --git a/test/cli/fixtures/uncaught-error-in-hook.tap.txt b/test/cli/fixtures/uncaught-error-in-hook.tap.txt new file mode 100644 index 000000000..3126b2169 --- /dev/null +++ b/test/cli/fixtures/uncaught-error-in-hook.tap.txt @@ -0,0 +1,21 @@ +# name: uncaught error in hook +# command: ["qunit", "uncaught-error-in-hook.js"] + +TAP version 13 +not ok 1 contains a hard error in hook > contains a hard error + --- + message: before failed on contains a hard error: expected error thrown in hook + severity: failed + actual : null + expected: undefined + stack: | + Error: expected error thrown in hook + at /qunit/test/cli/fixtures/uncaught-error-in-hook.js:3:11 + ... +1..1 +# pass 0 +# skip 0 +# todo 0 +# fail 1 + +# exit code: 1 diff --git a/test/cli/fixtures/unhandled-rejection.tap.txt b/test/cli/fixtures/unhandled-rejection.tap.txt new file mode 100644 index 000000000..e9afdcfa5 --- /dev/null +++ b/test/cli/fixtures/unhandled-rejection.tap.txt @@ -0,0 +1,34 @@ +# name: unhandled rejection +# command: ["qunit","unhandled-rejection.js"] + +not ok 1 global failure + --- + message: Error: outside of a test context + severity: failed + stack: | + Error: outside of a test context + at /qunit/test/cli/fixtures/unhandled-rejection.js:17:18 + at qunit.js + at /qunit/test/cli/fixtures/unhandled-rejection.js:3:7 + at internal + ... +Bail out! Error: outside of a test context +TAP version 13 +not ok 2 Unhandled Rejections > test passes just fine, but has a rejected promise + --- + message: global failure: Error: Error thrown in non-returned promise! + severity: failed + actual : undefined + expected: undefined + stack: | + Error: Error thrown in non-returned promise! + at /qunit/test/cli/fixtures/unhandled-rejection.js:10:13 + at internal + ... +1..2 +# pass 0 +# skip 0 +# todo 0 +# fail 2 + +# exit code: 1 diff --git a/test/cli/fixtures/zero-assertions.tap.txt b/test/cli/fixtures/zero-assertions.tap.txt new file mode 100644 index 000000000..076cdbc66 --- /dev/null +++ b/test/cli/fixtures/zero-assertions.tap.txt @@ -0,0 +1,10 @@ +# name: test with zero assertions +# command: ["qunit", "zero-assertions.js"] + +TAP version 13 +ok 1 Zero assertions > has a test +1..1 +# pass 1 +# skip 0 +# todo 0 +# fail 0 diff --git a/test/cli/helpers/execute.js b/test/cli/helpers/execute.js index 99d167598..f73711f62 100644 --- a/test/cli/helpers/execute.js +++ b/test/cli/helpers/execute.js @@ -147,34 +147,38 @@ async function execute (command, options = {}, hook) { return result; } -// Very loose command formatter. -// Not for the purpose of execution, but for the purpose -// of formatting the string key in fixtures/ files. -function prettyPrintCommand (command) { - return command.map(arg => { - // Quote spaces and stars - return /[ *]/.test(arg) ? `'${arg}'` : arg; - }).join(' '); -} - /** * @param {Array} input * @param {number} [concurrency=0] * @return {Array} */ function concurrentMap (input, concurrency, asyncFn) { - const ret = []; if (!concurrency) { concurrency = os.cpus().length; } if (concurrency < 1) { throw new Error('Concurrency must be non-zero'); } + const queue = []; + const ret = []; + function next () { + if (queue.length) { + queue.shift()(); + } + } for (let i = 0; i < input.length; i++) { const val = input[i]; - ret[i] = (i < concurrency) - ? Promise.resolve(asyncFn(val)) - : ret[i - concurrency].then(asyncFn.bind(null, val)); + if (i < concurrency) { + ret[i] = Promise.resolve(asyncFn(val)).finally(next); + } else { + let trigger; + const promise = new Promise((resolve) => { + trigger = resolve; + }); + queue.push(trigger); + + ret[i] = promise.then(asyncFn.bind(null, val)).finally(next); + } } return ret; } @@ -199,7 +203,6 @@ function concurrentMapKeys (input, concurrency, asyncFn) { module.exports = { normalize, execute, - prettyPrintCommand, concurrentMap, concurrentMapKeys }; diff --git a/test/cli/helpers/fixtures.js b/test/cli/helpers/fixtures.js new file mode 100644 index 000000000..ee816c9a2 --- /dev/null +++ b/test/cli/helpers/fixtures.js @@ -0,0 +1,48 @@ +'use strict'; + +const path = require('path'); +const fsPromises = require('fs').promises; +const glob = require('tiny-glob/sync'); +const { execute } = require('./execute.js'); + +const NAME_DIRECTIVE = '# name: '; +const CMD_DIRECTIVE = '# command: '; + +async function parseFixture (file) { + const contents = await fsPromises.readFile(file, 'utf8'); + const lines = contents.replace(/\r\n/g, '\n').split('\n'); + if (!lines[0].startsWith(NAME_DIRECTIVE)) { + throw new Error('Name declaration missing'); + } + const name = lines[0].slice(NAME_DIRECTIVE.length); + if (!lines[1].startsWith(CMD_DIRECTIVE)) { + throw new Error('Command declaration missing'); + } + const command = JSON.parse(lines[1].slice(CMD_DIRECTIVE.length)); + return { + name, + command, + expected: lines.slice(2).join('\n').trim() + }; +} + +function readFixtures (dir) { + const fixtures = {}; + + for (const file of glob('*.tap.txt', { cwd: dir, absolute: true })) { + fixtures[path.basename(file, '.tap.txt')] = async function runFixture () { + const fixture = await parseFixture(file); + const result = await execute(fixture.command); + return { + expected: fixture.expected, + snapshot: result.snapshot + }; + }; + } + + return fixtures; +} + +module.exports = { + readFixtures +}; diff --git a/test/cli/utils.js b/test/cli/utils.js index d339b0c78..113b57c2d 100644 --- a/test/cli/utils.js +++ b/test/cli/utils.js @@ -202,9 +202,9 @@ QUnit.module('CLI utils', function () { 4, 4, 4, + 4, // no more remaining - 3, - 1 + 4 ]); }); });