From d26d9e738b45a216bb6e013ce36bfd2479126645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Fri, 28 Jun 2024 14:32:50 +0200 Subject: [PATCH] =?UTF-8?q?[test-visibility]=C2=A0Add=20support=20for=20vi?= =?UTF-8?q?test=20(#4415)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/project.yml | 17 ++ docs/test.ts | 2 + ext/exporters.d.ts | 2 +- index.d.ts | 9 +- .../ci-visibility/vitest-tests/sum.mjs | 3 + .../test-visibility-failed-hooks.mjs | 26 ++ .../test-visibility-failed-suite.mjs | 29 ++ .../test-visibility-passed-suite.mjs | 32 ++ integration-tests/vitest.config.mjs | 9 + integration-tests/vitest/vitest.spec.js | 137 +++++++++ package.json | 1 + packages/datadog-esbuild/index.js | 6 +- .../src/helpers/hook.js | 11 +- .../src/helpers/hooks.js | 2 + .../src/helpers/instrument.js | 7 +- .../src/helpers/register.js | 30 +- .../datadog-instrumentations/src/vitest.js | 282 ++++++++++++++++++ packages/datadog-plugin-vitest/src/index.js | 156 ++++++++++ packages/dd-trace/src/plugins/ci_plugin.js | 7 + packages/dd-trace/src/plugins/index.js | 2 + 20 files changed, 757 insertions(+), 13 deletions(-) create mode 100644 integration-tests/ci-visibility/vitest-tests/sum.mjs create mode 100644 integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs create mode 100644 integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs create mode 100644 integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs create mode 100644 integration-tests/vitest.config.mjs create mode 100644 integration-tests/vitest/vitest.spec.js create mode 100644 packages/datadog-instrumentations/src/vitest.js create mode 100644 packages/datadog-plugin-vitest/src/index.js diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml index a74051512ba..f16227e81a6 100644 --- a/.github/workflows/project.yml +++ b/.github/workflows/project.yml @@ -110,6 +110,23 @@ jobs: CYPRESS_VERSION: ${{ matrix.cypress-version }} NODE_OPTIONS: '-r ./ci/init' + integration-vitest: + runs-on: ubuntu-latest + env: + DD_SERVICE: dd-trace-js-integration-tests + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1 + DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/node/setup + - run: yarn install + - uses: actions/setup-node@v3 + with: + node-version: 20 + - run: yarn test:integration:vitest + env: + NODE_OPTIONS: '-r ./ci/init' + lint: runs-on: ubuntu-latest steps: diff --git a/docs/test.ts b/docs/test.ts index 380bfd6bba9..9b11d344841 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -363,6 +363,8 @@ tracer.use('sharedb'); tracer.use('sharedb', sharedbOptions); tracer.use('tedious'); tracer.use('undici'); +tracer.use('vitest'); +tracer.use('vitest', { service: 'vitest-service' }); tracer.use('winston'); tracer.use('express', false) diff --git a/ext/exporters.d.ts b/ext/exporters.d.ts index 07bc2cd29e3..2f462dd93e7 100644 --- a/ext/exporters.d.ts +++ b/ext/exporters.d.ts @@ -4,7 +4,7 @@ declare const exporters: { DATADOG: 'datadog', AGENT_PROXY: 'agent_proxy', JEST_WORKER: 'jest_worker', - CUCUMBER_WORKER: 'cucumber_worker' + CUCUMBER_WORKER: 'cucumber_worker', MOCHA_WORKER: 'mocha_worker' } diff --git a/index.d.ts b/index.d.ts index d0b634f5dd8..7d5846f8af2 100644 --- a/index.d.ts +++ b/index.d.ts @@ -198,6 +198,7 @@ interface Plugins { "sharedb": tracer.plugins.sharedb; "tedious": tracer.plugins.tedious; "undici": tracer.plugins.undici; + "vitest": tracer.plugins.vitest; "winston": tracer.plugins.winston; } @@ -1556,7 +1557,7 @@ declare namespace tracer { /** * This plugin automatically instruments the - * [jest](https://github.com/facebook/jest) module. + * [jest](https://github.com/jestjs/jest) module. */ interface jest extends Integration {} @@ -1839,6 +1840,12 @@ declare namespace tracer { */ interface undici extends HttpClient {} + /** + * This plugin automatically instruments the + * [vitest](https://github.com/vitest-dev/vitest) module. + */ + interface vitest extends Integration {} + /** * This plugin patches the [winston](https://github.com/winstonjs/winston) * to automatically inject trace identifiers in log records when the diff --git a/integration-tests/ci-visibility/vitest-tests/sum.mjs b/integration-tests/ci-visibility/vitest-tests/sum.mjs new file mode 100644 index 00000000000..f1c6520acbd --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/sum.mjs @@ -0,0 +1,3 @@ +export function sum (a, b) { + return a + b +} diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs new file mode 100644 index 00000000000..a97f95e0df1 --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs @@ -0,0 +1,26 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { sum } from './sum' + +describe('context', () => { + beforeEach(() => { + throw new Error('failed before each') + }) + test('can report failed test', () => { + expect(sum(1, 2)).to.equal(4) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) + +describe('other context', () => { + afterEach(() => { + throw new Error('failed after each') + }) + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs new file mode 100644 index 00000000000..f2df345a87f --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs @@ -0,0 +1,29 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { sum } from './sum' + +let preparedValue = 1 + +describe('test-visibility-failed-suite-first-describe', () => { + beforeEach(() => { + preparedValue = 2 + }) + test('can report failed test', () => { + expect(sum(1, 2)).to.equal(4) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + expect(preparedValue).to.equal(2) + }) +}) + +describe('test-visibility-failed-suite-second-describe', () => { + afterEach(() => { + preparedValue = 1 + }) + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs new file mode 100644 index 00000000000..c2cf93431d8 --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs @@ -0,0 +1,32 @@ +import { describe, test, expect } from 'vitest' +import { sum } from './sum' + +describe('context', () => { + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) + +describe('other context', () => { + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) + test.skip('can skip', () => { + expect(sum(1, 2)).to.equal(3) + }) + test.todo('can todo', () => { + expect(sum(1, 2)).to.equal(3) + }) + // eslint-disable-next-line + test('can programmatic skip', (context) => { + // eslint-disable-next-line + context.skip() + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/vitest.config.mjs b/integration-tests/vitest.config.mjs new file mode 100644 index 00000000000..f04d63785fd --- /dev/null +++ b/integration-tests/vitest.config.mjs @@ -0,0 +1,9 @@ +import { defineConfig } from 'vite' + +export default defineConfig({ + test: { + include: [ + 'ci-visibility/vitest-tests/test-visibility*' + ] + } +}) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js new file mode 100644 index 00000000000..4a3151c2b72 --- /dev/null +++ b/integration-tests/vitest/vitest.spec.js @@ -0,0 +1,137 @@ +'use strict' + +const { exec } = require('child_process') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_STATUS, + TEST_TYPE +} = require('../../packages/dd-trace/src/plugins/util/test') + +// tested with 1.6.0 +const versions = ['latest'] + +versions.forEach((version) => { + describe(`vitest@${version}`, () => { + let sandbox, cwd, receiver, childProcess + + before(async function () { + sandbox = await createSandbox([`vitest@${version}`], true) + cwd = sandbox.folder + }) + + after(async () => { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + childProcess.kill() + await receiver.stop() + }) + + it('can run and report tests', (done) => { + receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSessionEvent = events.find(event => event.type === 'test_session_end') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + const testEvents = events.filter(event => event.type === 'test') + + assert.include(testSessionEvent.content.resource, 'test_session.vitest run') + assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail') + assert.include(testModuleEvent.content.resource, 'test_module.vitest run') + assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail') + assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test') + assert.equal(testModuleEvent.content.meta[TEST_TYPE], 'test') + + const passedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-passed-suite.mjs' + ) + assert.equal(passedSuite.content.meta[TEST_STATUS], 'pass') + + const failedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + ) + assert.equal(failedSuite.content.meta[TEST_STATUS], 'fail') + + const failedSuiteHooks = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs' + ) + assert.equal(failedSuiteHooks.content.meta[TEST_STATUS], 'fail') + + assert.includeMembers(testEvents.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' + ] + ) + + const failedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'fail') + + assert.includeMembers( + failedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' + ] + ) + + const skippedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'skip') + + assert.includeMembers( + skippedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip' + ] + ) + // TODO: check error messages + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + // maybe only in node@20 + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' // ESM requires more flags + }, + stdio: 'pipe' + } + ) + }) + }) +}) diff --git a/package.json b/package.json index e5a16b4d2c5..1fc31a5eafe 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "test:integration:cypress": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/cypress/*.spec.js\"", "test:integration:playwright": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/playwright/*.spec.js\"", "test:integration:selenium": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/selenium/*.spec.js\"", + "test:integration:vitest": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/vitest/*.spec.js\"", "test:integration:profiler": "mocha --colors --timeout 180000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/profiler/*.spec.js\"", "test:integration:serverless": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/serverless/*.spec.js\"", "test:integration:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", diff --git a/packages/datadog-esbuild/index.js b/packages/datadog-esbuild/index.js index 95a0e8ddd16..ce263799023 100644 --- a/packages/datadog-esbuild/index.js +++ b/packages/datadog-esbuild/index.js @@ -7,7 +7,11 @@ const hooks = require('../datadog-instrumentations/src/helpers/hooks.js') const extractPackageAndModulePath = require('../datadog-instrumentations/src/utils/src/extract-package-and-module-path') for (const hook of Object.values(hooks)) { - hook() + if (typeof hook === 'object') { + hook.fn() + } else { + hook() + } } const modulesOfInterest = new Set() diff --git a/packages/datadog-instrumentations/src/helpers/hook.js b/packages/datadog-instrumentations/src/helpers/hook.js index 7bec453187a..0177744ea1c 100644 --- a/packages/datadog-instrumentations/src/helpers/hook.js +++ b/packages/datadog-instrumentations/src/helpers/hook.js @@ -11,8 +11,13 @@ const ritm = require('../../../dd-trace/src/ritm') * @param {string[]} modules list of modules to hook into * @param {Function} onrequire callback to be executed upon encountering module */ -function Hook (modules, onrequire) { - if (!(this instanceof Hook)) return new Hook(modules, onrequire) +function Hook (modules, hookOptions, onrequire) { + if (!(this instanceof Hook)) return new Hook(modules, hookOptions, onrequire) + + if (typeof hookOptions === 'function') { + onrequire = hookOptions + hookOptions = {} + } this._patched = Object.create(null) @@ -28,7 +33,7 @@ function Hook (modules, onrequire) { } this._ritmHook = ritm(modules, {}, safeHook) - this._iitmHook = iitm(modules, {}, (moduleExports, moduleName, moduleBaseDir) => { + this._iitmHook = iitm(modules, hookOptions, (moduleExports, moduleName, moduleBaseDir) => { // TODO: Move this logic to import-in-the-middle and only do it for CommonJS // modules and not ESM. In the meantime, all the modules we instrument are // CommonJS modules for which the default export is always moved to diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index 0723ceabd84..94f3318fb62 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -23,6 +23,7 @@ module.exports = { '@opentelemetry/sdk-trace-node': () => require('../otel-sdk-trace'), '@redis/client': () => require('../redis'), '@smithy/smithy-client': () => require('../aws-sdk'), + '@vitest/runner': { esmFirst: true, fn: () => require('../vitest') }, aerospike: () => require('../aerospike'), amqp10: () => require('../amqp10'), amqplib: () => require('../amqplib'), @@ -110,6 +111,7 @@ module.exports = { sharedb: () => require('../sharedb'), tedious: () => require('../tedious'), undici: () => require('../undici'), + vitest: { esmFirst: true, fn: () => require('../vitest') }, when: () => require('../when'), winston: () => require('../winston') } diff --git a/packages/datadog-instrumentations/src/helpers/instrument.js b/packages/datadog-instrumentations/src/helpers/instrument.js index 0889f1e5402..20657335044 100644 --- a/packages/datadog-instrumentations/src/helpers/instrument.js +++ b/packages/datadog-instrumentations/src/helpers/instrument.js @@ -17,10 +17,11 @@ exports.channel = function (name) { /** * @param {string} args.name module name * @param {string[]} args.versions array of semver range strings - * @param {string} args.file path to file within package to instrument? + * @param {string} args.file path to file within package to instrument + * @param {string} args.filePattern pattern to match files within package to instrument * @param Function hook */ -exports.addHook = function addHook ({ name, versions, file }, hook) { +exports.addHook = function addHook ({ name, versions, file, filePattern }, hook) { if (typeof name === 'string') { name = [name] } @@ -29,7 +30,7 @@ exports.addHook = function addHook ({ name, versions, file }, hook) { if (!instrumentations[val]) { instrumentations[val] = [] } - instrumentations[val].push({ name: val, versions, file, hook }) + instrumentations[val].push({ name: val, versions, file, filePattern, hook }) } } diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index 2850664c8e2..e45a0c0cd14 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -42,19 +42,32 @@ const seenCombo = new Set() for (const packageName of names) { if (disabledInstrumentations.has(packageName)) continue - Hook([packageName], (moduleExports, moduleName, moduleBaseDir, moduleVersion) => { + const hookOptions = {} + + let hook = hooks[packageName] + + if (typeof hook === 'object') { + hookOptions.internals = hook.esmFirst + hook = hook.fn + } + + Hook([packageName], hookOptions, (moduleExports, moduleName, moduleBaseDir, moduleVersion) => { moduleName = moduleName.replace(pathSepExpr, '/') // This executes the integration file thus adding its entries to `instrumentations` - hooks[packageName]() + hook() if (!instrumentations[packageName]) { return moduleExports } const namesAndSuccesses = {} - for (const { name, file, versions, hook } of instrumentations[packageName]) { + for (const { name, file, versions, hook, filePattern } of instrumentations[packageName]) { + let fullFilePattern = filePattern const fullFilename = filename(name, file) + if (fullFilePattern) { + fullFilePattern = filename(name, fullFilePattern) + } // Create a WeakMap associated with the hook function so that patches on the same moduleExport only happens once // for example by instrumenting both dns and node:dns double the spans would be created @@ -62,8 +75,17 @@ for (const packageName of names) { if (!hook[HOOK_SYMBOL]) { hook[HOOK_SYMBOL] = new WeakMap() } + let matchesFile = false + + matchesFile = moduleName === fullFilename + + if (fullFilePattern) { + // Some libraries include a hash in their filenames when installed, + // so our instrumentation has to include a '.*' to match them for more than a single version. + matchesFile = matchesFile || new RegExp(fullFilePattern).test(moduleName) + } - if (moduleName === fullFilename) { + if (matchesFile) { const version = moduleVersion || getVersion(moduleBaseDir) if (!Object.hasOwnProperty(namesAndSuccesses, name)) { namesAndSuccesses[name] = { diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js new file mode 100644 index 00000000000..42f32b1ac42 --- /dev/null +++ b/packages/datadog-instrumentations/src/vitest.js @@ -0,0 +1,282 @@ +const { addHook, channel, AsyncResource } = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') + +// test hooks +const testStartCh = channel('ci:vitest:test:start') +const testFinishTimeCh = channel('ci:vitest:test:finish-time') +const testPassCh = channel('ci:vitest:test:pass') +const testErrorCh = channel('ci:vitest:test:error') +const testSkipCh = channel('ci:vitest:test:skip') + +// test suite hooks +const testSuiteStartCh = channel('ci:vitest:test-suite:start') +const testSuiteFinishCh = channel('ci:vitest:test-suite:finish') +const testSuiteErrorCh = channel('ci:vitest:test-suite:error') + +// test session hooks +const testSessionStartCh = channel('ci:vitest:session:start') +const testSessionFinishCh = channel('ci:vitest:session:finish') + +const taskToAsync = new WeakMap() + +const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') + +function isReporterPackage (vitestPackage) { + return vitestPackage.B?.name === 'BaseSequencer' +} + +function getSessionStatus (state) { + if (state.getCountOfFailedTests() > 0) { + return 'fail' + } + if (state.pathsSet.size === 0) { + return 'skip' + } + return 'pass' +} + +// eslint-disable-next-line +// From https://github.com/vitest-dev/vitest/blob/51c04e2f44d91322b334f8ccbcdb368facc3f8ec/packages/runner/src/run.ts#L243-L250 +function getVitestTestStatus (test, retryCount) { + if (test.result.state !== 'fail') { + if (!test.repeats) { + return 'pass' + } else if (test.repeats && (test.retry ?? 0) === retryCount) { + return 'pass' + } + } + return 'fail' +} + +function getTypeTasks (fileTasks, type = 'test') { + const typeTasks = [] + + function getTasks (tasks) { + for (const task of tasks) { + if (task.type === type) { + typeTasks.push(task) + } else if (task.tasks) { + getTasks(task.tasks) + } + } + } + + getTasks(fileTasks) + + return typeTasks +} + +function getTestName (task) { + let testName = task.name + let currentTask = task.suite + + while (currentTask) { + if (currentTask.name) { + testName = `${currentTask.name} ${testName}` + } + currentTask = currentTask.suite + } + + return testName +} + +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + file: 'dist/runners.js' +}, (vitestPackage) => { + const { VitestTestRunner } = vitestPackage + // test start (only tests that are not marked as skip or todo) + shimmer.wrap(VitestTestRunner.prototype, 'onBeforeTryTask', onBeforeTryTask => async function (task) { + if (!testStartCh.hasSubscribers) { + return onBeforeTryTask.apply(this, arguments) + } + const asyncResource = new AsyncResource('bound-anonymous-fn') + taskToAsync.set(task, asyncResource) + + asyncResource.runInAsyncScope(() => { + testStartCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + }) + return onBeforeTryTask.apply(this, arguments) + }) + + // test finish (only passed tests) + shimmer.wrap(VitestTestRunner.prototype, 'onAfterTryTask', onAfterTryTask => + async function (task, { retry: retryCount }) { + if (!testFinishTimeCh.hasSubscribers) { + return onAfterTryTask.apply(this, arguments) + } + const result = await onAfterTryTask.apply(this, arguments) + + const status = getVitestTestStatus(task, retryCount) + const asyncResource = taskToAsync.get(task) + + if (asyncResource) { + // We don't finish here because the test might fail in a later hook + asyncResource.runInAsyncScope(() => { + testFinishTimeCh.publish({ status, task }) + }) + } + + return result + }) + + return vitestPackage +}) + +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + filePattern: 'dist/vendor/index.*' +}, (vitestPackage) => { + // there are multiple index* files so we have to check the exported values + if (!isReporterPackage(vitestPackage)) { + return vitestPackage + } + shimmer.wrap(vitestPackage.B.prototype, 'sort', sort => async function () { + if (!testSessionFinishCh.hasSubscribers) { + return sort.apply(this, arguments) + } + shimmer.wrap(this.ctx, 'exit', exit => async function () { + let onFinish + + const flushPromise = new Promise(resolve => { + onFinish = resolve + }) + const failedSuites = this.state.getFailedFilepaths() + let error + if (failedSuites.length) { + error = new Error(`Test suites failed: ${failedSuites.length}.`) + } + + sessionAsyncResource.runInAsyncScope(() => { + testSessionFinishCh.publish({ + status: getSessionStatus(this.state), + onFinish, + error + }) + }) + + await flushPromise + + return exit.apply(this, arguments) + }) + + return sort.apply(this, arguments) + }) + + return vitestPackage +}) + +// Can't specify file because compiled vitest includes hashes in their files +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + filePattern: 'dist/vendor/cac.*' +}, (vitestPackage, frameworkVersion) => { + shimmer.wrap(vitestPackage, 'c', oldCreateCli => function () { + if (!testSessionStartCh.hasSubscribers) { + return oldCreateCli.apply(this, arguments) + } + sessionAsyncResource.runInAsyncScope(() => { + const processArgv = process.argv.slice(2).join(' ') + testSessionStartCh.publish({ command: `vitest ${processArgv}`, frameworkVersion }) + }) + return oldCreateCli.apply(this, arguments) + }) + + return vitestPackage +}) + +// test suite start and finish +// only relevant for workers +addHook({ + name: '@vitest/runner', + versions: ['>=1.6.0'], + file: 'dist/index.js' +}, vitestPackage => { + shimmer.wrap(vitestPackage, 'startTests', startTests => async function (testPath) { + let testSuiteError = null + if (!testSuiteStartCh.hasSubscribers) { + return startTests.apply(this, arguments) + } + + const testSuiteAsyncResource = new AsyncResource('bound-anonymous-fn') + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish(testPath[0]) + }) + const startTestsResponse = await startTests.apply(this, arguments) + + let onFinish = null + const onFinishPromise = new Promise(resolve => { + onFinish = resolve + }) + + const testTasks = getTypeTasks(startTestsResponse[0].tasks) + + testTasks.forEach(task => { + const testAsyncResource = taskToAsync.get(task) + const { result } = task + + if (result) { + const { state, duration, errors } = result + if (state === 'skip') { // programmatic skip + testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + } else if (state === 'pass') { + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + testPassCh.publish({ task }) + }) + } + } else if (state === 'fail') { + // If it's failing, we have no accurate finish time, so we have to use `duration` + let testError + + if (errors?.length) { + testError = errors[0] + } + + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + testErrorCh.publish({ duration, error: testError }) + }) + } + if (errors?.length) { + testSuiteError = testError // we store the error to bubble it up to the suite + } + } + } else { // test.skip or test.todo + testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + } + }) + + const testSuiteResult = startTestsResponse[0].result + + if (testSuiteResult.errors?.length) { // Errors from root level hooks + testSuiteError = testSuiteResult.errors[0] + } else if (testSuiteResult.state === 'fail') { // Errors from `describe` level hooks + const suiteTasks = getTypeTasks(startTestsResponse[0].tasks, 'suite') + const failedSuites = suiteTasks.filter(task => task.result?.state === 'fail') + if (failedSuites.length && failedSuites[0].result?.errors?.length) { + testSuiteError = failedSuites[0].result.errors[0] + } + } + + if (testSuiteError) { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish({ error: testSuiteError }) + }) + } + + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish({ status: testSuiteResult.state, onFinish }) + }) + + // TODO: fix too frequent flushes + await onFinishPromise + + return startTestsResponse + }) + + return vitestPackage +}) diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js new file mode 100644 index 00000000000..c47467528e6 --- /dev/null +++ b/packages/datadog-plugin-vitest/src/index.js @@ -0,0 +1,156 @@ +const CiPlugin = require('../../dd-trace/src/plugins/ci_plugin') +const { storage } = require('../../datadog-core') + +const { + TEST_STATUS, + finishAllTraceSpans, + getTestSuitePath, + getTestSuiteCommonTags, + TEST_SOURCE_FILE +} = require('../../dd-trace/src/plugins/util/test') +const { COMPONENT } = require('../../dd-trace/src/constants') + +// Milliseconds that we subtract from the error test duration +// so that they do not overlap with the following test +// This is because there's some loss of resolution. +const MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION = 5 + +class VitestPlugin extends CiPlugin { + static get id () { + return 'vitest' + } + + constructor (...args) { + super(...args) + + this.taskToFinishTime = new WeakMap() + + this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const store = storage.getStore() + const span = this.startTestSpan( + testName, + testSuite, + this.testSuiteSpan, + { + [TEST_SOURCE_FILE]: testSuite + } + ) + + this.enter(span, store) + }) + + this.addSub('ci:vitest:test:finish-time', ({ status, task }) => { + const store = storage.getStore() + const span = store?.span + + // we store the finish time to finish at a later hook + // this is because the test might fail at a `afterEach` hook + if (span) { + span.setTag(TEST_STATUS, status) + this.taskToFinishTime.set(task, span._getTime()) + } + }) + + this.addSub('ci:vitest:test:pass', ({ task }) => { + const store = storage.getStore() + const span = store?.span + + if (span) { + span.setTag(TEST_STATUS, 'pass') + span.finish(this.taskToFinishTime.get(task)) + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:vitest:test:error', ({ duration, error }) => { + const store = storage.getStore() + const span = store?.span + + if (span) { + span.setTag(TEST_STATUS, 'fail') + + if (error) { + span.setTag('error', error) + } + span.finish(span._startTime + duration - MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION) // milliseconds + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + this.startTestSpan( + testName, + testSuite, + this.testSuiteSpan, + { + [TEST_SOURCE_FILE]: testSuite, + [TEST_STATUS]: 'skip' + } + ).finish() + }) + + this.addSub('ci:vitest:test-suite:start', (testSuiteAbsolutePath) => { + const testSessionSpanContext = this.tracer.extract('text_map', { + 'x-datadog-trace-id': process.env.DD_CIVISIBILITY_TEST_SESSION_ID, + 'x-datadog-parent-id': process.env.DD_CIVISIBILITY_TEST_MODULE_ID + }) + + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const testSuiteMetadata = getTestSuiteCommonTags( + this.command, + this.frameworkVersion, + testSuite, + 'vitest' + ) + const testSuiteSpan = this.tracer.startSpan('vitest.test_suite', { + childOf: testSessionSpanContext, + tags: { + [COMPONENT]: this.constructor.id, + ...this.testEnvironmentMetadata, + ...testSuiteMetadata + } + }) + const store = storage.getStore() + this.enter(testSuiteSpan, store) + this.testSuiteSpan = testSuiteSpan + }) + + this.addSub('ci:vitest:test-suite:finish', ({ status, onFinish }) => { + const store = storage.getStore() + const span = store?.span + if (span) { + span.setTag(TEST_STATUS, status) + span.finish() + finishAllTraceSpans(span) + } + // TODO: too frequent flush - find for method in worker to decrease frequency + this.tracer._exporter.flush(onFinish) + }) + + this.addSub('ci:vitest:test-suite:error', ({ error }) => { + const store = storage.getStore() + const span = store?.span + if (span && error) { + span.setTag('error', error) + span.setTag(TEST_STATUS, 'fail') + } + }) + + this.addSub('ci:vitest:session:finish', ({ status, onFinish, error }) => { + this.testSessionSpan.setTag(TEST_STATUS, status) + this.testModuleSpan.setTag(TEST_STATUS, status) + if (error) { + this.testModuleSpan.setTag('error', error) + this.testSessionSpan.setTag('error', error) + } + this.testModuleSpan.finish() + this.testSessionSpan.finish() + finishAllTraceSpans(this.testSessionSpan) + this.tracer._exporter.flush(onFinish) + }) + } +} + +module.exports = VitestPlugin diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 310cb2dc940..4daeb02a4bf 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -91,6 +91,13 @@ module.exports = class CiPlugin extends Plugin { ...testModuleSpanMetadata } }) + // only for vitest + // These are added for the worker threads to use + if (this.constructor.id === 'vitest') { + process.env.DD_CIVISIBILITY_TEST_SESSION_ID = this.testSessionSpan.context().toTraceId() + process.env.DD_CIVISIBILITY_TEST_MODULE_ID = this.testModuleSpan.context().toSpanId() + } + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'module') }) diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index 0b98cd9c076..fd9288afcc4 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -18,6 +18,7 @@ module.exports = { get '@opensearch-project/opensearch' () { return require('../../../datadog-plugin-opensearch/src') }, get '@redis/client' () { return require('../../../datadog-plugin-redis/src') }, get '@smithy/smithy-client' () { return require('../../../datadog-plugin-aws-sdk/src') }, + get '@vitest/runner' () { return require('../../../datadog-plugin-vitest/src') }, get aerospike () { return require('../../../datadog-plugin-aerospike/src') }, get amqp10 () { return require('../../../datadog-plugin-amqp10/src') }, get amqplib () { return require('../../../datadog-plugin-amqplib/src') }, @@ -54,6 +55,7 @@ module.exports = { get 'microgateway-core' () { return require('../../../datadog-plugin-microgateway-core/src') }, get mocha () { return require('../../../datadog-plugin-mocha/src') }, get 'mocha-each' () { return require('../../../datadog-plugin-mocha/src') }, + get vitest () { return require('../../../datadog-plugin-vitest/src') }, get workerpool () { return require('../../../datadog-plugin-mocha/src') }, get moleculer () { return require('../../../datadog-plugin-moleculer/src') }, get mongodb () { return require('../../../datadog-plugin-mongodb-core/src') },