diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml index a74051512b..f16227e81a 100644 --- a/.github/workflows/project.yml +++ b/.github/workflows/project.yml @@ -110,6 +110,23 @@ jobs: CYPRESS_VERSION: ${{ matrix.cypress-version }} NODE_OPTIONS: '-r ./ci/init' + integration-vitest: + runs-on: ubuntu-latest + env: + DD_SERVICE: dd-trace-js-integration-tests + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1 + DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/node/setup + - run: yarn install + - uses: actions/setup-node@v3 + with: + node-version: 20 + - run: yarn test:integration:vitest + env: + NODE_OPTIONS: '-r ./ci/init' + lint: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 6b1413e877..03b2fe21bf 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -86,9 +86,10 @@ jobs: parametric: needs: - build-artifacts - uses: DataDog/system-tests/.github/workflows/system-tests.yml@main + uses: DataDog/system-tests/.github/workflows/run-parametric.yml@main secrets: inherit with: - scenarios: PARAMETRIC library: nodejs binaries_artifact: system_tests_binaries + _experimental_job_count: 8 + _experimental_job_matrix: '[1,2,3,4,5,6,7,8]' diff --git a/docs/test.ts b/docs/test.ts index 380bfd6bba..9b11d34484 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -363,6 +363,8 @@ tracer.use('sharedb'); tracer.use('sharedb', sharedbOptions); tracer.use('tedious'); tracer.use('undici'); +tracer.use('vitest'); +tracer.use('vitest', { service: 'vitest-service' }); tracer.use('winston'); tracer.use('express', false) diff --git a/ext/exporters.d.ts b/ext/exporters.d.ts index 07bc2cd29e..2f462dd93e 100644 --- a/ext/exporters.d.ts +++ b/ext/exporters.d.ts @@ -4,7 +4,7 @@ declare const exporters: { DATADOG: 'datadog', AGENT_PROXY: 'agent_proxy', JEST_WORKER: 'jest_worker', - CUCUMBER_WORKER: 'cucumber_worker' + CUCUMBER_WORKER: 'cucumber_worker', MOCHA_WORKER: 'mocha_worker' } diff --git a/index.d.ts b/index.d.ts index d0b634f5dd..7d5846f8af 100644 --- a/index.d.ts +++ b/index.d.ts @@ -198,6 +198,7 @@ interface Plugins { "sharedb": tracer.plugins.sharedb; "tedious": tracer.plugins.tedious; "undici": tracer.plugins.undici; + "vitest": tracer.plugins.vitest; "winston": tracer.plugins.winston; } @@ -1556,7 +1557,7 @@ declare namespace tracer { /** * This plugin automatically instruments the - * [jest](https://github.com/facebook/jest) module. + * [jest](https://github.com/jestjs/jest) module. */ interface jest extends Integration {} @@ -1839,6 +1840,12 @@ declare namespace tracer { */ interface undici extends HttpClient {} + /** + * This plugin automatically instruments the + * [vitest](https://github.com/vitest-dev/vitest) module. + */ + interface vitest extends Integration {} + /** * This plugin patches the [winston](https://github.com/winstonjs/winston) * to automatically inject trace identifiers in log records when the diff --git a/integration-tests/ci-visibility/vitest-tests/sum.mjs b/integration-tests/ci-visibility/vitest-tests/sum.mjs new file mode 100644 index 0000000000..f1c6520acb --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/sum.mjs @@ -0,0 +1,3 @@ +export function sum (a, b) { + return a + b +} diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs new file mode 100644 index 0000000000..a97f95e0df --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs @@ -0,0 +1,26 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { sum } from './sum' + +describe('context', () => { + beforeEach(() => { + throw new Error('failed before each') + }) + test('can report failed test', () => { + expect(sum(1, 2)).to.equal(4) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) + +describe('other context', () => { + afterEach(() => { + throw new Error('failed after each') + }) + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs new file mode 100644 index 0000000000..f2df345a87 --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs @@ -0,0 +1,29 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { sum } from './sum' + +let preparedValue = 1 + +describe('test-visibility-failed-suite-first-describe', () => { + beforeEach(() => { + preparedValue = 2 + }) + test('can report failed test', () => { + expect(sum(1, 2)).to.equal(4) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + expect(preparedValue).to.equal(2) + }) +}) + +describe('test-visibility-failed-suite-second-describe', () => { + afterEach(() => { + preparedValue = 1 + }) + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs new file mode 100644 index 0000000000..c2cf93431d --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs @@ -0,0 +1,32 @@ +import { describe, test, expect } from 'vitest' +import { sum } from './sum' + +describe('context', () => { + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) + +describe('other context', () => { + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) + test.skip('can skip', () => { + expect(sum(1, 2)).to.equal(3) + }) + test.todo('can todo', () => { + expect(sum(1, 2)).to.equal(3) + }) + // eslint-disable-next-line + test('can programmatic skip', (context) => { + // eslint-disable-next-line + context.skip() + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/esbuild/basic-test.js b/integration-tests/esbuild/basic-test.js index 20f53708b4..dc41b4efa5 100755 --- a/integration-tests/esbuild/basic-test.js +++ b/integration-tests/esbuild/basic-test.js @@ -1,11 +1,5 @@ #!/usr/bin/env node -// TODO: add support for Node.js v14.17+ and v16.0+ -if (Number(process.versions.node.split('.')[0]) < 16) { - console.error(`Skip esbuild test for node@${process.version}`) // eslint-disable-line no-console - process.exit(0) -} - const tracer = require('../../').init() // dd-trace const assert = require('assert') diff --git a/integration-tests/vitest.config.mjs b/integration-tests/vitest.config.mjs new file mode 100644 index 0000000000..f04d63785f --- /dev/null +++ b/integration-tests/vitest.config.mjs @@ -0,0 +1,9 @@ +import { defineConfig } from 'vite' + +export default defineConfig({ + test: { + include: [ + 'ci-visibility/vitest-tests/test-visibility*' + ] + } +}) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js new file mode 100644 index 0000000000..4a3151c2b7 --- /dev/null +++ b/integration-tests/vitest/vitest.spec.js @@ -0,0 +1,137 @@ +'use strict' + +const { exec } = require('child_process') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_STATUS, + TEST_TYPE +} = require('../../packages/dd-trace/src/plugins/util/test') + +// tested with 1.6.0 +const versions = ['latest'] + +versions.forEach((version) => { + describe(`vitest@${version}`, () => { + let sandbox, cwd, receiver, childProcess + + before(async function () { + sandbox = await createSandbox([`vitest@${version}`], true) + cwd = sandbox.folder + }) + + after(async () => { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + childProcess.kill() + await receiver.stop() + }) + + it('can run and report tests', (done) => { + receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSessionEvent = events.find(event => event.type === 'test_session_end') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + const testEvents = events.filter(event => event.type === 'test') + + assert.include(testSessionEvent.content.resource, 'test_session.vitest run') + assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail') + assert.include(testModuleEvent.content.resource, 'test_module.vitest run') + assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail') + assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test') + assert.equal(testModuleEvent.content.meta[TEST_TYPE], 'test') + + const passedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-passed-suite.mjs' + ) + assert.equal(passedSuite.content.meta[TEST_STATUS], 'pass') + + const failedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + ) + assert.equal(failedSuite.content.meta[TEST_STATUS], 'fail') + + const failedSuiteHooks = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs' + ) + assert.equal(failedSuiteHooks.content.meta[TEST_STATUS], 'fail') + + assert.includeMembers(testEvents.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' + ] + ) + + const failedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'fail') + + assert.includeMembers( + failedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' + ] + ) + + const skippedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'skip') + + assert.includeMembers( + skippedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip' + ] + ) + // TODO: check error messages + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + // maybe only in node@20 + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' // ESM requires more flags + }, + stdio: 'pipe' + } + ) + }) + }) +}) diff --git a/package.json b/package.json index b98aaca358..a9a09cb3bf 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "test:integration:cypress": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/cypress/*.spec.js\"", "test:integration:playwright": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/playwright/*.spec.js\"", "test:integration:selenium": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/selenium/*.spec.js\"", + "test:integration:vitest": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/vitest/*.spec.js\"", "test:integration:profiler": "mocha --colors --timeout 180000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/profiler/*.spec.js\"", "test:integration:serverless": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/serverless/*.spec.js\"", "test:integration:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", diff --git a/packages/datadog-esbuild/index.js b/packages/datadog-esbuild/index.js index 95a0e8ddd1..ce26379902 100644 --- a/packages/datadog-esbuild/index.js +++ b/packages/datadog-esbuild/index.js @@ -7,7 +7,11 @@ const hooks = require('../datadog-instrumentations/src/helpers/hooks.js') const extractPackageAndModulePath = require('../datadog-instrumentations/src/utils/src/extract-package-and-module-path') for (const hook of Object.values(hooks)) { - hook() + if (typeof hook === 'object') { + hook.fn() + } else { + hook() + } } const modulesOfInterest = new Set() diff --git a/packages/datadog-instrumentations/src/helpers/hook.js b/packages/datadog-instrumentations/src/helpers/hook.js index 7bec453187..0177744ea1 100644 --- a/packages/datadog-instrumentations/src/helpers/hook.js +++ b/packages/datadog-instrumentations/src/helpers/hook.js @@ -11,8 +11,13 @@ const ritm = require('../../../dd-trace/src/ritm') * @param {string[]} modules list of modules to hook into * @param {Function} onrequire callback to be executed upon encountering module */ -function Hook (modules, onrequire) { - if (!(this instanceof Hook)) return new Hook(modules, onrequire) +function Hook (modules, hookOptions, onrequire) { + if (!(this instanceof Hook)) return new Hook(modules, hookOptions, onrequire) + + if (typeof hookOptions === 'function') { + onrequire = hookOptions + hookOptions = {} + } this._patched = Object.create(null) @@ -28,7 +33,7 @@ function Hook (modules, onrequire) { } this._ritmHook = ritm(modules, {}, safeHook) - this._iitmHook = iitm(modules, {}, (moduleExports, moduleName, moduleBaseDir) => { + this._iitmHook = iitm(modules, hookOptions, (moduleExports, moduleName, moduleBaseDir) => { // TODO: Move this logic to import-in-the-middle and only do it for CommonJS // modules and not ESM. In the meantime, all the modules we instrument are // CommonJS modules for which the default export is always moved to diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index 0723ceabd8..94f3318fb6 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -23,6 +23,7 @@ module.exports = { '@opentelemetry/sdk-trace-node': () => require('../otel-sdk-trace'), '@redis/client': () => require('../redis'), '@smithy/smithy-client': () => require('../aws-sdk'), + '@vitest/runner': { esmFirst: true, fn: () => require('../vitest') }, aerospike: () => require('../aerospike'), amqp10: () => require('../amqp10'), amqplib: () => require('../amqplib'), @@ -110,6 +111,7 @@ module.exports = { sharedb: () => require('../sharedb'), tedious: () => require('../tedious'), undici: () => require('../undici'), + vitest: { esmFirst: true, fn: () => require('../vitest') }, when: () => require('../when'), winston: () => require('../winston') } diff --git a/packages/datadog-instrumentations/src/helpers/instrument.js b/packages/datadog-instrumentations/src/helpers/instrument.js index 0889f1e540..2065733504 100644 --- a/packages/datadog-instrumentations/src/helpers/instrument.js +++ b/packages/datadog-instrumentations/src/helpers/instrument.js @@ -17,10 +17,11 @@ exports.channel = function (name) { /** * @param {string} args.name module name * @param {string[]} args.versions array of semver range strings - * @param {string} args.file path to file within package to instrument? + * @param {string} args.file path to file within package to instrument + * @param {string} args.filePattern pattern to match files within package to instrument * @param Function hook */ -exports.addHook = function addHook ({ name, versions, file }, hook) { +exports.addHook = function addHook ({ name, versions, file, filePattern }, hook) { if (typeof name === 'string') { name = [name] } @@ -29,7 +30,7 @@ exports.addHook = function addHook ({ name, versions, file }, hook) { if (!instrumentations[val]) { instrumentations[val] = [] } - instrumentations[val].push({ name: val, versions, file, hook }) + instrumentations[val].push({ name: val, versions, file, filePattern, hook }) } } diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index dfdb680c1c..7bb5e94005 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -46,19 +46,32 @@ const seenCombo = new Set() for (const packageName of names) { if (disabledInstrumentations.has(packageName)) continue - Hook([packageName], (moduleExports, moduleName, moduleBaseDir, moduleVersion) => { + const hookOptions = {} + + let hook = hooks[packageName] + + if (typeof hook === 'object') { + hookOptions.internals = hook.esmFirst + hook = hook.fn + } + + Hook([packageName], hookOptions, (moduleExports, moduleName, moduleBaseDir, moduleVersion) => { moduleName = moduleName.replace(pathSepExpr, '/') // This executes the integration file thus adding its entries to `instrumentations` - hooks[packageName]() + hook() if (!instrumentations[packageName]) { return moduleExports } const namesAndSuccesses = {} - for (const { name, file, versions, hook } of instrumentations[packageName]) { + for (const { name, file, versions, hook, filePattern } of instrumentations[packageName]) { + let fullFilePattern = filePattern const fullFilename = filename(name, file) + if (fullFilePattern) { + fullFilePattern = filename(name, fullFilePattern) + } // Create a WeakMap associated with the hook function so that patches on the same moduleExport only happens once // for example by instrumenting both dns and node:dns double the spans would be created @@ -66,8 +79,17 @@ for (const packageName of names) { if (!hook[HOOK_SYMBOL]) { hook[HOOK_SYMBOL] = new WeakMap() } + let matchesFile = false + + matchesFile = moduleName === fullFilename + + if (fullFilePattern) { + // Some libraries include a hash in their filenames when installed, + // so our instrumentation has to include a '.*' to match them for more than a single version. + matchesFile = matchesFile || new RegExp(fullFilePattern).test(moduleName) + } - if (moduleName === fullFilename) { + if (matchesFile) { const version = moduleVersion || getVersion(moduleBaseDir) if (!Object.hasOwnProperty(namesAndSuccesses, name)) { namesAndSuccesses[name] = { diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js new file mode 100644 index 0000000000..42f32b1ac4 --- /dev/null +++ b/packages/datadog-instrumentations/src/vitest.js @@ -0,0 +1,282 @@ +const { addHook, channel, AsyncResource } = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') + +// test hooks +const testStartCh = channel('ci:vitest:test:start') +const testFinishTimeCh = channel('ci:vitest:test:finish-time') +const testPassCh = channel('ci:vitest:test:pass') +const testErrorCh = channel('ci:vitest:test:error') +const testSkipCh = channel('ci:vitest:test:skip') + +// test suite hooks +const testSuiteStartCh = channel('ci:vitest:test-suite:start') +const testSuiteFinishCh = channel('ci:vitest:test-suite:finish') +const testSuiteErrorCh = channel('ci:vitest:test-suite:error') + +// test session hooks +const testSessionStartCh = channel('ci:vitest:session:start') +const testSessionFinishCh = channel('ci:vitest:session:finish') + +const taskToAsync = new WeakMap() + +const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') + +function isReporterPackage (vitestPackage) { + return vitestPackage.B?.name === 'BaseSequencer' +} + +function getSessionStatus (state) { + if (state.getCountOfFailedTests() > 0) { + return 'fail' + } + if (state.pathsSet.size === 0) { + return 'skip' + } + return 'pass' +} + +// eslint-disable-next-line +// From https://github.com/vitest-dev/vitest/blob/51c04e2f44d91322b334f8ccbcdb368facc3f8ec/packages/runner/src/run.ts#L243-L250 +function getVitestTestStatus (test, retryCount) { + if (test.result.state !== 'fail') { + if (!test.repeats) { + return 'pass' + } else if (test.repeats && (test.retry ?? 0) === retryCount) { + return 'pass' + } + } + return 'fail' +} + +function getTypeTasks (fileTasks, type = 'test') { + const typeTasks = [] + + function getTasks (tasks) { + for (const task of tasks) { + if (task.type === type) { + typeTasks.push(task) + } else if (task.tasks) { + getTasks(task.tasks) + } + } + } + + getTasks(fileTasks) + + return typeTasks +} + +function getTestName (task) { + let testName = task.name + let currentTask = task.suite + + while (currentTask) { + if (currentTask.name) { + testName = `${currentTask.name} ${testName}` + } + currentTask = currentTask.suite + } + + return testName +} + +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + file: 'dist/runners.js' +}, (vitestPackage) => { + const { VitestTestRunner } = vitestPackage + // test start (only tests that are not marked as skip or todo) + shimmer.wrap(VitestTestRunner.prototype, 'onBeforeTryTask', onBeforeTryTask => async function (task) { + if (!testStartCh.hasSubscribers) { + return onBeforeTryTask.apply(this, arguments) + } + const asyncResource = new AsyncResource('bound-anonymous-fn') + taskToAsync.set(task, asyncResource) + + asyncResource.runInAsyncScope(() => { + testStartCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + }) + return onBeforeTryTask.apply(this, arguments) + }) + + // test finish (only passed tests) + shimmer.wrap(VitestTestRunner.prototype, 'onAfterTryTask', onAfterTryTask => + async function (task, { retry: retryCount }) { + if (!testFinishTimeCh.hasSubscribers) { + return onAfterTryTask.apply(this, arguments) + } + const result = await onAfterTryTask.apply(this, arguments) + + const status = getVitestTestStatus(task, retryCount) + const asyncResource = taskToAsync.get(task) + + if (asyncResource) { + // We don't finish here because the test might fail in a later hook + asyncResource.runInAsyncScope(() => { + testFinishTimeCh.publish({ status, task }) + }) + } + + return result + }) + + return vitestPackage +}) + +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + filePattern: 'dist/vendor/index.*' +}, (vitestPackage) => { + // there are multiple index* files so we have to check the exported values + if (!isReporterPackage(vitestPackage)) { + return vitestPackage + } + shimmer.wrap(vitestPackage.B.prototype, 'sort', sort => async function () { + if (!testSessionFinishCh.hasSubscribers) { + return sort.apply(this, arguments) + } + shimmer.wrap(this.ctx, 'exit', exit => async function () { + let onFinish + + const flushPromise = new Promise(resolve => { + onFinish = resolve + }) + const failedSuites = this.state.getFailedFilepaths() + let error + if (failedSuites.length) { + error = new Error(`Test suites failed: ${failedSuites.length}.`) + } + + sessionAsyncResource.runInAsyncScope(() => { + testSessionFinishCh.publish({ + status: getSessionStatus(this.state), + onFinish, + error + }) + }) + + await flushPromise + + return exit.apply(this, arguments) + }) + + return sort.apply(this, arguments) + }) + + return vitestPackage +}) + +// Can't specify file because compiled vitest includes hashes in their files +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + filePattern: 'dist/vendor/cac.*' +}, (vitestPackage, frameworkVersion) => { + shimmer.wrap(vitestPackage, 'c', oldCreateCli => function () { + if (!testSessionStartCh.hasSubscribers) { + return oldCreateCli.apply(this, arguments) + } + sessionAsyncResource.runInAsyncScope(() => { + const processArgv = process.argv.slice(2).join(' ') + testSessionStartCh.publish({ command: `vitest ${processArgv}`, frameworkVersion }) + }) + return oldCreateCli.apply(this, arguments) + }) + + return vitestPackage +}) + +// test suite start and finish +// only relevant for workers +addHook({ + name: '@vitest/runner', + versions: ['>=1.6.0'], + file: 'dist/index.js' +}, vitestPackage => { + shimmer.wrap(vitestPackage, 'startTests', startTests => async function (testPath) { + let testSuiteError = null + if (!testSuiteStartCh.hasSubscribers) { + return startTests.apply(this, arguments) + } + + const testSuiteAsyncResource = new AsyncResource('bound-anonymous-fn') + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish(testPath[0]) + }) + const startTestsResponse = await startTests.apply(this, arguments) + + let onFinish = null + const onFinishPromise = new Promise(resolve => { + onFinish = resolve + }) + + const testTasks = getTypeTasks(startTestsResponse[0].tasks) + + testTasks.forEach(task => { + const testAsyncResource = taskToAsync.get(task) + const { result } = task + + if (result) { + const { state, duration, errors } = result + if (state === 'skip') { // programmatic skip + testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + } else if (state === 'pass') { + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + testPassCh.publish({ task }) + }) + } + } else if (state === 'fail') { + // If it's failing, we have no accurate finish time, so we have to use `duration` + let testError + + if (errors?.length) { + testError = errors[0] + } + + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + testErrorCh.publish({ duration, error: testError }) + }) + } + if (errors?.length) { + testSuiteError = testError // we store the error to bubble it up to the suite + } + } + } else { // test.skip or test.todo + testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + } + }) + + const testSuiteResult = startTestsResponse[0].result + + if (testSuiteResult.errors?.length) { // Errors from root level hooks + testSuiteError = testSuiteResult.errors[0] + } else if (testSuiteResult.state === 'fail') { // Errors from `describe` level hooks + const suiteTasks = getTypeTasks(startTestsResponse[0].tasks, 'suite') + const failedSuites = suiteTasks.filter(task => task.result?.state === 'fail') + if (failedSuites.length && failedSuites[0].result?.errors?.length) { + testSuiteError = failedSuites[0].result.errors[0] + } + } + + if (testSuiteError) { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish({ error: testSuiteError }) + }) + } + + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish({ status: testSuiteResult.state, onFinish }) + }) + + // TODO: fix too frequent flushes + await onFinishPromise + + return startTestsResponse + }) + + return vitestPackage +}) diff --git a/packages/datadog-plugin-openai/src/index.js b/packages/datadog-plugin-openai/src/index.js index 6b7bf6dc2c..739a80e8e7 100644 --- a/packages/datadog-plugin-openai/src/index.js +++ b/packages/datadog-plugin-openai/src/index.js @@ -309,6 +309,7 @@ class OpenApiPlugin extends TracingPlugin { } sendLog (methodName, span, tags, store, error) { + if (!store) return if (!Object.keys(store).length) return if (!this.sampler.isSampled()) return @@ -329,9 +330,22 @@ function countPromptTokens (methodName, payload, model) { const messages = payload.messages for (const message of messages) { const content = message.content - const { tokens, estimated } = countTokens(content, model) - promptTokens += tokens - promptEstimated = estimated + if (typeof content === 'string') { + const { tokens, estimated } = countTokens(content, model) + promptTokens += tokens + promptEstimated = estimated + } else if (Array.isArray(content)) { + for (const c of content) { + if (c.type === 'text') { + const { tokens, estimated } = countTokens(c.text, model) + promptTokens += tokens + promptEstimated = estimated + } + // unsupported token computation for image_url + // as even though URL is a string, its true token count + // is based on the image itself, something onerous to do client-side + } + } } } else if (methodName === 'completions.create') { let prompt = payload.prompt @@ -403,7 +417,7 @@ function createChatCompletionRequestExtraction (tags, payload, store) { store.messages = payload.messages for (let i = 0; i < payload.messages.length; i++) { const message = payload.messages[i] - tags[`openai.request.messages.${i}.content`] = truncateText(message.content) + tagChatCompletionRequestContent(message.content, i, tags) tags[`openai.request.messages.${i}.role`] = message.role tags[`openai.request.messages.${i}.name`] = message.name tags[`openai.request.messages.${i}.finish_reason`] = message.finish_reason @@ -692,7 +706,7 @@ function commonCreateResponseExtraction (tags, body, store, methodName) { for (let choiceIdx = 0; choiceIdx < body.choices.length; choiceIdx++) { const choice = body.choices[choiceIdx] - // logprobs can be nullm and we still want to tag it as 'returned' even when set to 'null' + // logprobs can be null and we still want to tag it as 'returned' even when set to 'null' const specifiesLogProb = Object.keys(choice).indexOf('logprobs') !== -1 tags[`openai.response.choices.${choiceIdx}.finish_reason`] = choice.finish_reason @@ -766,6 +780,7 @@ function truncateApiKey (apiKey) { */ function truncateText (text) { if (!text) return + if (typeof text !== 'string' || !text || (typeof text === 'string' && text.length === 0)) return text = text .replace(RE_NEWLINE, '\\n') @@ -778,6 +793,28 @@ function truncateText (text) { return text } +function tagChatCompletionRequestContent (contents, messageIdx, tags) { + if (typeof contents === 'string') { + tags[`openai.request.messages.${messageIdx}.content`] = contents + } else if (Array.isArray(contents)) { + // content can also be an array of objects + // which represent text input or image url + for (const contentIdx in contents) { + const content = contents[contentIdx] + const type = content.type + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.type`] = content.type + if (type === 'text') { + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.text`] = truncateText(content.text) + } else if (type === 'image_url') { + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.image_url.url`] = + truncateText(content.image_url.url) + } + // unsupported type otherwise, won't be tagged + } + } + // unsupported type otherwise, won't be tagged +} + // The server almost always responds with JSON function coerceResponseBody (body, methodName) { switch (methodName) { diff --git a/packages/datadog-plugin-openai/test/index.spec.js b/packages/datadog-plugin-openai/test/index.spec.js index 39d57b6005..b9db0e27c0 100644 --- a/packages/datadog-plugin-openai/test/index.spec.js +++ b/packages/datadog-plugin-openai/test/index.spec.js @@ -2703,6 +2703,62 @@ describe('Plugin', () => { await checkTraces }) + + it('should tag image_url', async () => { + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + // image_url is only relevant on request/input, output has the same shape as a normal chat completion + expect(span.meta).to.have.property('openai.request.messages.0.content.0.type', 'text') + expect(span.meta).to.have.property( + 'openai.request.messages.0.content.0.text', 'I\'m allergic to peanuts. Should I avoid this food?' + ) + expect(span.meta).to.have.property('openai.request.messages.0.content.1.type', 'image_url') + expect(span.meta).to.have.property( + 'openai.request.messages.0.content.1.image_url.url', 'dummy/url/peanut_food.png' + ) + }) + + const params = { + model: 'gpt-4-visual-preview', + messages: [ + { + role: 'user', + name: 'hunter2', + content: [ + { + type: 'text', + text: 'I\'m allergic to peanuts. Should I avoid this food?' + }, + { + type: 'image_url', + image_url: { + url: 'dummy/url/peanut_food.png' + } + } + ] + } + ] + } + + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.chat.completions.create(params) + + expect(result.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') + expect(result.choices[0].message.role).to.eql('assistant') + expect(result.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') + expect(result.choices[0].finish_reason).to.eql('length') + } else { + const result = await openai.createChatCompletion(params) + + expect(result.data.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') + expect(result.data.choices[0].message.role).to.eql('assistant') + expect(result.data.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') + expect(result.data.choices[0].finish_reason).to.eql('length') + } + + await checkTraces + }) }) describe('create chat completion with tools', () => { @@ -3267,6 +3323,54 @@ describe('Plugin', () => { expect(metricStub).to.have.been.calledWith('openai.tokens.total', 16, 'd', expectedTags) }) + it('makes a successful chat completion call without image_url usage computed', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/chat.completions.simple.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + + // we shouldn't be trying to capture the image_url tokens + expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens', 1) + }) + + const stream = await openai.chat.completions.create({ + stream: 1, + model: 'gpt-4o', + messages: [ + { + role: 'user', + name: 'hunter2', + content: [ + { + type: 'text', + text: 'One' // one token, for ease of testing + }, + { + type: 'image_url', + image_url: { + url: 'dummy/url/peanut_food.png' + } + } + ] + } + ] + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + } + + await checkTraces + }) + it('makes a successful completion call', async () => { nock('https://api.openai.com:443') .post('/v1/completions') diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js new file mode 100644 index 0000000000..c47467528e --- /dev/null +++ b/packages/datadog-plugin-vitest/src/index.js @@ -0,0 +1,156 @@ +const CiPlugin = require('../../dd-trace/src/plugins/ci_plugin') +const { storage } = require('../../datadog-core') + +const { + TEST_STATUS, + finishAllTraceSpans, + getTestSuitePath, + getTestSuiteCommonTags, + TEST_SOURCE_FILE +} = require('../../dd-trace/src/plugins/util/test') +const { COMPONENT } = require('../../dd-trace/src/constants') + +// Milliseconds that we subtract from the error test duration +// so that they do not overlap with the following test +// This is because there's some loss of resolution. +const MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION = 5 + +class VitestPlugin extends CiPlugin { + static get id () { + return 'vitest' + } + + constructor (...args) { + super(...args) + + this.taskToFinishTime = new WeakMap() + + this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const store = storage.getStore() + const span = this.startTestSpan( + testName, + testSuite, + this.testSuiteSpan, + { + [TEST_SOURCE_FILE]: testSuite + } + ) + + this.enter(span, store) + }) + + this.addSub('ci:vitest:test:finish-time', ({ status, task }) => { + const store = storage.getStore() + const span = store?.span + + // we store the finish time to finish at a later hook + // this is because the test might fail at a `afterEach` hook + if (span) { + span.setTag(TEST_STATUS, status) + this.taskToFinishTime.set(task, span._getTime()) + } + }) + + this.addSub('ci:vitest:test:pass', ({ task }) => { + const store = storage.getStore() + const span = store?.span + + if (span) { + span.setTag(TEST_STATUS, 'pass') + span.finish(this.taskToFinishTime.get(task)) + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:vitest:test:error', ({ duration, error }) => { + const store = storage.getStore() + const span = store?.span + + if (span) { + span.setTag(TEST_STATUS, 'fail') + + if (error) { + span.setTag('error', error) + } + span.finish(span._startTime + duration - MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION) // milliseconds + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + this.startTestSpan( + testName, + testSuite, + this.testSuiteSpan, + { + [TEST_SOURCE_FILE]: testSuite, + [TEST_STATUS]: 'skip' + } + ).finish() + }) + + this.addSub('ci:vitest:test-suite:start', (testSuiteAbsolutePath) => { + const testSessionSpanContext = this.tracer.extract('text_map', { + 'x-datadog-trace-id': process.env.DD_CIVISIBILITY_TEST_SESSION_ID, + 'x-datadog-parent-id': process.env.DD_CIVISIBILITY_TEST_MODULE_ID + }) + + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const testSuiteMetadata = getTestSuiteCommonTags( + this.command, + this.frameworkVersion, + testSuite, + 'vitest' + ) + const testSuiteSpan = this.tracer.startSpan('vitest.test_suite', { + childOf: testSessionSpanContext, + tags: { + [COMPONENT]: this.constructor.id, + ...this.testEnvironmentMetadata, + ...testSuiteMetadata + } + }) + const store = storage.getStore() + this.enter(testSuiteSpan, store) + this.testSuiteSpan = testSuiteSpan + }) + + this.addSub('ci:vitest:test-suite:finish', ({ status, onFinish }) => { + const store = storage.getStore() + const span = store?.span + if (span) { + span.setTag(TEST_STATUS, status) + span.finish() + finishAllTraceSpans(span) + } + // TODO: too frequent flush - find for method in worker to decrease frequency + this.tracer._exporter.flush(onFinish) + }) + + this.addSub('ci:vitest:test-suite:error', ({ error }) => { + const store = storage.getStore() + const span = store?.span + if (span && error) { + span.setTag('error', error) + span.setTag(TEST_STATUS, 'fail') + } + }) + + this.addSub('ci:vitest:session:finish', ({ status, onFinish, error }) => { + this.testSessionSpan.setTag(TEST_STATUS, status) + this.testModuleSpan.setTag(TEST_STATUS, status) + if (error) { + this.testModuleSpan.setTag('error', error) + this.testSessionSpan.setTag('error', error) + } + this.testModuleSpan.finish() + this.testSessionSpan.finish() + finishAllTraceSpans(this.testSessionSpan) + this.tracer._exporter.flush(onFinish) + }) + } +} + +module.exports = VitestPlugin diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 310cb2dc94..4daeb02a4b 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -91,6 +91,13 @@ module.exports = class CiPlugin extends Plugin { ...testModuleSpanMetadata } }) + // only for vitest + // These are added for the worker threads to use + if (this.constructor.id === 'vitest') { + process.env.DD_CIVISIBILITY_TEST_SESSION_ID = this.testSessionSpan.context().toTraceId() + process.env.DD_CIVISIBILITY_TEST_MODULE_ID = this.testModuleSpan.context().toSpanId() + } + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'module') }) diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index 0b98cd9c07..fd9288afcc 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -18,6 +18,7 @@ module.exports = { get '@opensearch-project/opensearch' () { return require('../../../datadog-plugin-opensearch/src') }, get '@redis/client' () { return require('../../../datadog-plugin-redis/src') }, get '@smithy/smithy-client' () { return require('../../../datadog-plugin-aws-sdk/src') }, + get '@vitest/runner' () { return require('../../../datadog-plugin-vitest/src') }, get aerospike () { return require('../../../datadog-plugin-aerospike/src') }, get amqp10 () { return require('../../../datadog-plugin-amqp10/src') }, get amqplib () { return require('../../../datadog-plugin-amqplib/src') }, @@ -54,6 +55,7 @@ module.exports = { get 'microgateway-core' () { return require('../../../datadog-plugin-microgateway-core/src') }, get mocha () { return require('../../../datadog-plugin-mocha/src') }, get 'mocha-each' () { return require('../../../datadog-plugin-mocha/src') }, + get vitest () { return require('../../../datadog-plugin-vitest/src') }, get workerpool () { return require('../../../datadog-plugin-mocha/src') }, get moleculer () { return require('../../../datadog-plugin-moleculer/src') }, get mongodb () { return require('../../../datadog-plugin-mongodb-core/src') },