diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 52b434b894d..2da72474c5a 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -363,6 +363,9 @@ jobs: - uses: codecov/codecov-action@v2 couchbase: + strategy: + matrix: + range: ['^2.6.12', '^3.0.7', '>=4.2.0'] runs-on: ubuntu-latest services: couchbase: @@ -373,6 +376,7 @@ jobs: env: PLUGINS: couchbase SERVICES: couchbase + PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start @@ -938,9 +942,11 @@ jobs: version: - 18 - latest + range: ['>=9.5 <11.1', '>=11.1 <13.2', '>=13.2'] runs-on: ubuntu-latest env: PLUGINS: next + PACKAGE_VERSION_RANGE: ${{ matrix.range }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/testagent/start diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml index 258d827bdf2..fca848638cf 100644 --- a/.github/workflows/project.yml +++ b/.github/workflows/project.yml @@ -30,11 +30,25 @@ jobs: - run: sudo sysctl -w kernel.core_pattern='|/bin/false' - run: yarn test:integration + # We'll run these separately for earlier (i.e. unsupported) versions + integration-guardrails: + strategy: + matrix: + version: [12, 14, 16] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.version }} + - run: yarn install --ignore-engines + - run: node node_modules/.bin/mocha --colors --timeout 30000 -r packages/dd-trace/test/setup/core.js integration-tests/init.spec.js + integration-ci: strategy: matrix: version: [18, latest] - framework: [cucumber, playwright, selenium] + framework: [cucumber, playwright, selenium, jest, mocha] runs-on: ubuntu-latest env: DD_SERVICE: dd-trace-js-integration-tests @@ -96,6 +110,23 @@ jobs: CYPRESS_VERSION: ${{ matrix.cypress-version }} NODE_OPTIONS: '-r ./ci/init' + integration-vitest: + runs-on: ubuntu-latest + env: + DD_SERVICE: dd-trace-js-integration-tests + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1 + DD_API_KEY: ${{ secrets.DD_API_KEY_CI_APP }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/node/setup + - run: yarn install + - uses: actions/setup-node@v3 + with: + node-version: 20 + - run: yarn test:integration:vitest + env: + NODE_OPTIONS: '-r ./ci/init' + lint: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 4a72cdcdb15..03b2fe21bf6 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -11,6 +11,19 @@ on: - cron: '00 04 * * 2-6' jobs: + build-artifacts: + runs-on: ubuntu-latest + steps: + - name: Checkout dd-trace-js + uses: actions/checkout@v4 + with: + path: dd-trace-js + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: system_tests_binaries + path: . + get-essential-scenarios: runs-on: ubuntu-latest outputs: @@ -71,31 +84,12 @@ jobs: path: artifact.tar.gz parametric: - runs-on: ubuntu-latest - env: - TEST_LIBRARY: nodejs - steps: - - name: Checkout system tests - uses: actions/checkout@v4 - with: - repository: 'DataDog/system-tests' - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Checkout dd-trace-js - uses: actions/checkout@v4 - with: - path: 'binaries/dd-trace-js' - - name: Build - run: ./build.sh -i runner - - name: Run - run: ./run.sh PARAMETRIC - - name: Compress artifact - if: ${{ always() }} - run: tar -czvf artifact.tar.gz $(ls | grep logs) - - name: Upload artifact - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: logs_parametric - path: artifact.tar.gz + needs: + - build-artifacts + uses: DataDog/system-tests/.github/workflows/run-parametric.yml@main + secrets: inherit + with: + library: nodejs + binaries_artifact: system_tests_binaries + _experimental_job_count: 8 + _experimental_job_matrix: '[1,2,3,4,5,6,7,8]' diff --git a/.gitlab/single-step-instrumentation-tests.yml b/.gitlab/single-step-instrumentation-tests.yml index 77cd4766d2b..441449ebac2 100644 --- a/.gitlab/single-step-instrumentation-tests.yml +++ b/.gitlab/single-step-instrumentation-tests.yml @@ -55,7 +55,7 @@ onboarding_tests: parallel: matrix: - ONBOARDING_FILTER_WEBLOG: [test-app-nodejs] - SCENARIO: [SIMPLE_HOST_AUTO_INJECTION] + SCENARIO: [SIMPLE_HOST_AUTO_INJECTION_PROFILING] - ONBOARDING_FILTER_WEBLOG: [test-app-nodejs-container] SCENARIO: [SIMPLE_CONTAINER_AUTO_INJECTION] script: diff --git a/CODEOWNERS b/CODEOWNERS index fc3236e1320..910e1f286b5 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -31,7 +31,12 @@ /integration-tests/cucumber/ @DataDog/ci-app-libraries /integration-tests/cypress/ @DataDog/ci-app-libraries /integration-tests/playwright/ @DataDog/ci-app-libraries -/integration-tests/ci-visibility.spec.js @DataDog/ci-app-libraries +/integration-tests/jest/jest.spec.js @DataDog/ci-app-libraries +/integration-tests/mocha/mocha.spec.js @DataDog/ci-app-libraries +/integration-tests/playwright/playwright.spec.js @DataDog/ci-app-libraries +/integration-tests/cucumber/cucumber.spec.js @DataDog/ci-app-libraries +/integration-tests/cypress/cypress.spec.js @DataDog/ci-app-libraries +/integration-tests/vitest/vitest.spec.js @DataDog/ci-app-libraries /integration-tests/test-api-manual.spec.js @DataDog/ci-app-libraries /packages/dd-trace/src/service-naming/ @Datadog/apm-idm-js diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index eca2504caaa..abb5bb48c18 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -64,6 +64,7 @@ dev,sinon,BSD-3-Clause,Copyright 2010-2017 Christian Johansen dev,sinon-chai,WTFPL and BSD-2-Clause,Copyright 2004 Sam Hocevar 2012–2017 Domenic Denicola dev,tap,ISC,Copyright 2011-2022 Isaac Z. Schlueter and Contributors dev,tape,MIT,Copyright James Halliday +dev,tiktoken,MIT,Copyright (c) 2022 OpenAI, Shantanu Jain file,aws-lambda-nodejs-runtime-interface-client,Apache 2.0,Copyright 2019 Amazon.com Inc. or its affiliates. All Rights Reserved. file,profile.proto,Apache license 2.0,Copyright 2016 Google Inc. file,is-git-url,MIT,Copyright (c) 2017 Jon Schlinkert. diff --git a/docs/test.ts b/docs/test.ts index 380bfd6bba9..fe17007bf0c 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -227,6 +227,7 @@ const elasticsearchOptions: plugins.elasticsearch = { const awsSdkOptions: plugins.aws_sdk = { service: 'test', splitByAwsService: false, + batchPropagationEnabled: false, hooks: { request: (span?: Span, response?) => {}, }, @@ -363,6 +364,8 @@ tracer.use('sharedb'); tracer.use('sharedb', sharedbOptions); tracer.use('tedious'); tracer.use('undici'); +tracer.use('vitest'); +tracer.use('vitest', { service: 'vitest-service' }); tracer.use('winston'); tracer.use('express', false) diff --git a/ext/exporters.d.ts b/ext/exporters.d.ts index 07bc2cd29e3..2f462dd93e7 100644 --- a/ext/exporters.d.ts +++ b/ext/exporters.d.ts @@ -4,7 +4,7 @@ declare const exporters: { DATADOG: 'datadog', AGENT_PROXY: 'agent_proxy', JEST_WORKER: 'jest_worker', - CUCUMBER_WORKER: 'cucumber_worker' + CUCUMBER_WORKER: 'cucumber_worker', MOCHA_WORKER: 'mocha_worker' } diff --git a/index.d.ts b/index.d.ts index d0b634f5dd8..34c5021b752 100644 --- a/index.d.ts +++ b/index.d.ts @@ -198,6 +198,7 @@ interface Plugins { "sharedb": tracer.plugins.sharedb; "tedious": tracer.plugins.tedious; "undici": tracer.plugins.undici; + "vitest": tracer.plugins.vitest; "winston": tracer.plugins.winston; } @@ -1222,6 +1223,13 @@ declare namespace tracer { */ splitByAwsService?: boolean; + /** + * Whether to inject all messages during batch AWS SQS, Kinesis, and SNS send operations. Normal + * behavior is to inject the first message in batch send operations. + * @default false + */ + batchPropagationEnabled?: boolean; + /** * Hooks to run before spans are finished. */ @@ -1556,7 +1564,7 @@ declare namespace tracer { /** * This plugin automatically instruments the - * [jest](https://github.com/facebook/jest) module. + * [jest](https://github.com/jestjs/jest) module. */ interface jest extends Integration {} @@ -1839,6 +1847,12 @@ declare namespace tracer { */ interface undici extends HttpClient {} + /** + * This plugin automatically instruments the + * [vitest](https://github.com/vitest-dev/vitest) module. + */ + interface vitest extends Integration {} + /** * This plugin patches the [winston](https://github.com/winstonjs/winston) * to automatically inject trace identifiers in log records when the diff --git a/init.js b/init.js index 328e287f506..7ddc8dbe91e 100644 --- a/init.js +++ b/init.js @@ -2,8 +2,26 @@ const path = require('path') const Module = require('module') +const telemetry = require('./packages/dd-trace/src/telemetry/init-telemetry') +const semver = require('semver') + +function isTrue (envVar) { + return ['1', 'true', 'True'].includes(envVar) +} + +// eslint-disable-next-line no-console +let log = { info: isTrue(process.env.DD_TRACE_DEBUG) ? console.log : () => {} } +if (semver.satisfies(process.versions.node, '>=16')) { + const Config = require('./packages/dd-trace/src/config') + log = require('./packages/dd-trace/src/log') + + // eslint-disable-next-line no-new + new Config() // we need this to initialize the logger +} let initBailout = false +let clobberBailout = false +const forced = isTrue(process.env.DD_INJECT_FORCE) if (process.env.DD_INJECTION_ENABLED) { // If we're running via single-step install, and we're not in the app's @@ -19,13 +37,34 @@ if (process.env.DD_INJECTION_ENABLED) { if (resolvedInApp) { const ourselves = path.join(__dirname, 'index.js') if (ourselves !== resolvedInApp) { + clobberBailout = true + } + } + + // If we're running via single-step install, and the runtime doesn't match + // the engines field in package.json, then we should not initialize the tracer. + if (!clobberBailout) { + const { engines } = require('./package.json') + const version = process.versions.node + if (!semver.satisfies(version, engines.node)) { initBailout = true + telemetry([ + { name: 'abort', tags: ['reason:incompatible_runtime'] }, + { name: 'abort.runtime', tags: [] } + ]) + log.info('Aborting application instrumentation due to incompatible_runtime.') + log.info(`Found incompatible runtime nodejs ${version}, Supported runtimes: nodejs ${engines.node}.`) + if (forced) { + log.info('DD_INJECT_FORCE enabled, allowing unsupported runtimes and continuing.') + } } } } -if (!initBailout) { +if (!clobberBailout && (!initBailout || forced)) { const tracer = require('.') tracer.init() module.exports = tracer + telemetry('complete', [`injection_forced:${forced && initBailout ? 'true' : 'false'}`]) + log.info('Application instrumentation bootstrapping complete') } diff --git a/initialize.mjs b/initialize.mjs index e7b33de492b..777f45cc046 100644 --- a/initialize.mjs +++ b/initialize.mjs @@ -44,9 +44,12 @@ export async function getSource (...args) { } if (isMainThread) { - await import('./init.js') - const { register } = await import('node:module') - if (register) { - register('./loader-hook.mjs', import.meta.url) - } + // Need this IIFE for versions of Node.js without top-level await. + (async () => { + await import('./init.js') + const { register } = await import('node:module') + if (register) { + register('./loader-hook.mjs', import.meta.url) + } + })() } diff --git a/integration-tests/ci-visibility.spec.js b/integration-tests/ci-visibility.spec.js deleted file mode 100644 index 4368f761cd0..00000000000 --- a/integration-tests/ci-visibility.spec.js +++ /dev/null @@ -1,2804 +0,0 @@ -'use strict' - -const { fork, exec } = require('child_process') -const path = require('path') - -const { assert } = require('chai') -const getPort = require('get-port') - -const { - createSandbox, - getCiVisAgentlessConfig, - getCiVisEvpProxyConfig -} = require('./helpers') -const { FakeCiVisIntake } = require('./ci-visibility-intake') - -const { - TEST_CODE_COVERAGE_ENABLED, - TEST_ITR_SKIPPING_ENABLED, - TEST_ITR_TESTS_SKIPPED, - TEST_CODE_COVERAGE_LINES_PCT, - TEST_SUITE, - TEST_STATUS, - TEST_SKIPPED_BY_ITR, - TEST_ITR_SKIPPING_TYPE, - TEST_ITR_SKIPPING_COUNT, - TEST_ITR_UNSKIPPABLE, - TEST_ITR_FORCED_RUN, - TEST_SOURCE_FILE, - TEST_IS_NEW, - TEST_IS_RETRY, - TEST_EARLY_FLAKE_ENABLED, - TEST_NAME, - JEST_DISPLAY_NAME, - TEST_EARLY_FLAKE_ABORT_REASON, - TEST_COMMAND, - TEST_MODULE, - MOCHA_IS_PARALLEL, - TEST_SOURCE_START -} = require('../packages/dd-trace/src/plugins/util/test') -const { ERROR_MESSAGE } = require('../packages/dd-trace/src/constants') - -const hookFile = 'dd-trace/loader-hook.mjs' - -const mochaCommonOptions = { - name: 'mocha', - expectedStdout: '2 passing', - extraStdout: 'end event: can add event listeners to mocha' -} - -const jestCommonOptions = { - name: 'jest', - dependencies: ['jest', 'chai@v4', 'jest-jasmine2', 'jest-environment-jsdom'], - expectedStdout: 'Test Suites: 2 passed', - expectedCoverageFiles: [ - 'ci-visibility/test/sum.js', - 'ci-visibility/test/ci-visibility-test.js', - 'ci-visibility/test/ci-visibility-test-2.js' - ] -} - -const testFrameworks = [ - { - ...mochaCommonOptions, - testFile: 'ci-visibility/run-mocha.js', - dependencies: ['mocha', 'chai@v4', 'nyc', 'mocha-each', 'workerpool'], - expectedCoverageFiles: [ - 'ci-visibility/run-mocha.js', - 'ci-visibility/test/sum.js', - 'ci-visibility/test/ci-visibility-test.js', - 'ci-visibility/test/ci-visibility-test-2.js' - ], - runTestsWithCoverageCommand: './node_modules/nyc/bin/nyc.js -r=text-summary node ./ci-visibility/run-mocha.js', - type: 'commonJS' - }, - { - ...jestCommonOptions, - testFile: 'ci-visibility/run-jest.js', - runTestsWithCoverageCommand: 'node ./ci-visibility/run-jest.js', - type: 'commonJS' - } -] - -// TODO: add ESM tests -testFrameworks.forEach(({ - name, - dependencies, - testFile, - expectedStdout, - extraStdout, - expectedCoverageFiles, - runTestsWithCoverageCommand, - type -}) => { - describe(`${name} ${type}`, () => { - let receiver - let childProcess - let sandbox - let cwd - let startupTestFile - let testOutput = '' - - before(async function () { - // add an explicit timeout to make esm tests less flaky - this.timeout(50000) - sandbox = await createSandbox(dependencies, true) - cwd = sandbox.folder - startupTestFile = path.join(cwd, testFile) - }) - - after(async function () { - await sandbox.remove() - }) - - beforeEach(async function () { - const port = await getPort() - receiver = await new FakeCiVisIntake(port).start() - }) - - afterEach(async () => { - childProcess.kill() - testOutput = '' - await receiver.stop() - }) - - if (name === 'mocha') { - it('does not change mocha config if CI Visibility fails to init', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report tests') - done(error) - }, ({ url }) => url === '/api/v2/citestcycle', 3000).catch(() => {}) - - const { DD_CIVISIBILITY_AGENTLESS_URL, ...restEnvVars } = getCiVisAgentlessConfig(receiver.port) - - // `runMocha` is only executed when using the CLI, which is where we modify mocha config - // if CI Visibility is init - childProcess = exec('mocha ./ci-visibility/test/ci-visibility-test.js', { - cwd, - env: { - ...restEnvVars, - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'error', - DD_SITE: '= invalid = url' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - assert.include(testOutput, 'Invalid URL') - assert.include(testOutput, '1 passing') // we only run one file here - done() - }) - }).timeout(50000) - - it('works with parallel mode', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const sessionEventContent = events.find(event => event.type === 'test_session_end').content - const moduleEventContent = events.find(event => event.type === 'test_module_end').content - const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') - assert.equal( - sessionEventContent.test_session_id.toString(10), - moduleEventContent.test_session_id.toString(10) - ) - suites.forEach(({ - meta, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) - }) - - tests.forEach(({ - meta, - metrics, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) - assert.propertyVal(meta, MOCHA_IS_PARALLEL, 'true') - assert.exists(metrics[TEST_SOURCE_START]) - }) - }) - - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - RUN_IN_PARALLEL: true, - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - eventsPromise.then(() => { - assert.notInclude(testOutput, 'TypeError') - assert.notInclude( - testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' - ) - done() - }).catch(done) - }) - }) - - it('works with parallel mode when run with the cli', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const sessionEventContent = events.find(event => event.type === 'test_session_end').content - const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') - assert.equal(suites.length, 2) - assert.equal(tests.length, 2) - }) - childProcess = exec('mocha --parallel --jobs 2 ./ci-visibility/test/ci-visibility-test*', { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - eventsPromise.then(() => { - assert.notInclude(testOutput, 'TypeError') - assert.notInclude( - testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' - ) - done() - }).catch(done) - }) - }) - - it('does not blow up when workerpool is used outside of a test', (done) => { - childProcess = exec('node ./ci-visibility/run-workerpool.js', { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', (code) => { - assert.include(testOutput, 'result 7') - assert.equal(code, 0) - done() - }) - }) - } - - if (name === 'jest') { - it('works when sharding', (done) => { - receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(events => { - const testSuiteEvents = events.payload.events.filter(event => event.type === 'test_suite_end') - assert.equal(testSuiteEvents.length, 3) - const testSuites = testSuiteEvents.map(span => span.content.meta[TEST_SUITE]) - - assert.includeMembers(testSuites, - [ - 'ci-visibility/sharding-test/sharding-test-5.js', - 'ci-visibility/sharding-test/sharding-test-4.js', - 'ci-visibility/sharding-test/sharding-test-1.js' - ] - ) - - const testSession = events.payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - - // We run the second shard - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/sharding-test/sharding-test-2.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/sharding-test/sharding-test-3.js' - } - } - ]) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'sharding-test/sharding-test', - TEST_SHARD: '2/2' - }, - stdio: 'inherit' - } - ) - - receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(secondShardEvents => { - const testSuiteEvents = secondShardEvents.payload.events.filter(event => event.type === 'test_suite_end') - - // The suites for this shard are to be skipped - assert.equal(testSuiteEvents.length, 2) - - testSuiteEvents.forEach(testSuite => { - assert.propertyVal(testSuite.content.meta, TEST_STATUS, 'skip') - assert.propertyVal(testSuite.content.meta, TEST_SKIPPED_BY_ITR, 'true') - }) - - const testSession = secondShardEvents - .payload - .events - .find(event => event.type === 'test_session_end').content - - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 2) - - done() - }) - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'sharding-test/sharding-test', - TEST_SHARD: '1/2' - }, - stdio: 'inherit' - } - ) - }) - it('does not crash when jest is badly initialized', (done) => { - childProcess = fork('ci-visibility/run-jest-bad-init.js', { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.notInclude(testOutput, 'TypeError') - assert.include(testOutput, expectedStdout) - done() - }) - }) - it('does not crash when jest uses jest-jasmine2', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - OLD_RUNNER: 1, - NODE_OPTIONS: '-r dd-trace/ci/init', - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.notInclude(testOutput, 'TypeError') - done() - }) - }) - describe('when jest is using workers to run tests in parallel', () => { - it('reports tests when using the agent', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - childProcess = fork(testFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: '-r dd-trace/ci/init', - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - - receiver.gatherPayloads(({ url }) => url === '/v0.4/traces', 5000).then(tracesRequests => { - const testSpans = tracesRequests.flatMap(trace => trace.payload).flatMap(request => request) - assert.equal(testSpans.length, 2) - const spanTypes = testSpans.map(span => span.type) - assert.includeMembers(spanTypes, ['test']) - assert.notInclude(spanTypes, ['test_session_end', 'test_suite_end', 'test_module_end']) - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v2'] }) - done() - }).catch(done) - }) - - it('reports tests when using agentless', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - - receiver.gatherPayloads(({ url }) => url === '/api/v2/citestcycle', 5000).then(eventsRequests => { - const eventTypes = eventsRequests.map(({ payload }) => payload) - .flatMap(({ events }) => events) - .map(event => event.type) - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - done() - }).catch(done) - }) - - it('reports tests when using evp proxy', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisEvpProxyConfig(receiver.port), - RUN_IN_PARALLEL: true - }, - stdio: 'pipe' - }) - - receiver.gatherPayloads(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle', 5000) - .then(eventsRequests => { - const eventTypes = eventsRequests.map(({ payload }) => payload) - .flatMap(({ events }) => events) - .map(event => event.type) - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - done() - }).catch(done) - }) - }) - it('reports timeout error message', (done) => { - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - NODE_OPTIONS: '-r dd-trace/ci/init', - RUN_IN_PARALLEL: true, - TESTS_TO_RUN: 'timeout-test/timeout-test.js' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.include(testOutput, 'Exceeded timeout of 100 ms for a test') - done() - }) - }) - it('reports parsing errors in the test file', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - assert.equal(suites.length, 2) - - const resourceNames = suites.map(suite => suite.content.resource) - - assert.includeMembers(resourceNames, [ - 'test_suite.ci-visibility/test-parsing-error/parsing-error-2.js', - 'test_suite.ci-visibility/test-parsing-error/parsing-error.js' - ]) - suites.forEach(suite => { - assert.equal(suite.content.meta[TEST_STATUS], 'fail') - assert.include(suite.content.meta[ERROR_MESSAGE], 'chao') - }) - }) - childProcess = fork(testFile, { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'test-parsing-error/parsing-error' - }, - stdio: 'pipe' - }) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not report total code coverage % if user has not configured coverage manually', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.assertPayloadReceived(({ payload }) => { - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT) - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - DISABLE_CODE_COVERAGE: '1' - }, - stdio: 'inherit' - } - ) - }) - it('reports total code coverage % even when ITR is disabled', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(({ payload }) => { - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - it('works with --forceExit and logs a warning', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - assert.include(testOutput, "Jest's '--forceExit' flag has been passed") - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end') - const testModule = events.find(event => event.type === 'test_module_end') - const testSuites = events.filter(event => event.type === 'test_suite_end') - const tests = events.filter(event => event.type === 'test') - - assert.exists(testSession) - assert.exists(testModule) - assert.equal(testSuites.length, 2) - assert.equal(tests.length, 2) - }) - // Needs to run with the CLI if we want --forceExit to work - childProcess = exec( - 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - DD_TRACE_DEBUG: '1', - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - }) - it('does not hang if server is not available and logs an error', (done) => { - // Very slow intake - receiver.setWaitingTime(30000) - // Needs to run with the CLI if we want --forceExit to work - childProcess = exec( - 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - DD_TRACE_DEBUG: '1', - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'inherit' - } - ) - const EXPECTED_FORCE_EXIT_LOG_MESSAGE = "Jest's '--forceExit' flag has been passed" - const EXPECTED_TIMEOUT_LOG_MESSAGE = 'Timeout waiting for the tracer to flush' - childProcess.on('exit', () => { - assert.include( - testOutput, - EXPECTED_FORCE_EXIT_LOG_MESSAGE, - `"${EXPECTED_FORCE_EXIT_LOG_MESSAGE}" log message is not in test output: ${testOutput}` - ) - assert.include( - testOutput, - EXPECTED_TIMEOUT_LOG_MESSAGE, - `"${EXPECTED_TIMEOUT_LOG_MESSAGE}" log message is not in the test output: ${testOutput}` - ) - done() - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - }) - it('intelligent test runner can skip when using a custom test sequencer', (done) => { - receiver.setSettings({ - itr_enabled: true, - tests_skipping: true - }) - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testEvents = events.filter(event => event.type === 'test') - // no tests end up running (suite is skipped) - assert.equal(testEvents.length, 0) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - - const skippedSuite = events.find(event => - event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - CUSTOM_TEST_SEQUENCER: './ci-visibility/jest-custom-test-sequencer.js', - TEST_SHARD: '2/2' - }, - stdio: 'inherit' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - - childProcess.on('exit', () => { - assert.include(testOutput, 'Running shard with a custom sequencer') - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('grabs the jest displayName config and sets tag in tests and suites', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.equal(tests.length, 4) // two per display name - const nodeTests = tests.filter(test => test.meta[JEST_DISPLAY_NAME] === 'node') - assert.equal(nodeTests.length, 2) - - const standardTests = tests.filter(test => test.meta[JEST_DISPLAY_NAME] === 'standard') - assert.equal(standardTests.length, 2) - - const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - assert.equal(suites.length, 4) - - const nodeSuites = suites.filter(suite => suite.meta[JEST_DISPLAY_NAME] === 'node') - assert.equal(nodeSuites.length, 2) - - const standardSuites = suites.filter(suite => suite.meta[JEST_DISPLAY_NAME] === 'standard') - assert.equal(standardSuites.length, 2) - }) - childProcess = exec( - 'node ./node_modules/jest/bin/jest --config config-jest-multiproject.js', - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('works with multi project setup and test skipping with intelligent test runner', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: true - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - // suites for both projects in the multi-project config are reported as skipped - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - - const skippedSuites = testSuites.filter( - suite => suite.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' - ) - assert.equal(skippedSuites.length, 2) - - skippedSuites.forEach(skippedSuite => { - assert.equal(skippedSuite.meta[TEST_STATUS], 'skip') - assert.equal(skippedSuite.meta[TEST_SKIPPED_BY_ITR], 'true') - }) - }) - - childProcess = exec( - 'node ./node_modules/jest/bin/jest --config config-jest-multiproject.js', - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('calculates executable lines even if there have been skipped suites', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: true - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test-total-code-coverage/test-skipped.js' - } - }]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - - // Before https://github.com/DataDog/dd-trace-js/pull/4336, this would've been 100% - // The reason is that skipping jest's `addUntestedFiles`, we would not see unexecuted lines. - // In this cause, these would be from the `unused-dependency.js` file. - // It is 50% now because we only cover 1 out of 2 files (`used-dependency.js`). - assert.propertyVal(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT, 50) - }) - - childProcess = exec( - runTestsWithCoverageCommand, // Requirement: the user must've opted in to code coverage - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN: 'ci-visibility/test-total-code-coverage/test-', - COLLECT_COVERAGE_FROM: '**/test-total-code-coverage/**' - }, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(done).catch(done) - }) - }) - } - const reportingOptions = ['agentless', 'evp proxy'] - - reportingOptions.forEach(reportingOption => { - context(`early flake detection when reporting by ${reportingOption}`, () => { - it('retries new tests', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new - receiver.setKnownTests({ - [name]: { - 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] - } - }) - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - // TODO: maybe check in stdout for the "Retried by Datadog" - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - // no other tests are considered new - const oldTests = tests.filter(test => - test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' - ) - oldTests.forEach(test => { - assert.notProperty(test.meta, TEST_IS_NEW) - }) - assert.equal(oldTests.length, 1) - - const newTests = tests.filter(test => - test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' - ) - newTests.forEach(test => { - assert.propertyVal(test.meta, TEST_IS_NEW, 'true') - }) - const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - // all but one has been retried - assert.equal( - newTests.length - 1, - retriedTests.length - ) - assert.equal(retriedTests.length, NUM_RETRIES_EFD) - // Test name does not change - newTests.forEach(test => { - assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') - }) - }) - - let TESTS_TO_RUN = 'test/ci-visibility-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/ci-visibility-test.js', - './test/ci-visibility-test-2.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { ...envVars, TESTS_TO_RUN }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('handles parameterized tests as a single unit', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new - receiver.setKnownTests({ - [name]: { - 'ci-visibility/test-early-flake-detection/test.js': ['ci visibility can report tests'] - } - }) - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': 3 - }, - faulty_session_threshold: 100 - } - }) - - const parameterizedTestFile = name === 'mocha' ? 'mocha-parameterized.js' : 'test-parameterized.js' - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const newTests = tests.filter(test => - test.meta[TEST_SUITE] === `ci-visibility/test-early-flake-detection/${parameterizedTestFile}` - ) - newTests.forEach(test => { - assert.propertyVal(test.meta, TEST_IS_NEW, 'true') - }) - // Each parameter is repeated independently - const testsForFirstParameter = tests.filter(test => test.resource === - `ci-visibility/test-early-flake-detection/${parameterizedTestFile}.parameterized test parameter 1` - ) - - const testsForSecondParameter = tests.filter(test => test.resource === - `ci-visibility/test-early-flake-detection/${parameterizedTestFile}.parameterized test parameter 2` - ) - - assert.equal(testsForFirstParameter.length, testsForSecondParameter.length) - - // all but one have been retried - assert.equal( - testsForFirstParameter.length - 1, - testsForFirstParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length - ) - - assert.equal( - testsForSecondParameter.length - 1, - testsForSecondParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length - ) - }) - - let TESTS_TO_RUN = 'test-early-flake-detection/test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test-early-flake-detection/test.js', - `./test-early-flake-detection/${parameterizedTestFile}` - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { ...envVars, TESTS_TO_RUN }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new - receiver.setKnownTests({ - [name]: { - 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] - } - }) - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': 3 - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - const newTests = tests.filter(test => - test.meta[TEST_IS_NEW] === 'true' - ) - // new tests are not detected - assert.equal(newTests.length, 0) - }) - - let TESTS_TO_RUN = 'test/ci-visibility-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/ci-visibility-test.js', - './test/ci-visibility-test-2.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...envVars, - TESTS_TO_RUN, - DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('retries flaky tests', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/occasionally-failing-test will be considered new - receiver.setKnownTests({}) - - const NUM_RETRIES_EFD = 5 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - // all but one has been retried - assert.equal( - tests.length - 1, - retriedTests.length - ) - assert.equal(retriedTests.length, NUM_RETRIES_EFD) - // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, - // based on the global counter in the test file - const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') - const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') - assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) - assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) - // Test name does not change - retriedTests.forEach(test => { - assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') - }) - }) - - let TESTS_TO_RUN = 'test-early-flake-detection/occasionally-failing-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test-early-flake-detection/occasionally-failing-test.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { ...envVars, TESTS_TO_RUN }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - // TODO: check exit code: if a new, retried test fails, the exit code should remain 0 - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not retry new tests that are skipped', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/skipped-and-todo-test will be considered new - receiver.setKnownTests({}) - - const NUM_RETRIES_EFD = 5 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - const newSkippedTests = tests.filter( - test => test.meta[TEST_NAME] === 'ci visibility skip will not be retried' - ) - assert.equal(newSkippedTests.length, 1) - assert.notProperty(newSkippedTests[0].meta, TEST_IS_RETRY) - - if (name === 'jest') { - const newTodoTests = tests.filter( - test => test.meta[TEST_NAME] === 'ci visibility todo will not be retried' - ) - assert.equal(newTodoTests.length, 1) - assert.notProperty(newTodoTests[0].meta, TEST_IS_RETRY) - } - }) - - let TESTS_TO_RUN = 'test-early-flake-detection/skipped-and-todo-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test-early-flake-detection/skipped-and-todo-test.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { ...envVars, TESTS_TO_RUN }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('handles spaces in test names', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': 3 - }, - faulty_session_threshold: 100 - } - }) - // Tests from ci-visibility/test/skipped-and-todo-test will be considered new - receiver.setKnownTests({ - [name]: { - 'ci-visibility/test-early-flake-detection/weird-test-names.js': [ - 'no describe can do stuff', - 'describe trailing space ' - ] - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.equal(tests.length, 2) - - const resourceNames = tests.map(test => test.resource) - - assert.includeMembers(resourceNames, - [ - 'ci-visibility/test-early-flake-detection/weird-test-names.js.no describe can do stuff', - 'ci-visibility/test-early-flake-detection/weird-test-names.js.describe trailing space ' - ] - ) - - const newTests = tests.filter( - test => test.meta[TEST_IS_NEW] === 'true' - ) - // no new tests - assert.equal(newTests.length, 0) - }) - - let TESTS_TO_RUN = 'test-early-flake-detection/weird-test-names' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test-early-flake-detection/weird-test-names.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...envVars, - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not run EFD if the known tests request fails', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - receiver.setKnownTestsResponseCode(500) - - const NUM_RETRIES_EFD = 5 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - assert.equal(tests.length, 2) - const newTests = tests.filter( - test => test.meta[TEST_IS_NEW] === 'true' - ) - assert.equal(newTests.length, 0) - }) - - let TESTS_TO_RUN = 'test/ci-visibility-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/ci-visibility-test.js', - './test/ci-visibility-test-2.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...envVars, - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => done()).catch(done) - }) - }) - it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/occasionally-failing-test will be considered new - receiver.setKnownTests({}) - - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - // all but one has been retried - assert.equal( - tests.length - 1, - retriedTests.length - ) - assert.equal(retriedTests.length, NUM_RETRIES_EFD) - // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, - // based on the global counter in the test file - const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') - const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') - assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) - assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) - // Test name does not change - retriedTests.forEach(test => { - assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') - }) - }) - - const command = name === 'jest' - ? 'node ./node_modules/jest/bin/jest --config config-jest.js' - : 'node ./node_modules/mocha/bin/mocha ci-visibility/test-early-flake-detection/occasionally-failing-test*' - - childProcess = exec( - command, - { - cwd, - env: { - ...envVars, - TESTS_TO_RUN: '**/ci-visibility/test-early-flake-detection/occasionally-failing-test*' - }, - stdio: 'inherit' - } - ) - - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - - childProcess.on('exit', (exitCode) => { - if (name === 'jest') { - assert.include(testOutput, '2 failed, 2 passed') - } else { - assert.include(testOutput, '2 passing') - assert.include(testOutput, '2 failing') - } - assert.equal(exitCode, 0) - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - if (name === 'jest') { - it('does not run early flake detection on snapshot tests', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test-early-flake-detection/jest-snapshot.js will be considered new - // but we don't retry them because they have snapshots - receiver.setKnownTests({}) - - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - assert.equal(tests.length, 1) - - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - - assert.equal(retriedTests.length, 0) - - // we still detect that it's new - const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 1) - }) - - childProcess = exec(runTestsWithCoverageCommand, { - cwd, - env: { - ...envVars, - TESTS_TO_RUN: 'ci-visibility/test-early-flake-detection/jest-snapshot', - CI: '1' // needs to be run as CI so snapshots are not written - }, - stdio: 'inherit' - }) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('bails out of EFD if the percentage of new tests is too high', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/ci-visibility-test* will be considered new - receiver.setKnownTests({}) - - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 1 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.equal(tests.length, 2) - - const newTests = tests.filter( - test => test.meta[TEST_IS_NEW] === 'true' - ) - // no new tests - assert.equal(newTests.length, 0) - }) - - childProcess = exec(runTestsWithCoverageCommand, { - cwd, - env: { - ...envVars, - TESTS_TO_RUN: 'test/ci-visibility-test' - }, - stdio: 'inherit' - }) - - childProcess.on('exit', () => { - eventsPromise.then(() => done()).catch(done) - }) - }) - - it('works with jsdom', (done) => { - const envVars = reportingOption === 'agentless' - ? getCiVisAgentlessConfig(receiver.port) - : getCiVisEvpProxyConfig(receiver.port) - if (reportingOption === 'evp proxy') { - receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) - } - // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new - receiver.setKnownTests({ - [name]: { - 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] - } - }) - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - }, - faulty_session_threshold: 100 - } - }) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const tests = events.filter(event => event.type === 'test').map(event => event.content) - - // no other tests are considered new - const oldTests = tests.filter(test => - test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' - ) - oldTests.forEach(test => { - assert.notProperty(test.meta, TEST_IS_NEW) - }) - assert.equal(oldTests.length, 1) - - const newTests = tests.filter(test => - test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' - ) - newTests.forEach(test => { - assert.propertyVal(test.meta, TEST_IS_NEW, 'true') - }) - const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - // all but one has been retried - assert.equal( - newTests.length - 1, - retriedTests.length - ) - assert.equal(retriedTests.length, NUM_RETRIES_EFD) - // Test name does not change - newTests.forEach(test => { - assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') - }) - }) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...envVars, - TESTS_TO_RUN: 'test/ci-visibility-test', - ENABLE_JSDOM: true, - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - } - }) - }) - - it('can run tests and report spans', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - receiver.payloadReceived(({ url }) => url === '/v0.4/traces').then(({ payload }) => { - const testSpans = payload.flatMap(trace => trace) - const resourceNames = testSpans.map(span => span.resource) - - assert.includeMembers(resourceNames, - [ - 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', - 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' - ] - ) - - const areAllTestSpans = testSpans.every(span => span.name === `${name}.test`) - assert.isTrue(areAllTestSpans) - - assert.include(testOutput, expectedStdout) - - if (extraStdout) { - assert.include(testOutput, extraStdout) - } - // Can read DD_TAGS - testSpans.forEach(testSpan => { - assert.propertyVal(testSpan.meta, 'test.customtag', 'customvalue') - assert.propertyVal(testSpan.meta, 'test.customtag2', 'customvalue2') - }) - - testSpans.forEach(testSpan => { - assert.equal(testSpan.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) - assert.exists(testSpan.metrics[TEST_SOURCE_START]) - }) - - done() - }) - - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: type === 'esm' ? `-r dd-trace/ci/init --loader=${hookFile}` : '-r dd-trace/ci/init', - DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - }) - const envVarSettings = ['DD_TRACING_ENABLED', 'DD_TRACE_ENABLED'] - - envVarSettings.forEach(envVar => { - context(`when ${envVar}=false`, () => { - it('does not report spans but still runs tests', (done) => { - receiver.assertMessageReceived(() => { - done(new Error('Should not create spans')) - }).catch(() => {}) - - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: '-r dd-trace/ci/init', - [envVar]: 'false' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.include(testOutput, expectedStdout) - done() - }) - }) - }) - }) - context('when no ci visibility init is used', () => { - it('does not crash', (done) => { - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_TRACE_AGENT_PORT: receiver.port, - NODE_OPTIONS: '-r dd-trace/init' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.notInclude(testOutput, 'TypeError') - assert.notInclude(testOutput, 'Uncaught error outside test suite') - assert.include(testOutput, expectedStdout) - done() - }) - }) - }) - - describe('agentless', () => { - it('reports errors in test sessions', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') - const errorMessage = name === 'mocha' ? 'Failed tests: 1' : 'Failed test suites: 1. Failed tests: 1' - assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) - }) - - let TESTS_TO_RUN = 'test/fail-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/fail-test.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('does not init if DD_API_KEY is not set', (done) => { - receiver.assertMessageReceived(() => { - done(new Error('Should not create spans')) - }).catch(() => {}) - - childProcess = fork(startupTestFile, { - cwd, - env: { - DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, - NODE_OPTIONS: '-r dd-trace/ci/init' - }, - stdio: 'pipe' - }) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('message', () => { - assert.include(testOutput, expectedStdout) - assert.include(testOutput, 'DD_CIVISIBILITY_AGENTLESS_ENABLED is set, ' + - 'but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, ' + - 'so dd-trace will not be initialized.' - ) - done() - }) - }) - - it('can report git metadata', (done) => { - const searchCommitsRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/api/v2/git/repository/search_commits' - ) - const packfileRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/git/repository/packfile') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') - - Promise.all([ - searchCommitsRequestPromise, - packfileRequestPromise, - eventsRequestPromise - ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { - assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') - assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - - done() - }).catch(done) - - childProcess = fork(startupTestFile, { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'pipe' - }) - }) - - it('can report code coverage', (done) => { - let testOutput - const libraryConfigRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/api/v2/libraries/tests/services/setting' - ) - const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') - - Promise.all([ - libraryConfigRequestPromise, - codeCovRequestPromise, - eventsRequestPromise - ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { - assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') - - const [coveragePayload] = codeCovRequest.payload - assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') - - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - assert.include(coveragePayload.content, { - version: 2 - }) - const allCoverageFiles = codeCovRequest.payload - .flatMap(coverage => coverage.content.coverages) - .flatMap(file => file.files) - .map(file => file.filename) - - assert.includeMembers(allCoverageFiles, expectedCoverageFiles) - assert.exists(coveragePayload.content.coverages[0].test_session_id) - assert.exists(coveragePayload.content.coverages[0].test_suite_id) - - const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'pipe' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - // coverage report - if (name === 'mocha') { - assert.include(testOutput, 'Lines ') - } - done() - }) - }) - - it('does not report code coverage if disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report code coverage') - done(error) - }, ({ url }) => url === '/api/v2/citestcov').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.propertyVal(headers, 'dd-api-key', '1') - const eventTypes = payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - const testModule = payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - const skippableRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/ci/tests/skippable') - const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') - - Promise.all([ - skippableRequestPromise, - coverageRequestPromise, - eventsRequestPromise - ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { - assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') - const [coveragePayload] = coverageRequest.payload - assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - - assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') - const eventTypes = eventsRequest.payload.events.map(event => event.type) - const skippedSuite = eventsRequest.payload.events.find(event => - event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) - const testModule = eventsRequest.payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) - done() - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('marks the test session as skipped if every suite is skipped', (done) => { - receiver.setSuitesToSkip( - [ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test-2.js' - } - } - ] - ) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('does not skip tests if git metadata upload fails', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - receiver.setGitUploadStatus(404) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.propertyVal(headers, 'dd-api-key', '1') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('does not skip tests if test skipping is disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.propertyVal(headers, 'dd-api-key', '1') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('does not skip suites if suite is marked as unskippable', (done) => { - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/unskippable-test/test-to-skip.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/unskippable-test/test-unskippable.js' - } - } - ]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 3) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - - const passedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' - ) - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' - ) - const forcedToRunSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' - ) - // It does not mark as unskippable if there is no docblock - assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') - assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) - - assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') - assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) - - assert.propertyVal(forcedToRunSuite.content.meta, TEST_STATUS, 'pass') - assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_FORCED_RUN, 'true') - }, 25000) - - let TESTS_TO_RUN = 'unskippable-test/test-' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './unskippable-test/test-to-run.js', - './unskippable-test/test-to-skip.js', - './unskippable-test/test-unskippable.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('only sets forced to run if suite was going to be skipped by ITR', (done) => { - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/unskippable-test/test-to-skip.js' - } - } - ]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 3) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_module_end').content - assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - - const passedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' - ) - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' - ).content - const nonSkippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' - ).content - - // It does not mark as unskippable if there is no docblock - assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') - assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) - - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - - assert.propertyVal(nonSkippedSuite.meta, TEST_STATUS, 'pass') - assert.propertyVal(nonSkippedSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') - // it was not forced to run because it wasn't going to be skipped - assert.notProperty(nonSkippedSuite.meta, TEST_ITR_FORCED_RUN) - }, 25000) - - let TESTS_TO_RUN = 'unskippable-test/test-' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './unskippable-test/test-to-run.js', - './unskippable-test/test-to-skip.js', - './unskippable-test/test-unskippable.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/not-existing-test.js' - } - }]) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, 25000) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('reports itr_correlation_id in test suites', (done) => { - const itrCorrelationId = '4321' - receiver.setItrCorrelationId(itrCorrelationId) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - testSuites.forEach(testSuite => { - assert.equal(testSuite.itr_correlation_id, itrCorrelationId) - }) - }, 25000) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - }) - - describe('evp proxy', () => { - context('if the agent is not event platform proxy compatible', () => { - it('does not do any intelligent test runner request', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits').catch(() => {}) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/api/v2/git/repository/search_commits').catch(() => {}) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/api/v2/libraries/tests/services/setting').catch(() => {}) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting').catch(() => {}) - - receiver.assertPayloadReceived(({ payload }) => { - const testSpans = payload.flatMap(trace => trace) - const resourceNames = testSpans.map(span => span.resource) - - assert.includeMembers(resourceNames, - [ - 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', - 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' - ] - ) - }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) - - childProcess = fork(startupTestFile, { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'pipe' - }) - }) - }) - - it('reports errors in test sessions', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') - const errorMessage = name === 'mocha' ? 'Failed tests: 1' : 'Failed test suites: 1. Failed tests: 1' - assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) - }) - - let TESTS_TO_RUN = 'test/fail-test' - if (name === 'mocha') { - TESTS_TO_RUN = JSON.stringify([ - './test/fail-test.js' - ]) - } - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: { - ...getCiVisEvpProxyConfig(receiver.port), - TESTS_TO_RUN - }, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('can report git metadata', (done) => { - const infoRequestPromise = receiver.payloadReceived(({ url }) => url === '/info') - const searchCommitsRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits' - ) - const packFileRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/packfile' - ) - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle') - - Promise.all([ - infoRequestPromise, - searchCommitsRequestPromise, - packFileRequestPromise, - eventsRequestPromise - ]).then(([infoRequest, searchCommitsRequest, packfileRequest, eventsRequest]) => { - assert.notProperty(infoRequest.headers, 'dd-api-key') - - assert.notProperty(searchCommitsRequest.headers, 'dd-api-key') - assert.propertyVal(searchCommitsRequest.headers, 'x-datadog-evp-subdomain', 'api') - - assert.notProperty(packfileRequest.headers, 'dd-api-key') - assert.propertyVal(packfileRequest.headers, 'x-datadog-evp-subdomain', 'api') - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - done() - }).catch(done) - - childProcess = fork(startupTestFile, { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'pipe' - }) - }) - - it('can report code coverage', (done) => { - let testOutput - const libraryConfigRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting' - ) - const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle') - - Promise.all([ - libraryConfigRequestPromise, - codeCovRequestPromise, - eventsRequestPromise - ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { - assert.notProperty(libraryConfigRequest.headers, 'dd-api-key') - assert.propertyVal(libraryConfigRequest.headers, 'x-datadog-evp-subdomain', 'api') - - const [coveragePayload] = codeCovRequest.payload - assert.notProperty(codeCovRequest.headers, 'dd-api-key') - - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - assert.include(coveragePayload.content, { - version: 2 - }) - const allCoverageFiles = codeCovRequest.payload - .flatMap(coverage => coverage.content.coverages) - .flatMap(file => file.files) - .map(file => file.filename) - - assert.includeMembers(allCoverageFiles, expectedCoverageFiles) - assert.exists(coveragePayload.content.coverages[0].test_session_id) - assert.exists(coveragePayload.content.coverages[0].test_suite_id) - - const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'pipe' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - // coverage report - if (name === 'mocha') { - assert.include(testOutput, 'Lines ') - } - done() - }) - }) - - it('does not report code coverage if disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report code coverage') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcov').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.notProperty(headers, 'dd-api-key') - assert.propertyVal(headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - const skippableRequestPromise = receiver.payloadReceived( - ({ url }) => url === '/evp_proxy/v2/api/v2/ci/tests/skippable' - ) - const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcov') - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle') - - Promise.all([ - skippableRequestPromise, - coverageRequestPromise, - eventsRequestPromise - ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { - assert.notProperty(skippableRequest.headers, 'dd-api-key') - assert.propertyVal(skippableRequest.headers, 'x-datadog-evp-subdomain', 'api') - - const [coveragePayload] = coverageRequest.payload - assert.notProperty(coverageRequest.headers, 'dd-api-key') - assert.propertyVal(coverageRequest.headers, 'x-datadog-evp-subdomain', 'citestcov-intake') - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - - assert.notProperty(eventsRequest.headers, 'dd-api-key') - assert.propertyVal(eventsRequest.headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = eventsRequest.payload.events.map(event => event.type) - const skippedSuite = eventsRequest.payload.events.find(event => - event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') - - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - done() - }).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('marks the test session as skipped if every suite is skipped', (done) => { - receiver.setSuitesToSkip( - [ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test-2.js' - } - } - ] - ) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('does not skip tests if git metadata upload fails', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.notProperty(headers, 'dd-api-key') - assert.propertyVal(headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle').then(() => done()).catch(done) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - receiver.setGitUploadStatus(404) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('does not skip tests if test skipping is disabled by the API', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/ci/tests/skippable').catch(() => {}) - - receiver.assertPayloadReceived(({ headers, payload }) => { - assert.notProperty(headers, 'dd-api-key') - assert.propertyVal(headers, 'x-datadog-evp-subdomain', 'citestcycle-intake') - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle').then(() => done()).catch(done) - - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }]) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - - it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: 'ci-visibility/test/not-existing-test.js' - } - }]) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, 25000) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - - it('reports itr_correlation_id in test suites', (done) => { - const itrCorrelationId = '4321' - receiver.setItrCorrelationId(itrCorrelationId) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - testSuites.forEach(testSuite => { - assert.equal(testSuite.itr_correlation_id, itrCorrelationId) - }) - }, 25000) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - }) - }) -}) diff --git a/integration-tests/ci-visibility/vitest-tests/sum.mjs b/integration-tests/ci-visibility/vitest-tests/sum.mjs new file mode 100644 index 00000000000..f1c6520acbd --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/sum.mjs @@ -0,0 +1,3 @@ +export function sum (a, b) { + return a + b +} diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs new file mode 100644 index 00000000000..a97f95e0df1 --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs @@ -0,0 +1,26 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { sum } from './sum' + +describe('context', () => { + beforeEach(() => { + throw new Error('failed before each') + }) + test('can report failed test', () => { + expect(sum(1, 2)).to.equal(4) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) + +describe('other context', () => { + afterEach(() => { + throw new Error('failed after each') + }) + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs new file mode 100644 index 00000000000..f2df345a87f --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-failed-suite.mjs @@ -0,0 +1,29 @@ +import { describe, test, expect, beforeEach, afterEach } from 'vitest' +import { sum } from './sum' + +let preparedValue = 1 + +describe('test-visibility-failed-suite-first-describe', () => { + beforeEach(() => { + preparedValue = 2 + }) + test('can report failed test', () => { + expect(sum(1, 2)).to.equal(4) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + expect(preparedValue).to.equal(2) + }) +}) + +describe('test-visibility-failed-suite-second-describe', () => { + afterEach(() => { + preparedValue = 1 + }) + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs new file mode 100644 index 00000000000..c2cf93431d8 --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs @@ -0,0 +1,32 @@ +import { describe, test, expect } from 'vitest' +import { sum } from './sum' + +describe('context', () => { + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) +}) + +describe('other context', () => { + test('can report passed test', () => { + expect(sum(1, 2)).to.equal(3) + }) + test('can report more', () => { + expect(sum(1, 2)).to.equal(3) + }) + test.skip('can skip', () => { + expect(sum(1, 2)).to.equal(3) + }) + test.todo('can todo', () => { + expect(sum(1, 2)).to.equal(3) + }) + // eslint-disable-next-line + test('can programmatic skip', (context) => { + // eslint-disable-next-line + context.skip() + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 3018fcd6ce0..eed8ce7253b 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -87,8 +87,7 @@ moduleType.forEach(({ }) beforeEach(async function () { - const port = await getPort() - receiver = await new FakeCiVisIntake(port).start() + receiver = await new FakeCiVisIntake().start() }) afterEach(async () => { @@ -117,7 +116,8 @@ moduleType.forEach(({ env: { ...restEnvVars, CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, - DD_SITE: '= invalid = url' + DD_SITE: '= invalid = url', + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -130,7 +130,7 @@ moduleType.forEach(({ }) childProcess.on('exit', () => { assert.notInclude(testOutput, 'TypeError') - assert.include(testOutput, '3 of 4 failed') + assert.include(testOutput, '1 of 1 failed') done() }) }) @@ -351,7 +351,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -383,7 +384,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -399,6 +401,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not report code coverage if disabled by the API', (done) => { receiver.setSettings({ code_coverage: false, @@ -428,7 +431,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -440,6 +444,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('can skip tests received by the intelligent test runner API and still reports code coverage', (done) => { receiver.setSuitesToSkip([{ type: 'test', @@ -498,7 +503,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js' }, stdio: 'pipe' } @@ -509,6 +515,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not skip tests if test skipping is disabled by the API', (done) => { receiver.setSettings({ code_coverage: true, @@ -535,6 +542,7 @@ moduleType.forEach(({ event.content.resource === 'cypress/e2e/other.cy.js.context passes' ) assert.exists(notSkippedTest) + assert.equal(notSkippedTest.content.meta[TEST_STATUS], 'pass') }, 25000) const { @@ -548,7 +556,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/other.cy.js' }, stdio: 'pipe' } @@ -560,6 +569,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not skip tests if suite is marked as unskippable', (done) => { receiver.setSettings({ code_coverage: true, @@ -621,7 +631,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js' }, stdio: 'pipe' } @@ -633,6 +644,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('only sets forced to run if test was going to be skipped by ITR', (done) => { receiver.setSettings({ code_coverage: true, @@ -689,7 +701,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js' }, stdio: 'pipe' } @@ -701,6 +714,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('sets _dd.ci.itr.tests_skipped to false if the received test is not skipped', (done) => { receiver.setSuitesToSkip([{ type: 'test', @@ -741,7 +755,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -752,6 +767,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('reports itr_correlation_id in tests', (done) => { const itrCorrelationId = '4321' receiver.setItrCorrelationId(itrCorrelationId) @@ -775,7 +791,8 @@ moduleType.forEach(({ cwd, env: { ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -816,7 +833,8 @@ moduleType.forEach(({ env: { ...restEnvVars, CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, - CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN: '1' + CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN: '1', + SPEC_PATTERN: 'cypress/e2e/spec.cy.js' }, stdio: 'pipe' } @@ -841,7 +859,7 @@ moduleType.forEach(({ assert.equal(testSuiteEvents.length, 4) const testEvents = events.filter(event => event.type === 'test') assert.equal(testEvents.length, 9) - }) + }, 30000) const { NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress @@ -880,7 +898,7 @@ moduleType.forEach(({ assert.equal(testSuiteEvents.length, 4) const testEvents = events.filter(event => event.type === 'test') assert.equal(testEvents.length, 9) - }) + }, 30000) const { NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress @@ -980,6 +998,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { receiver.setSettings({ itr_enabled: false, @@ -1041,6 +1060,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not retry tests that are skipped', (done) => { receiver.setSettings({ itr_enabled: false, @@ -1097,6 +1117,7 @@ moduleType.forEach(({ }).catch(done) }) }) + it('does not run EFD if the known tests request fails', (done) => { receiver.setSettings({ itr_enabled: false, diff --git a/integration-tests/cypress/plugins-old/index.js b/integration-tests/cypress/plugins-old/index.js index 6e7415900a9..52fc380ba09 100644 --- a/integration-tests/cypress/plugins-old/index.js +++ b/integration-tests/cypress/plugins-old/index.js @@ -1,6 +1,27 @@ +const ddAfterRun = require('dd-trace/ci/cypress/after-run') +const ddAfterSpec = require('dd-trace/ci/cypress/after-spec') + module.exports = (on, config) => { if (process.env.CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN) { require('cypress-fail-fast/plugin')(on, config) } + if (process.env.SPEC_PATTERN) { + config.testFiles = process.env.SPEC_PATTERN.replace('cypress/e2e/', '') + } + if (process.env.CYPRESS_ENABLE_AFTER_RUN_CUSTOM) { + on('after:run', (...args) => { + // do custom stuff + // and call after-run at the end + return ddAfterRun(...args) + }) + } + if (process.env.CYPRESS_ENABLE_AFTER_SPEC_CUSTOM) { + on('after:spec', (...args) => { + // do custom stuff + // and call after-spec at the end + return ddAfterSpec(...args) + }) + } require('dd-trace/ci/cypress/plugin')(on, config) + return config } diff --git a/integration-tests/esbuild/basic-test.js b/integration-tests/esbuild/basic-test.js index 20f53708b44..dc41b4efa53 100755 --- a/integration-tests/esbuild/basic-test.js +++ b/integration-tests/esbuild/basic-test.js @@ -1,11 +1,5 @@ #!/usr/bin/env node -// TODO: add support for Node.js v14.17+ and v16.0+ -if (Number(process.versions.node.split('.')[0]) < 16) { - console.error(`Skip esbuild test for node@${process.version}`) // eslint-disable-line no-console - process.exit(0) -} - const tracer = require('../../').init() // dd-trace const assert = require('assert') diff --git a/integration-tests/helpers.js b/integration-tests/helpers.js index b8972540e1f..98695c60156 100644 --- a/integration-tests/helpers.js +++ b/integration-tests/helpers.js @@ -7,15 +7,16 @@ const msgpack = require('msgpack-lite') const codec = msgpack.createCodec({ int64: true }) const EventEmitter = require('events') const childProcess = require('child_process') -const { fork } = childProcess +const { fork, spawn } = childProcess const exec = promisify(childProcess.exec) const http = require('http') -const fs = require('fs/promises') +const fs = require('fs') const os = require('os') const path = require('path') const rimraf = promisify(require('rimraf')) const id = require('../packages/dd-trace/src/id') const upload = require('multer')() +const assert = require('assert') const hookFile = 'dd-trace/loader-hook.mjs' @@ -79,7 +80,7 @@ class FakeAgent extends EventEmitter { // but it't not guaranteed to be the last one (so, expectedMessageCount would not be helpful). // It can still fail if it takes longer than `timeout` duration or if none pass the assertions (timeout still called) assertMessageReceived (fn, timeout, expectedMessageCount = 1, resolveAtFirstSuccess) { - timeout = timeout || 5000 + timeout = timeout || 30000 let resultResolve let resultReject let msgCount = 0 @@ -119,7 +120,7 @@ class FakeAgent extends EventEmitter { } assertTelemetryReceived (fn, timeout, requestType, expectedMessageCount = 1) { - timeout = timeout || 5000 + timeout = timeout || 30000 let resultResolve let resultReject let msgCount = 0 @@ -162,6 +163,95 @@ class FakeAgent extends EventEmitter { } } +async function runAndCheckOutput (filename, cwd, expectedOut) { + const proc = spawn('node', [filename], { cwd, stdio: 'pipe' }) + const pid = proc.pid + let out = await new Promise((resolve, reject) => { + proc.on('error', reject) + let out = Buffer.alloc(0) + proc.stdout.on('data', data => { + out = Buffer.concat([out, data]) + }) + proc.stderr.pipe(process.stdout) + proc.on('exit', () => resolve(out.toString('utf8'))) + setTimeout(() => { + if (proc.exitCode === null) proc.kill() + }, 1000) // TODO this introduces flakiness. find a better way to end the process. + }) + if (typeof expectedOut === 'function') { + expectedOut(out) + } else { + if (process.env.DD_TRACE_DEBUG) { + // Debug adds this, which we don't care about in these tests + out = out.replace('Flushing 0 metrics via HTTP\n', '') + } + assert.strictEqual(out, expectedOut) + } + return pid +} + +// This is set by the useSandbox function +let sandbox + +// This _must_ be used with the useSandbox function +async function runAndCheckWithTelemetry (filename, expectedOut, ...expectedTelemetryPoints) { + const cwd = sandbox.folder + const cleanup = telemetryForwarder(expectedTelemetryPoints) + const pid = await runAndCheckOutput(filename, cwd, expectedOut) + const msgs = await cleanup() + if (expectedTelemetryPoints.length === 0) { + // assert no telemetry sent + try { + assert.deepStrictEqual(msgs.length, 0) + } catch (e) { + // This console.log is useful for debugging telemetry. Plz don't remove. + // eslint-disable-next-line no-console + console.error('Expected no telemetry, but got:\n', msgs.map(msg => JSON.stringify(msg[1].points)).join('\n')) + throw e + } + return + } + let points = [] + for (const [telemetryType, data] of msgs) { + assert.strictEqual(telemetryType, 'library_entrypoint') + assert.deepStrictEqual(data.metadata, meta(pid)) + points = points.concat(data.points) + } + let expectedPoints = getPoints(...expectedTelemetryPoints) + // We now have to sort both the expected and actual telemetry points. + // This is because data can come in in any order. + // We'll just contatenate all the data together for each point and sort them. + points = points.map(p => p.name + '\t' + p.tags.join(',')).sort().join('\n') + expectedPoints = expectedPoints.map(p => p.name + '\t' + p.tags.join(',')).sort().join('\n') + assert.strictEqual(points, expectedPoints) + + function getPoints (...args) { + const expectedPoints = [] + let currentPoint = {} + for (const arg of args) { + if (!currentPoint.name) { + currentPoint.name = 'library_entrypoint.' + arg + } else { + currentPoint.tags = arg.split(',') + expectedPoints.push(currentPoint) + currentPoint = {} + } + } + return expectedPoints + } + + function meta (pid) { + return { + language_name: 'nodejs', + language_version: process.versions.node, + runtime_name: 'nodejs', + runtime_version: process.versions.node, + tracer_version: require('../package.json').version, + pid: Number(pid) + } + } +} + function spawnProc (filename, options = {}, stdioHandler) { const proc = fork(filename, { ...options, stdio: 'pipe' }) return new Promise((resolve, reject) => { @@ -205,9 +295,9 @@ async function createSandbox (dependencies = [], isGitRepo = false, // We might use NODE_OPTIONS to init the tracer. We don't want this to affect this operations const { NODE_OPTIONS, ...restOfEnv } = process.env - await fs.mkdir(folder) - await exec(`yarn pack --filename ${out}`) // TODO: cache this - await exec(`yarn add ${allDependencies.join(' ')}`, { cwd: folder, env: restOfEnv }) + fs.mkdirSync(folder) + await exec(`yarn pack --filename ${out}`, { env: restOfEnv }) // TODO: cache this + await exec(`yarn add ${allDependencies.join(' ')} --ignore-engines`, { cwd: folder, env: restOfEnv }) for (const path of integrationTestsPaths) { if (process.platform === 'win32') { @@ -229,7 +319,7 @@ async function createSandbox (dependencies = [], isGitRepo = false, if (isGitRepo) { await exec('git init', { cwd: folder }) - await fs.writeFile(path.join(folder, '.gitignore'), 'node_modules/', { flush: true }) + fs.writeFileSync(path.join(folder, '.gitignore'), 'node_modules/', { flush: true }) await exec('git config user.email "john@doe.com"', { cwd: folder }) await exec('git config user.name "John Doe"', { cwd: folder }) await exec('git config commit.gpgsign false', { cwd: folder }) @@ -245,6 +335,54 @@ async function createSandbox (dependencies = [], isGitRepo = false, } } +function telemetryForwarder (expectedTelemetryPoints) { + process.env.DD_TELEMETRY_FORWARDER_PATH = + path.join(__dirname, 'telemetry-forwarder.sh') + process.env.FORWARDER_OUT = path.join(__dirname, `forwarder-${Date.now()}.out`) + + let retries = 0 + + const tryAgain = async function () { + retries += 1 + await new Promise(resolve => setTimeout(resolve, 100)) + return cleanup() + } + + const cleanup = function () { + let msgs + try { + msgs = fs.readFileSync(process.env.FORWARDER_OUT, 'utf8').trim().split('\n') + } catch (e) { + if (expectedTelemetryPoints.length && e.code === 'ENOENT' && retries < 10) { + return tryAgain() + } + return [] + } + for (let i = 0; i < msgs.length; i++) { + const [telemetryType, data] = msgs[i].split('\t') + if (!data && retries < 10) { + return tryAgain() + } + let parsed + try { + parsed = JSON.parse(data) + } catch (e) { + if (!data && retries < 10) { + return tryAgain() + } + throw new SyntaxError(`error parsing data: ${e.message}\n${data}`) + } + msgs[i] = [telemetryType, parsed] + } + fs.unlinkSync(process.env.FORWARDER_OUT) + delete process.env.FORWARDER_OUT + delete process.env.DD_TELEMETRY_FORWARDER_PATH + return msgs + } + + return cleanup +} + async function curl (url, useHttp2 = false) { if (typeof url === 'object') { if (url.then) { @@ -313,14 +451,43 @@ async function spawnPluginIntegrationTestProc (cwd, serverFile, agentPort, stdio }, stdioHandler) } +function useEnv (env) { + before(() => { + Object.assign(process.env, env) + }) + after(() => { + for (const key of Object.keys(env)) { + delete process.env[key] + } + }) +} + +function useSandbox (...args) { + before(async () => { + sandbox = await createSandbox(...args) + }) + after(() => { + const oldSandbox = sandbox + sandbox = undefined + return oldSandbox.remove() + }) +} +function sandboxCwd () { + return sandbox.folder +} + module.exports = { FakeAgent, spawnProc, + runAndCheckWithTelemetry, createSandbox, curl, curlAndAssertMessage, getCiVisAgentlessConfig, getCiVisEvpProxyConfig, checkSpansForServiceName, - spawnPluginIntegrationTestProc + spawnPluginIntegrationTestProc, + useEnv, + useSandbox, + sandboxCwd } diff --git a/integration-tests/init.spec.js b/integration-tests/init.spec.js index f90968fc8c6..e61237311b8 100644 --- a/integration-tests/init.spec.js +++ b/integration-tests/init.spec.js @@ -1,107 +1,177 @@ +const semver = require('semver') const { - createSandbox, - spawnProc + runAndCheckWithTelemetry: testFile, + useEnv, + useSandbox, + sandboxCwd } = require('./helpers') -const { assert } = require('chai') const path = require('path') +const fs = require('fs') const DD_INJECTION_ENABLED = 'tracing' +const DD_INJECT_FORCE = 'true' +const DD_TRACE_DEBUG = 'true' -let cwd, proc, sandbox +const telemetryAbort = ['abort', 'reason:incompatible_runtime', 'abort.runtime', ''] +const telemetryForced = ['complete', 'injection_forced:true'] +const telemetryGood = ['complete', 'injection_forced:false'] -async function runTest (cwd, file, env, expected) { - return new Promise((resolve, reject) => { - spawnProc(path.join(cwd, file), { cwd, env, silent: true }, data => { - try { - assert.strictEqual(data.toString(), expected) - resolve() - } catch (e) { - reject(e) - } - }).then(subproc => { - proc = subproc - }) - }) -} +const { engines } = require('../package.json') +const supportedRange = engines.node +const currentVersionIsSupported = semver.satisfies(process.versions.node, supportedRange) function testInjectionScenarios (arg, filename, esmWorks = false) { - context('when dd-trace is not in the app dir', () => { - const NODE_OPTIONS = `--no-warnings --${arg} ${path.join(__dirname, '..', filename)}` - it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/trace.js', { NODE_OPTIONS }, 'true\n') - }) - it('should not initialize the tracer, if DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/trace.js', { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'false\n') - }) - it('should initialize instrumentation, if no DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/instrument.js', { NODE_OPTIONS }, 'true\n') - }) - it('should not initialize instrumentation, if DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/instrument.js', { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'false\n') - }) - it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation, if no DD_INJECTION_ENABLED`, () => { - return runTest(cwd, 'init/instrument.mjs', { NODE_OPTIONS }, `${esmWorks}\n`) + if (!currentVersionIsSupported) return + const doTest = (file, ...args) => testFile(file, ...args) + context('preferring app-dir dd-trace', () => { + context('when dd-trace is not in the app dir', () => { + const NODE_OPTIONS = `--no-warnings --${arg} ${path.join(__dirname, '..', filename)}` + useEnv({ NODE_OPTIONS }) + + context('without DD_INJECTION_ENABLED', () => { + it('should initialize the tracer', () => doTest('init/trace.js', 'true\n')) + it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n')) + it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () => + doTest('init/instrument.mjs', `${esmWorks}\n`)) + }) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + it('should not initialize the tracer', () => doTest('init/trace.js', 'false\n')) + it('should not initialize instrumentation', () => doTest('init/instrument.js', 'false\n')) + it('should not initialize ESM instrumentation', () => doTest('init/instrument.mjs', 'false\n')) + }) }) - it('should not initialize ESM instrumentation, if DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/instrument.mjs', { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'false\n') + context('when dd-trace in the app dir', () => { + const NODE_OPTIONS = `--no-warnings --${arg} dd-trace/${filename}` + useEnv({ NODE_OPTIONS }) + + context('without DD_INJECTION_ENABLED', () => { + it('should initialize the tracer', () => doTest('init/trace.js', 'true\n')) + it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n')) + it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () => + doTest('init/instrument.mjs', `${esmWorks}\n`)) + }) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + it('should initialize the tracer', () => doTest('init/trace.js', 'true\n', ...telemetryGood)) + it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n', ...telemetryGood)) + it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () => + doTest('init/instrument.mjs', `${esmWorks}\n`, ...telemetryGood)) + }) }) }) - context('when dd-trace in the app dir', () => { - const NODE_OPTIONS = `--no-warnings --${arg} dd-trace/${filename}` - it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/trace.js', { NODE_OPTIONS }, 'true\n') - }) - it('should initialize the tracer, if DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/trace.js', { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'true\n') - }) - it('should initialize instrumentation, if no DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/instrument.js', { NODE_OPTIONS }, 'true\n') - }) - it('should initialize instrumentation, if DD_INJECTION_ENABLED', () => { - return runTest(cwd, 'init/instrument.js', { NODE_OPTIONS, DD_INJECTION_ENABLED }, 'true\n') - }) - it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation, if no DD_INJECTION_ENABLED`, () => { - return runTest(cwd, 'init/instrument.mjs', { NODE_OPTIONS }, `${esmWorks}\n`) - }) - it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation, if DD_INJECTION_ENABLED`, () => { - return runTest(cwd, 'init/instrument.mjs', { NODE_OPTIONS, DD_INJECTION_ENABLED }, `${esmWorks}\n`) - }) +} + +function testRuntimeVersionChecks (arg, filename) { + context('runtime version check', () => { + const NODE_OPTIONS = `--${arg} dd-trace/${filename}` + const doTest = (...args) => testFile('init/trace.js', ...args) + const doTestForced = async (...args) => { + Object.assign(process.env, { DD_INJECT_FORCE }) + try { + await testFile('init/trace.js', ...args) + } finally { + delete process.env.DD_INJECT_FORCE + } + } + + if (!currentVersionIsSupported) { + context('when node version is less than engines field', () => { + useEnv({ NODE_OPTIONS }) + + it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => + doTest('true\n')) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + context('without debug', () => { + it('should not initialize the tracer', () => doTest('false\n', ...telemetryAbort)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => doTestForced('true\n', ...telemetryForced)) + }) + context('with debug', () => { + useEnv({ DD_TRACE_DEBUG }) + + it('should not initialize the tracer', () => + doTest(`Aborting application instrumentation due to incompatible_runtime. +Found incompatible runtime nodejs ${process.versions.node}, Supported runtimes: nodejs >=18. +false +`, ...telemetryAbort)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => + doTestForced(`Aborting application instrumentation due to incompatible_runtime. +Found incompatible runtime nodejs ${process.versions.node}, Supported runtimes: nodejs >=18. +DD_INJECT_FORCE enabled, allowing unsupported runtimes and continuing. +Application instrumentation bootstrapping complete +true +`, ...telemetryForced)) + }) + }) + }) + } else { + context('when node version is more than engines field', () => { + useEnv({ NODE_OPTIONS }) + + it('should initialize the tracer, if no DD_INJECTION_ENABLED', () => doTest('true\n')) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + + context('without debug', () => { + it('should initialize the tracer', () => doTest('true\n', ...telemetryGood)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => + doTestForced('true\n', ...telemetryGood)) + }) + context('with debug', () => { + useEnv({ DD_TRACE_DEBUG }) + + it('should initialize the tracer', () => + doTest('Application instrumentation bootstrapping complete\ntrue\n', ...telemetryGood)) + it('should initialize the tracer, if DD_INJECT_FORCE', () => + doTestForced('Application instrumentation bootstrapping complete\ntrue\n', ...telemetryGood)) + }) + }) + }) + } }) } +function stubTracerIfNeeded () { + if (!currentVersionIsSupported) { + before(() => { + // Stub out the tracer in the sandbox, since it will not likely load properly. + // We're only doing this on versions we don't support, since the forcing + // action results in undefined behavior in the tracer. + fs.writeFileSync( + path.join(sandboxCwd(), 'node_modules/dd-trace/index.js'), + 'exports.init = () => { Object.assign(global, { _ddtrace: true }) }' + ) + }) + } +} + describe('init.js', () => { - before(async () => { - sandbox = await createSandbox() - cwd = sandbox.folder - }) - afterEach(() => { - proc && proc.kill() - }) - after(() => { - return sandbox.remove() - }) + useSandbox() + stubTracerIfNeeded() testInjectionScenarios('require', 'init.js', false) + testRuntimeVersionChecks('require', 'init.js') }) -describe('initialize.mjs', () => { - before(async () => { - sandbox = await createSandbox() - cwd = sandbox.folder - }) - afterEach(() => { - proc && proc.kill() - }) - after(() => { - return sandbox.remove() - }) +// ESM is not supportable prior to Node.js 12 +if (semver.satisfies(process.versions.node, '>=12')) { + describe('initialize.mjs', () => { + useSandbox() + stubTracerIfNeeded() - context('as --loader', () => { - testInjectionScenarios('loader', 'initialize.mjs', true) - }) - if (Number(process.versions.node.split('.')[0]) >= 18) { - context('as --import', () => { - testInjectionScenarios('import', 'initialize.mjs', true) + context('as --loader', () => { + testInjectionScenarios('loader', 'initialize.mjs', true) + testRuntimeVersionChecks('loader', 'initialize.mjs') }) - } -}) + if (Number(process.versions.node.split('.')[0]) >= 18) { + context('as --import', () => { + testInjectionScenarios('import', 'initialize.mjs', true) + testRuntimeVersionChecks('loader', 'initialize.mjs') + }) + } + }) +} diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js new file mode 100644 index 00000000000..4f9d01dbf8b --- /dev/null +++ b/integration-tests/jest/jest.spec.js @@ -0,0 +1,2117 @@ +'use strict' + +const { fork, exec } = require('child_process') +const path = require('path') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig, + getCiVisEvpProxyConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_CODE_COVERAGE_ENABLED, + TEST_ITR_SKIPPING_ENABLED, + TEST_ITR_TESTS_SKIPPED, + TEST_CODE_COVERAGE_LINES_PCT, + TEST_SUITE, + TEST_STATUS, + TEST_SKIPPED_BY_ITR, + TEST_ITR_SKIPPING_TYPE, + TEST_ITR_SKIPPING_COUNT, + TEST_ITR_UNSKIPPABLE, + TEST_ITR_FORCED_RUN, + TEST_SOURCE_FILE, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_NAME, + JEST_DISPLAY_NAME, + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_SOURCE_START +} = require('../../packages/dd-trace/src/plugins/util/test') +const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') + +const testFile = 'ci-visibility/run-jest.js' +const expectedStdout = 'Test Suites: 2 passed' +const expectedCoverageFiles = [ + 'ci-visibility/test/sum.js', + 'ci-visibility/test/ci-visibility-test.js', + 'ci-visibility/test/ci-visibility-test-2.js' +] +const runTestsWithCoverageCommand = 'node ./ci-visibility/run-jest.js' + +// TODO: add ESM tests +describe('jest CommonJS', () => { + let receiver + let childProcess + let sandbox + let cwd + let startupTestFile + let testOutput = '' + + before(async function () { + sandbox = await createSandbox(['jest', 'chai@v4', 'jest-jasmine2', 'jest-environment-jsdom'], true) + cwd = sandbox.folder + startupTestFile = path.join(cwd, testFile) + }) + + after(async function () { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + childProcess.kill() + testOutput = '' + await receiver.stop() + }) + + it('can run tests and report tests with the APM protocol (old agents)', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + receiver.payloadReceived(({ url }) => url === '/v0.4/traces').then(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + + const areAllTestSpans = testSpans.every(span => span.name === 'jest.test') + assert.isTrue(areAllTestSpans) + + assert.include(testOutput, expectedStdout) + + // Can read DD_TAGS + testSpans.forEach(testSpan => { + assert.propertyVal(testSpan.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testSpan.meta, 'test.customtag2', 'customvalue2') + }) + + testSpans.forEach(testSpan => { + assert.equal(testSpan.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testSpan.metrics[TEST_SOURCE_START]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + const nonLegacyReportingOptions = ['agentless', 'evp proxy'] + + nonLegacyReportingOptions.forEach((reportingOption) => { + it(`can run and report tests with ${reportingOption}`, (done) => { + const envVars = reportingOption === 'agentless' + ? getCiVisAgentlessConfig(receiver.port) + : getCiVisEvpProxyConfig(receiver.port) + if (reportingOption === 'evp proxy') { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + } + receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + assert.equal(suites.length, 2) + assert.exists(sessionEventContent) + assert.exists(moduleEventContent) + + assert.include(testOutput, expectedStdout) + + // Can read DD_TAGS + tests.forEach(testEvent => { + assert.propertyVal(testEvent.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testEvent.meta, 'test.customtag2', 'customvalue2') + }) + + tests.forEach(testEvent => { + assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testEvent.metrics[TEST_SOURCE_START]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + ...envVars, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + }) + + const envVarSettings = ['DD_TRACING_ENABLED', 'DD_TRACE_ENABLED'] + + envVarSettings.forEach(envVar => { + context(`when ${envVar}=false`, () => { + it('does not report spans but still runs tests', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + [envVar]: 'false' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + }) + + context('when no ci visibility init is used', () => { + it('does not crash', (done) => { + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude(testOutput, 'Uncaught error outside test suite') + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + + it('works when sharding', (done) => { + receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(events => { + const testSuiteEvents = events.payload.events.filter(event => event.type === 'test_suite_end') + assert.equal(testSuiteEvents.length, 3) + const testSuites = testSuiteEvents.map(span => span.content.meta[TEST_SUITE]) + + assert.includeMembers(testSuites, + [ + 'ci-visibility/sharding-test/sharding-test-5.js', + 'ci-visibility/sharding-test/sharding-test-4.js', + 'ci-visibility/sharding-test/sharding-test-1.js' + ] + ) + + const testSession = events.payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + + // We run the second shard + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/sharding-test/sharding-test-2.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/sharding-test/sharding-test-3.js' + } + } + ]) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'sharding-test/sharding-test', + TEST_SHARD: '2/2' + }, + stdio: 'inherit' + } + ) + + receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle').then(secondShardEvents => { + const testSuiteEvents = secondShardEvents.payload.events.filter(event => event.type === 'test_suite_end') + + // The suites for this shard are to be skipped + assert.equal(testSuiteEvents.length, 2) + + testSuiteEvents.forEach(testSuite => { + assert.propertyVal(testSuite.content.meta, TEST_STATUS, 'skip') + assert.propertyVal(testSuite.content.meta, TEST_SKIPPED_BY_ITR, 'true') + }) + + const testSession = secondShardEvents + .payload + .events + .find(event => event.type === 'test_session_end').content + + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 2) + + done() + }) + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'sharding-test/sharding-test', + TEST_SHARD: '1/2' + }, + stdio: 'inherit' + } + ) + }) + + it('does not crash when jest is badly initialized', (done) => { + childProcess = fork('ci-visibility/run-jest-bad-init.js', { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + assert.include(testOutput, expectedStdout) + done() + }) + }) + + it('does not crash when jest uses jest-jasmine2', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + OLD_RUNNER: 1, + NODE_OPTIONS: '-r dd-trace/ci/init', + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + done() + }) + }) + + context('when jest is using workers to run tests in parallel', () => { + it('reports tests when using the old agents', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + childProcess = fork(testFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + + receiver.gatherPayloads(({ url }) => url === '/v0.4/traces', 5000).then(tracesRequests => { + const testSpans = tracesRequests.flatMap(trace => trace.payload).flatMap(request => request) + assert.equal(testSpans.length, 2) + const spanTypes = testSpans.map(span => span.type) + assert.includeMembers(spanTypes, ['test']) + assert.notInclude(spanTypes, ['test_session_end', 'test_suite_end', 'test_module_end']) + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v2'] }) + done() + }).catch(done) + }) + + it('reports tests when using agentless', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + + receiver.gatherPayloads(({ url }) => url === '/api/v2/citestcycle', 5000).then(eventsRequests => { + const eventTypes = eventsRequests.map(({ payload }) => payload) + .flatMap(({ events }) => events) + .map(event => event.type) + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + done() + }).catch(done) + }) + + it('reports tests when using evp proxy', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + RUN_IN_PARALLEL: true + }, + stdio: 'pipe' + }) + + receiver.gatherPayloads(({ url }) => url === '/evp_proxy/v2/api/v2/citestcycle', 5000) + .then(eventsRequests => { + const eventTypes = eventsRequests.map(({ payload }) => payload) + .flatMap(({ events }) => events) + .map(event => event.type) + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + done() + }).catch(done) + }) + }) + + it('reports timeout error message', (done) => { + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NODE_OPTIONS: '-r dd-trace/ci/init', + RUN_IN_PARALLEL: true, + TESTS_TO_RUN: 'timeout-test/timeout-test.js' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, 'Exceeded timeout of 100 ms for a test') + done() + }) + }) + + it('reports parsing errors in the test file', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + assert.equal(suites.length, 2) + + const resourceNames = suites.map(suite => suite.content.resource) + + assert.includeMembers(resourceNames, [ + 'test_suite.ci-visibility/test-parsing-error/parsing-error-2.js', + 'test_suite.ci-visibility/test-parsing-error/parsing-error.js' + ]) + suites.forEach(suite => { + assert.equal(suite.content.meta[TEST_STATUS], 'fail') + assert.include(suite.content.meta[ERROR_MESSAGE], 'chao') + }) + }) + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'test-parsing-error/parsing-error' + }, + stdio: 'pipe' + }) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not report total code coverage % if user has not configured coverage manually', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) + + receiver.assertPayloadReceived(({ payload }) => { + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DISABLE_CODE_COVERAGE: '1' + }, + stdio: 'inherit' + } + ) + }) + + it('reports total code coverage % even when ITR is disabled', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(({ payload }) => { + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('works with --forceExit and logs a warning', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + assert.include(testOutput, "Jest's '--forceExit' flag has been passed") + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end') + const testModule = events.find(event => event.type === 'test_module_end') + const testSuites = events.filter(event => event.type === 'test_suite_end') + const tests = events.filter(event => event.type === 'test') + + assert.exists(testSession) + assert.exists(testModule) + assert.equal(testSuites.length, 2) + assert.equal(tests.length, 2) + }) + // Needs to run with the CLI if we want --forceExit to work + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_TRACE_DEBUG: '1', + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + it('does not hang if server is not available and logs an error', (done) => { + // Very slow intake + receiver.setWaitingTime(30000) + // Needs to run with the CLI if we want --forceExit to work + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest.js --forceExit', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_TRACE_DEBUG: '1', + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'inherit' + } + ) + const EXPECTED_FORCE_EXIT_LOG_MESSAGE = "Jest's '--forceExit' flag has been passed" + const EXPECTED_TIMEOUT_LOG_MESSAGE = 'Timeout waiting for the tracer to flush' + childProcess.on('exit', () => { + assert.include( + testOutput, + EXPECTED_FORCE_EXIT_LOG_MESSAGE, + `"${EXPECTED_FORCE_EXIT_LOG_MESSAGE}" log message is not in test output: ${testOutput}` + ) + assert.include( + testOutput, + EXPECTED_TIMEOUT_LOG_MESSAGE, + `"${EXPECTED_TIMEOUT_LOG_MESSAGE}" log message is not in the test output: ${testOutput}` + ) + done() + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + it('grabs the jest displayName config and sets tag in tests and suites', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 4) // two per display name + const nodeTests = tests.filter(test => test.meta[JEST_DISPLAY_NAME] === 'node') + assert.equal(nodeTests.length, 2) + + const standardTests = tests.filter(test => test.meta[JEST_DISPLAY_NAME] === 'standard') + assert.equal(standardTests.length, 2) + + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + assert.equal(suites.length, 4) + + const nodeSuites = suites.filter(suite => suite.meta[JEST_DISPLAY_NAME] === 'node') + assert.equal(nodeSuites.length, 2) + + const standardSuites = suites.filter(suite => suite.meta[JEST_DISPLAY_NAME] === 'standard') + assert.equal(standardSuites.length, 2) + }) + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest-multiproject.js', + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('reports errors in test sessions', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') + const errorMessage = 'Failed test suites: 1. Failed tests: 1' + assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'test/fail-test' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not init if DD_API_KEY is not set', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, + NODE_OPTIONS: '-r dd-trace/ci/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + assert.include(testOutput, 'DD_CIVISIBILITY_AGENTLESS_ENABLED is set, ' + + 'but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, ' + + 'so dd-trace will not be initialized.' + ) + done() + }) + }) + + it('can report git metadata', (done) => { + const searchCommitsRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/git/repository/search_commits' + ) + const packfileRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/git/repository/packfile') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + searchCommitsRequestPromise, + packfileRequestPromise, + eventsRequestPromise + ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { + assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') + assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + + done() + }).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + }) + + context('intelligent test runner', () => { + context('if the agent is not event platform proxy compatible', () => { + it('does not do any intelligent test runner request', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/api/v2/libraries/tests/services/setting').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting').catch(() => {}) + + receiver.assertPayloadReceived(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisEvpProxyConfig(receiver.port), + stdio: 'pipe' + }) + }) + }) + it('can report code coverage', (done) => { + const libraryConfigRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/libraries/tests/services/setting' + ) + const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + libraryConfigRequestPromise, + codeCovRequestPromise, + eventsRequestPromise + ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { + assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') + + const [coveragePayload] = codeCovRequest.payload + assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + assert.include(coveragePayload.content, { + version: 2 + }) + const allCoverageFiles = codeCovRequest.payload + .flatMap(coverage => coverage.content.coverages) + .flatMap(file => file.files) + .map(file => file.filename) + + assert.includeMembers(allCoverageFiles, expectedCoverageFiles) + assert.exists(coveragePayload.content.coverages[0].test_session_id) + assert.exists(coveragePayload.content.coverages[0].test_suite_id) + + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + done() + }) + }) + + it('does not report code coverage if disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report code coverage') + done(error) + }, ({ url }) => url === '/api/v2/citestcov').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const skippableRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/ci/tests/skippable') + const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + skippableRequestPromise, + coverageRequestPromise, + eventsRequestPromise + ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { + assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') + const [coveragePayload] = coverageRequest.payload + assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + + assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') + const eventTypes = eventsRequest.payload.events.map(event => event.type) + const skippedSuite = eventsRequest.payload.events.find(event => + event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) + const testModule = eventsRequest.payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) + done() + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('marks the test session as skipped if every suite is skipped', (done) => { + receiver.setSuitesToSkip( + [ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test-2.js' + } + } + ] + ) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not skip tests if git metadata upload fails', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.setGitUploadStatus(404) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip tests if test skipping is disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip suites if suite is marked as unskippable', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-unskippable.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ) + const forcedToRunSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ) + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(forcedToRunSuite.content.meta, TEST_STATUS, 'pass') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_FORCED_RUN, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'unskippable-test/test-' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ).content + const nonSkippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ).content + + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + + assert.propertyVal(nonSkippedSuite.meta, TEST_STATUS, 'pass') + assert.propertyVal(nonSkippedSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') + // it was not forced to run because it wasn't going to be skipped + assert.notProperty(nonSkippedSuite.meta, TEST_ITR_FORCED_RUN) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'unskippable-test/test-' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/not-existing-test.js' + } + }]) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('reports itr_correlation_id in test suites', (done) => { + const itrCorrelationId = '4321' + receiver.setItrCorrelationId(itrCorrelationId) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + testSuites.forEach(testSuite => { + assert.equal(testSuite.itr_correlation_id, itrCorrelationId) + }) + }, 25000) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('can skip when using a custom test sequencer', (done) => { + receiver.setSettings({ + itr_enabled: true, + tests_skipping: true + }) + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testEvents = events.filter(event => event.type === 'test') + // no tests end up running (suite is skipped) + assert.equal(testEvents.length, 0) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + + const skippedSuite = events.find(event => + event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + CUSTOM_TEST_SEQUENCER: './ci-visibility/jest-custom-test-sequencer.js', + TEST_SHARD: '2/2' + }, + stdio: 'inherit' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', () => { + assert.include(testOutput, 'Running shard with a custom sequencer') + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('works with multi project setup and test skipping', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + // suites for both projects in the multi-project config are reported as skipped + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + + const skippedSuites = testSuites.filter( + suite => suite.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ) + assert.equal(skippedSuites.length, 2) + + skippedSuites.forEach(skippedSuite => { + assert.equal(skippedSuite.meta[TEST_STATUS], 'skip') + assert.equal(skippedSuite.meta[TEST_SKIPPED_BY_ITR], 'true') + }) + }) + + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest-multiproject.js', + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('calculates executable lines even if there have been skipped suites', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test-total-code-coverage/test-skipped.js' + } + }]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + + // Before https://github.com/DataDog/dd-trace-js/pull/4336, this would've been 100% + // The reason is that skipping jest's `addUntestedFiles`, we would not see unexecuted lines. + // In this cause, these would be from the `unused-dependency.js` file. + // It is 50% now because we only cover 1 out of 2 files (`used-dependency.js`). + assert.propertyVal(testSession.metrics, TEST_CODE_COVERAGE_LINES_PCT, 50) + }) + + childProcess = exec( + runTestsWithCoverageCommand, // Requirement: the user must've opted in to code coverage + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'ci-visibility/test-total-code-coverage/test-', + COLLECT_COVERAGE_FROM: '**/test-total-code-coverage/**' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(done).catch(done) + }) + }) + }) + + context('early flake detection', () => { + it('retries new tests', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + // TODO: maybe check in stdout for the "Retried by Datadog" + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test/ci-visibility-test' }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('handles parameterized tests as a single unit', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test-early-flake-detection/test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const parameterizedTestFile = 'test-parameterized.js' + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === `ci-visibility/test-early-flake-detection/${parameterizedTestFile}` + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + // Each parameter is repeated independently + const testsForFirstParameter = tests.filter(test => test.resource === + `ci-visibility/test-early-flake-detection/${parameterizedTestFile}.parameterized test parameter 1` + ) + + const testsForSecondParameter = tests.filter(test => test.resource === + `ci-visibility/test-early-flake-detection/${parameterizedTestFile}.parameterized test parameter 2` + ) + + assert.equal(testsForFirstParameter.length, testsForSecondParameter.length) + + // all but one have been retried + assert.equal( + testsForFirstParameter.length - 1, + testsForFirstParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + + assert.equal( + testsForSecondParameter.length - 1, + testsForSecondParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test-early-flake-detection/test' }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + // new tests are not detected + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test/ci-visibility-test', + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('retries flaky tests', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test-early-flake-detection/occasionally-failing-test' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + // TODO: check exit code: if a new, retried test fails, the exit code should remain 0 + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not retry new tests that are skipped', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newSkippedTests = tests.filter( + test => test.meta[TEST_NAME] === 'ci visibility skip will not be retried' + ) + assert.equal(newSkippedTests.length, 1) + assert.notProperty(newSkippedTests[0].meta, TEST_IS_RETRY) + + const newTodoTests = tests.filter( + test => test.meta[TEST_NAME] === 'ci visibility todo will not be retried' + ) + assert.equal(newTodoTests.length, 1) + assert.notProperty(newTodoTests[0].meta, TEST_IS_RETRY) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test-early-flake-detection/skipped-and-todo-test' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('handles spaces in test names', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test-early-flake-detection/weird-test-names.js': [ + 'no describe can do stuff', + 'describe trailing space ' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const resourceNames = tests.map(test => test.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-early-flake-detection/weird-test-names.js.no describe can do stuff', + 'ci-visibility/test-early-flake-detection/weird-test-names.js.describe trailing space ' + ] + ) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test-early-flake-detection/weird-test-names' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not run EFD if the known tests request fails', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + receiver.setKnownTestsResponseCode(500) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 2) + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test/ci-visibility-test' + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + 'node ./node_modules/jest/bin/jest --config config-jest.js', + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: '**/ci-visibility/test-early-flake-detection/occasionally-failing-test*' + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', (exitCode) => { + assert.include(testOutput, '2 failed, 2 passed') + assert.equal(exitCode, 0) + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not run early flake detection on snapshot tests', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test-early-flake-detection/jest-snapshot.js will be considered new + // but we don't retry them because they have snapshots + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 1) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + + assert.equal(retriedTests.length, 0) + + // we still detect that it's new + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 1) + }) + + childProcess = exec(runTestsWithCoverageCommand, { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'ci-visibility/test-early-flake-detection/jest-snapshot', + CI: '1' // needs to be run as CI so snapshots are not written + }, + stdio: 'inherit' + }) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('bails out of EFD if the percentage of new tests is too high', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test* will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 1 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec(runTestsWithCoverageCommand, { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + TESTS_TO_RUN: 'test/ci-visibility-test' + }, + stdio: 'inherit' + }) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + + it('works with jsdom', (done) => { + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), // use agentless for this test, just for variety + TESTS_TO_RUN: 'test/ci-visibility-test', + ENABLE_JSDOM: true, + DD_TRACE_DEBUG: 1, + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) +}) diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js new file mode 100644 index 00000000000..9661c809c8f --- /dev/null +++ b/integration-tests/mocha/mocha.spec.js @@ -0,0 +1,1555 @@ +'use strict' + +const { fork, exec } = require('child_process') +const path = require('path') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig, + getCiVisEvpProxyConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_CODE_COVERAGE_ENABLED, + TEST_ITR_SKIPPING_ENABLED, + TEST_ITR_TESTS_SKIPPED, + TEST_CODE_COVERAGE_LINES_PCT, + TEST_SUITE, + TEST_STATUS, + TEST_SKIPPED_BY_ITR, + TEST_ITR_SKIPPING_TYPE, + TEST_ITR_SKIPPING_COUNT, + TEST_ITR_UNSKIPPABLE, + TEST_ITR_FORCED_RUN, + TEST_SOURCE_FILE, + TEST_IS_NEW, + TEST_IS_RETRY, + TEST_EARLY_FLAKE_ENABLED, + TEST_NAME, + TEST_COMMAND, + TEST_MODULE, + MOCHA_IS_PARALLEL, + TEST_SOURCE_START +} = require('../../packages/dd-trace/src/plugins/util/test') +const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') + +const runTestsWithCoverageCommand = './node_modules/nyc/bin/nyc.js -r=text-summary node ./ci-visibility/run-mocha.js' +const testFile = 'ci-visibility/run-mocha.js' +const expectedStdout = '2 passing' +const extraStdout = 'end event: can add event listeners to mocha' + +describe('mocha CommonJS', function () { + let receiver + let childProcess + let sandbox + let cwd + let startupTestFile + let testOutput = '' + + before(async function () { + sandbox = await createSandbox(['mocha', 'chai@v4', 'nyc', 'mocha-each', 'workerpool'], true) + cwd = sandbox.folder + startupTestFile = path.join(cwd, testFile) + }) + + after(async function () { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + childProcess.kill() + testOutput = '' + await receiver.stop() + }) + + it('can run tests and report tests with the APM protocol (old agents)', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + receiver.payloadReceived(({ url }) => url === '/v0.4/traces').then(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + + const areAllTestSpans = testSpans.every(span => span.name === 'mocha.test') + assert.isTrue(areAllTestSpans) + + assert.include(testOutput, expectedStdout) + + if (extraStdout) { + assert.include(testOutput, extraStdout) + } + // Can read DD_TAGS + testSpans.forEach(testSpan => { + assert.propertyVal(testSpan.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testSpan.meta, 'test.customtag2', 'customvalue2') + }) + + testSpans.forEach(testSpan => { + assert.equal(testSpan.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testSpan.metrics[TEST_SOURCE_START]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + const nonLegacyReportingOptions = ['agentless', 'evp proxy'] + + nonLegacyReportingOptions.forEach((reportingOption) => { + it(`can run and report tests with ${reportingOption}`, (done) => { + const envVars = reportingOption === 'agentless' + ? getCiVisAgentlessConfig(receiver.port) + : getCiVisEvpProxyConfig(receiver.port) + if (reportingOption === 'evp proxy') { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + } + receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + assert.equal(suites.length, 2) + assert.exists(sessionEventContent) + assert.exists(moduleEventContent) + + assert.include(testOutput, expectedStdout) + assert.include(testOutput, extraStdout) + + // Can read DD_TAGS + tests.forEach(testEvent => { + assert.propertyVal(testEvent.meta, 'test.customtag', 'customvalue') + assert.propertyVal(testEvent.meta, 'test.customtag2', 'customvalue2') + }) + + tests.forEach(testEvent => { + assert.equal(testEvent.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testEvent.metrics[TEST_SOURCE_START]) + }) + + done() + }) + + childProcess = fork(startupTestFile, { + cwd, + env: { + ...envVars, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + }) + + const envVarSettings = ['DD_TRACING_ENABLED', 'DD_TRACE_ENABLED'] + + envVarSettings.forEach(envVar => { + context(`when ${envVar}=false`, () => { + it('does not report spans but still runs tests', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/ci/init', + [envVar]: 'false' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + }) + + context('when no ci visibility init is used', () => { + it('does not crash', (done) => { + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: receiver.port, + NODE_OPTIONS: '-r dd-trace/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude(testOutput, 'Uncaught error outside test suite') + assert.include(testOutput, expectedStdout) + done() + }) + }) + }) + + it('does not change mocha config if CI Visibility fails to init', (done) => { + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report tests') + done(error) + }, ({ url }) => url === '/api/v2/citestcycle', 3000).catch(() => {}) + + const { DD_CIVISIBILITY_AGENTLESS_URL, ...restEnvVars } = getCiVisAgentlessConfig(receiver.port) + + // `runMocha` is only executed when using the CLI, which is where we modify mocha config + // if CI Visibility is init + childProcess = exec('mocha ./ci-visibility/test/ci-visibility-test.js', { + cwd, + env: { + ...restEnvVars, + DD_TRACE_DEBUG: 1, + DD_TRACE_LOG_LEVEL: 'error', + DD_SITE: '= invalid = url' + }, + stdio: 'pipe' + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + assert.include(testOutput, 'Invalid URL') + assert.include(testOutput, '1 passing') // we only run one file here + done() + }) + }) + + it('works with parallel mode', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') + assert.equal( + sessionEventContent.test_session_id.toString(10), + moduleEventContent.test_session_id.toString(10) + ) + suites.forEach(({ + meta, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) + }) + + tests.forEach(({ + meta, + metrics, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) + assert.propertyVal(meta, MOCHA_IS_PARALLEL, 'true') + assert.exists(metrics[TEST_SOURCE_START]) + }) + }) + + childProcess = fork(testFile, { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + RUN_IN_PARALLEL: true, + DD_TRACE_DEBUG: 1, + DD_TRACE_LOG_LEVEL: 'warn' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + eventsPromise.then(() => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude( + testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' + ) + done() + }).catch(done) + }) + }) + + it('works with parallel mode when run with the cli', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') + assert.equal(suites.length, 2) + assert.equal(tests.length, 2) + }) + + childProcess = exec('mocha --parallel --jobs 2 ./ci-visibility/test/ci-visibility-test*', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + eventsPromise.then(() => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude( + testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' + ) + done() + }).catch(done) + }) + }) + + it('does not blow up when workerpool is used outside of a test', (done) => { + childProcess = exec('node ./ci-visibility/run-workerpool.js', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', (code) => { + assert.include(testOutput, 'result 7') + assert.equal(code, 0) + done() + }) + }) + + it('reports errors in test sessions', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'fail') + const errorMessage = 'Failed tests: 1' + assert.include(testSession.meta[ERROR_MESSAGE], errorMessage) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/fail-test.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not init if DD_API_KEY is not set', (done) => { + receiver.assertMessageReceived(() => { + done(new Error('Should not create spans')) + }).catch(() => {}) + + childProcess = fork(startupTestFile, { + cwd, + env: { + DD_CIVISIBILITY_AGENTLESS_ENABLED: 1, + NODE_OPTIONS: '-r dd-trace/ci/init' + }, + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('message', () => { + assert.include(testOutput, expectedStdout) + assert.include(testOutput, 'DD_CIVISIBILITY_AGENTLESS_ENABLED is set, ' + + 'but neither DD_API_KEY nor DATADOG_API_KEY are set in your environment, ' + + 'so dd-trace will not be initialized.' + ) + done() + }) + }) + + it('can report git metadata', (done) => { + const searchCommitsRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/git/repository/search_commits' + ) + const packfileRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/git/repository/packfile') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + searchCommitsRequestPromise, + packfileRequestPromise, + eventsRequestPromise + ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { + assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') + assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + + done() + }).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + }) + + context('intelligent test runner', () => { + context('if the agent is not event platform proxy compatible', () => { + it('does not do any intelligent test runner request', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/api/v2/git/repository/search_commits').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/api/v2/libraries/tests/services/setting').catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting').catch(() => {}) + + receiver.assertPayloadReceived(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test/ci-visibility-test.js.ci visibility can report tests', + 'ci-visibility/test/ci-visibility-test-2.js.ci visibility 2 can report tests 2' + ] + ) + }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) + + childProcess = fork(startupTestFile, { + cwd, + env: getCiVisEvpProxyConfig(receiver.port), + stdio: 'pipe' + }) + }) + }) + it('can report code coverage', (done) => { + let testOutput + const libraryConfigRequestPromise = receiver.payloadReceived( + ({ url }) => url === '/api/v2/libraries/tests/services/setting' + ) + const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + libraryConfigRequestPromise, + codeCovRequestPromise, + eventsRequestPromise + ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { + assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') + + const [coveragePayload] = codeCovRequest.payload + assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + assert.include(coveragePayload.content, { + version: 2 + }) + const allCoverageFiles = codeCovRequest.payload + .flatMap(coverage => coverage.content.coverages) + .flatMap(file => file.files) + .map(file => file.filename) + + assert.includeMembers(allCoverageFiles, + [ + 'ci-visibility/test/sum.js', + 'ci-visibility/test/ci-visibility-test.js', + 'ci-visibility/test/ci-visibility-test-2.js' + ] + ) + assert.exists(coveragePayload.content.coverages[0].test_session_id) + assert.exists(coveragePayload.content.coverages[0].test_suite_id) + + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + // coverage report + assert.include(testOutput, 'Lines ') + done() + }) + }) + + it('does not report code coverage if disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report code coverage') + done(error) + }, ({ url }) => url === '/api/v2/citestcov').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + const skippableRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/ci/tests/skippable') + const coverageRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcov') + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url === '/api/v2/citestcycle') + + Promise.all([ + skippableRequestPromise, + coverageRequestPromise, + eventsRequestPromise + ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { + assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') + const [coveragePayload] = coverageRequest.payload + assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + + assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') + const eventTypes = eventsRequest.payload.events.map(event => event.type) + const skippedSuite = eventsRequest.payload.events.find(event => + event.content.resource === 'test_suite.ci-visibility/test/ci-visibility-test.js' + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = eventsRequest.payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) + const testModule = eventsRequest.payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) + done() + }).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('marks the test session as skipped if every suite is skipped', (done) => { + receiver.setSuitesToSkip( + [ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test-2.js' + } + } + ] + ) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') + }) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('does not skip tests if git metadata upload fails', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.setGitUploadStatus(404) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip tests if test skipping is disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) + + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/ci-visibility-test.js' + } + }]) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url === '/api/v2/ci/tests/skippable').catch(() => {}) + + receiver.assertPayloadReceived(({ headers, payload }) => { + assert.propertyVal(headers, 'dd-api-key', '1') + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }, ({ url }) => url === '/api/v2/citestcycle').then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + + it('does not skip suites if suite is marked as unskippable', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + }, + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-unskippable.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ) + const forcedToRunSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ) + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(forcedToRunSuite.content.meta, TEST_STATUS, 'pass') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(forcedToRunSuite.content.meta, TEST_ITR_FORCED_RUN, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './unskippable-test/test-to-run.js', + './unskippable-test/test-to-skip.js', + './unskippable-test/test-unskippable.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: 'ci-visibility/unskippable-test/test-to-skip.js' + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 3) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_module_end').content + assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + + const passedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-run.js' + ) + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-to-skip.js' + ).content + const nonSkippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/unskippable-test/test-unskippable.js' + ).content + + // It does not mark as unskippable if there is no docblock + assert.propertyVal(passedSuite.content.meta, TEST_STATUS, 'pass') + assert.notProperty(passedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(passedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + + assert.propertyVal(nonSkippedSuite.meta, TEST_STATUS, 'pass') + assert.propertyVal(nonSkippedSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') + // it was not forced to run because it wasn't going to be skipped + assert.notProperty(nonSkippedSuite.meta, TEST_ITR_FORCED_RUN) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './unskippable-test/test-to-run.js', + './unskippable-test/test-to-skip.js', + './unskippable-test/test-unskippable.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: 'ci-visibility/test/not-existing-test.js' + } + }]) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('reports itr_correlation_id in test suites', (done) => { + const itrCorrelationId = '4321' + receiver.setItrCorrelationId(itrCorrelationId) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + testSuites.forEach(testSuite => { + assert.equal(testSuite.itr_correlation_id, itrCorrelationId) + }) + }, 25000) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) + + context('early flake detection', () => { + it('retries new tests', (done) => { + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + // TODO: maybe check in stdout for the "Retried by Datadog" + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('handles parameterized tests as a single unit', (done) => { + // Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test-early-flake-detection/test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test-early-flake-detection/mocha-parameterized.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + // Each parameter is repeated independently + const testsForFirstParameter = tests.filter(test => test.resource === + 'ci-visibility/test-early-flake-detection/mocha-parameterized.js.parameterized test parameter 1' + ) + + const testsForSecondParameter = tests.filter(test => test.resource === + 'ci-visibility/test-early-flake-detection/mocha-parameterized.js.parameterized test parameter 2' + ) + + assert.equal(testsForFirstParameter.length, testsForSecondParameter.length) + + // all but one have been retried + assert.equal( + testsForFirstParameter.length - 1, + testsForFirstParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + + assert.equal( + testsForSecondParameter.length - 1, + testsForSecondParameter.filter(test => test.meta[TEST_IS_RETRY] === 'true').length + ) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/test.js', + './test-early-flake-detection/mocha-parameterized.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + // new tests are not detected + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]), + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('retries flaky tests', (done) => { + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/occasionally-failing-test.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + // TODO: check exit code: if a new, retried test fails, the exit code should remain 0 + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('does not retry new tests that are skipped', (done) => { + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const newSkippedTests = tests.filter( + test => test.meta[TEST_NAME] === 'ci visibility skip will not be retried' + ) + assert.equal(newSkippedTests.length, 1) + assert.notProperty(newSkippedTests[0].meta, TEST_IS_RETRY) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/skipped-and-todo-test.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('handles spaces in test names', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + }, + faulty_session_threshold: 100 + } + }) + // Tests from ci-visibility/test/skipped-and-todo-test will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test-early-flake-detection/weird-test-names.js': [ + 'no describe can do stuff', + 'describe trailing space ' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const resourceNames = tests.map(test => test.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/test-early-flake-detection/weird-test-names.js.no describe can do stuff', + 'ci-visibility/test-early-flake-detection/weird-test-names.js.describe trailing space ' + ] + ) + + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + // no new tests + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test-early-flake-detection/weird-test-names.js' + ]) + }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('does not run EFD if the known tests request fails', (done) => { + receiver.setKnownTestsResponseCode(500) + + const NUM_RETRIES_EFD = 5 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 2) + const newTests = tests.filter( + test => test.meta[TEST_IS_NEW] === 'true' + ) + assert.equal(newTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => done()).catch(done) + }) + }) + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + // Tests from ci-visibility/test/occasionally-failing-test will be considered new + receiver.setKnownTests({}) + + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 100 + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + tests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Out of NUM_RETRIES_EFD + 1 total runs, half will be passing and half will be failing, + // based on the global counter in the test file + const passingTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const failingTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(passingTests.length, (NUM_RETRIES_EFD + 1) / 2) + assert.equal(failingTests.length, (NUM_RETRIES_EFD + 1) / 2) + // Test name does not change + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + }) + }) + + childProcess = exec( + 'node ./node_modules/mocha/bin/mocha ci-visibility/test-early-flake-detection/occasionally-failing-test*', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: '**/ci-visibility/test-early-flake-detection/occasionally-failing-test*' + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + + childProcess.on('exit', (exitCode) => { + assert.include(testOutput, '2 passing') + assert.include(testOutput, '2 failing') + assert.equal(exitCode, 0) + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) +}) diff --git a/integration-tests/package-guardrails.spec.js b/integration-tests/package-guardrails.spec.js new file mode 100644 index 00000000000..8fff27db3b2 --- /dev/null +++ b/integration-tests/package-guardrails.spec.js @@ -0,0 +1,95 @@ +const { + runAndCheckWithTelemetry: testFile, + useEnv, + useSandbox, + sandboxCwd +} = require('./helpers') +const path = require('path') +const fs = require('fs') +const assert = require('assert') + +const NODE_OPTIONS = '--require dd-trace/init.js' +const DD_TRACE_DEBUG = 'true' +const DD_INJECTION_ENABLED = 'tracing' +const DD_LOG_LEVEL = 'error' + +describe('package guardrails', () => { + useEnv({ NODE_OPTIONS }) + const runTest = (...args) => + testFile('package-guardrails/index.js', ...args) + + context('when package is out of range', () => { + useSandbox(['bluebird@1.0.0']) + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + it('should not instrument the package, and send telemetry', () => + runTest('false\n', + 'complete', 'injection_forced:false', + 'abort.integration', 'integration:bluebird,integration_version:1.0.0' + )) + }) + context('with logging disabled', () => { + it('should not instrument the package', () => runTest('false\n')) + }) + context('with logging enabled', () => { + useEnv({ DD_TRACE_DEBUG }) + it('should not instrument the package', () => + runTest(`Application instrumentation bootstrapping complete +Found incompatible integration version: bluebird@1.0.0 +false +`)) + }) + }) + + context('when package is in range', () => { + context('when bluebird is 2.9.0', () => { + useSandbox(['bluebird@2.9.0']) + it('should instrument the package', () => runTest('true\n')) + }) + context('when bluebird is 3.7.2', () => { + useSandbox(['bluebird@3.7.2']) + it('should instrument the package', () => runTest('true\n')) + }) + }) + + context('when package errors out', () => { + useSandbox(['bluebird']) + before(() => { + const file = path.join(sandboxCwd(), 'node_modules/dd-trace/packages/datadog-instrumentations/src/bluebird.js') + fs.writeFileSync(file, ` +const { addHook } = require('./helpers/instrument') + +addHook({ name: 'bluebird', versions: ['*'] }, Promise => { + throw new ReferenceError('this is a test error') + return Promise +}) + `) + }) + + context('with DD_INJECTION_ENABLED', () => { + useEnv({ DD_INJECTION_ENABLED }) + it('should not instrument the package, and send telemetry', () => + runTest('false\n', + 'complete', 'injection_forced:false', + 'error', 'error_type:ReferenceError,integration:bluebird,integration_version:3.7.2' + )) + }) + + context('with logging disabled', () => { + it('should not instrument the package', () => runTest('false\n')) + }) + + context('with logging enabled', () => { + useEnv({ DD_TRACE_DEBUG, DD_LOG_LEVEL }) + it('should not instrument the package', () => + runTest( + log => { + assert.ok(log.includes(` +Error during ddtrace instrumentation of application, aborting. +ReferenceError: this is a test error + at `)) + assert.ok(log.includes('\nfalse\n')) + })) + }) + }) +}) diff --git a/integration-tests/package-guardrails/index.js b/integration-tests/package-guardrails/index.js new file mode 100644 index 00000000000..efaf37abcd8 --- /dev/null +++ b/integration-tests/package-guardrails/index.js @@ -0,0 +1,8 @@ +'use strict' + +const P = require('bluebird') + +const isWrapped = P.prototype._then.toString().includes('AsyncResource') + +// eslint-disable-next-line no-console +console.log(isWrapped) diff --git a/integration-tests/telemetry-forwarder.sh b/integration-tests/telemetry-forwarder.sh new file mode 100755 index 00000000000..5fe156993be --- /dev/null +++ b/integration-tests/telemetry-forwarder.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Implemented this in bash instead of Node.js for two reasons: +# 1. It's trivial in bash. +# 2. We're using NODE_OPTIONS in tests to init the tracer, and we don't want that for this script. + +echo "$1 $(cat -)" >> $FORWARDER_OUT diff --git a/integration-tests/vitest.config.mjs b/integration-tests/vitest.config.mjs new file mode 100644 index 00000000000..f04d63785fd --- /dev/null +++ b/integration-tests/vitest.config.mjs @@ -0,0 +1,9 @@ +import { defineConfig } from 'vite' + +export default defineConfig({ + test: { + include: [ + 'ci-visibility/vitest-tests/test-visibility*' + ] + } +}) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js new file mode 100644 index 00000000000..4a3151c2b72 --- /dev/null +++ b/integration-tests/vitest/vitest.spec.js @@ -0,0 +1,137 @@ +'use strict' + +const { exec } = require('child_process') + +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + TEST_STATUS, + TEST_TYPE +} = require('../../packages/dd-trace/src/plugins/util/test') + +// tested with 1.6.0 +const versions = ['latest'] + +versions.forEach((version) => { + describe(`vitest@${version}`, () => { + let sandbox, cwd, receiver, childProcess + + before(async function () { + sandbox = await createSandbox([`vitest@${version}`], true) + cwd = sandbox.folder + }) + + after(async () => { + await sandbox.remove() + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + }) + + afterEach(async () => { + childProcess.kill() + await receiver.stop() + }) + + it('can run and report tests', (done) => { + receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSessionEvent = events.find(event => event.type === 'test_session_end') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + const testEvents = events.filter(event => event.type === 'test') + + assert.include(testSessionEvent.content.resource, 'test_session.vitest run') + assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail') + assert.include(testModuleEvent.content.resource, 'test_module.vitest run') + assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail') + assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test') + assert.equal(testModuleEvent.content.meta[TEST_TYPE], 'test') + + const passedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-passed-suite.mjs' + ) + assert.equal(passedSuite.content.meta[TEST_STATUS], 'pass') + + const failedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + ) + assert.equal(failedSuite.content.meta[TEST_STATUS], 'fail') + + const failedSuiteHooks = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs' + ) + assert.equal(failedSuiteHooks.content.meta[TEST_STATUS], 'fail') + + assert.includeMembers(testEvents.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' + ] + ) + + const failedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'fail') + + assert.includeMembers( + failedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' + ] + ) + + const skippedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'skip') + + assert.includeMembers( + skippedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip' + ] + ) + // TODO: check error messages + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + // maybe only in node@20 + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' // ESM requires more flags + }, + stdio: 'pipe' + } + ) + }) + }) +}) diff --git a/package.json b/package.json index 481878c38db..75d79dde2f5 100644 --- a/package.json +++ b/package.json @@ -35,8 +35,11 @@ "test:integration": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/*.spec.js\"", "test:integration:cucumber": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/cucumber/*.spec.js\"", "test:integration:cypress": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/cypress/*.spec.js\"", + "test:integration:jest": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/jest/*.spec.js\"", + "test:integration:mocha": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/mocha/*.spec.js\"", "test:integration:playwright": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/playwright/*.spec.js\"", "test:integration:selenium": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/selenium/*.spec.js\"", + "test:integration:vitest": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/vitest/*.spec.js\"", "test:integration:profiler": "mocha --colors --timeout 180000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/profiler/*.spec.js\"", "test:integration:serverless": "mocha --colors --timeout 30000 -r \"packages/dd-trace/test/setup/core.js\" \"integration-tests/serverless/*.spec.js\"", "test:integration:plugins": "mocha --colors --exit -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", @@ -88,7 +91,7 @@ "koalas": "^1.0.2", "limiter": "1.1.5", "lodash.sortby": "^4.7.0", - "lru-cache": "^10.2.2", + "lru-cache": "^7.14.0", "module-details-from-path": "^1.0.3", "msgpack-lite": "^0.1.26", "opentracing": ">=0.12.1", @@ -136,6 +139,7 @@ "sinon": "^15.2.0", "sinon-chai": "^3.7.0", "tap": "^16.3.7", - "tape": "^5.6.5" + "tape": "^5.6.5", + "tiktoken": "^1.0.15" } } diff --git a/packages/datadog-core/src/storage/async_hooks.js b/packages/datadog-core/src/storage/async_hooks.js deleted file mode 100644 index d8e71e1df2d..00000000000 --- a/packages/datadog-core/src/storage/async_hooks.js +++ /dev/null @@ -1,49 +0,0 @@ -'use strict' - -const { executionAsyncId } = require('async_hooks') -const AsyncResourceStorage = require('./async_resource') - -class AsyncHooksStorage extends AsyncResourceStorage { - constructor () { - super() - - this._resources = new Map() - } - - disable () { - super.disable() - - this._resources.clear() - } - - _createHook () { - return { - ...super._createHook(), - destroy: this._destroy.bind(this) - } - } - - _init (asyncId, type, triggerAsyncId, resource) { - super._init.apply(this, arguments) - - this._resources.set(asyncId, resource) - } - - _destroy (asyncId) { - this._resources.delete(asyncId) - } - - _executionAsyncResource () { - const asyncId = executionAsyncId() - - let resource = this._resources.get(asyncId) - - if (!resource) { - this._resources.set(asyncId, resource = {}) - } - - return resource - } -} - -module.exports = AsyncHooksStorage diff --git a/packages/datadog-core/src/storage/index.js b/packages/datadog-core/src/storage/index.js index 0d48defbc3c..e522e61ced2 100644 --- a/packages/datadog-core/src/storage/index.js +++ b/packages/datadog-core/src/storage/index.js @@ -2,13 +2,4 @@ // TODO: default to AsyncLocalStorage when it supports triggerAsyncResource -const semver = require('semver') - -// https://github.com/nodejs/node/pull/33801 -const hasJavaScriptAsyncHooks = semver.satisfies(process.versions.node, '>=14.5') - -if (hasJavaScriptAsyncHooks) { - module.exports = require('./async_resource') -} else { - module.exports = require('./async_hooks') -} +module.exports = require('./async_resource') diff --git a/packages/datadog-core/test/storage/async_hooks.spec.js b/packages/datadog-core/test/storage/async_hooks.spec.js deleted file mode 100644 index dc990ab94f4..00000000000 --- a/packages/datadog-core/test/storage/async_hooks.spec.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict' - -require('../setup') - -const StorageBackend = require('../../src/storage/async_hooks') -const testStorage = require('./test') - -describe('storage/async_hooks', () => { - let storage - - beforeEach(() => { - storage = new StorageBackend() - }) - - afterEach(() => { - storage.disable() - }) - - testStorage(() => storage) -}) diff --git a/packages/datadog-esbuild/index.js b/packages/datadog-esbuild/index.js index 95a0e8ddd16..ce263799023 100644 --- a/packages/datadog-esbuild/index.js +++ b/packages/datadog-esbuild/index.js @@ -7,7 +7,11 @@ const hooks = require('../datadog-instrumentations/src/helpers/hooks.js') const extractPackageAndModulePath = require('../datadog-instrumentations/src/utils/src/extract-package-and-module-path') for (const hook of Object.values(hooks)) { - hook() + if (typeof hook === 'object') { + hook.fn() + } else { + hook() + } } const modulesOfInterest = new Set() diff --git a/packages/datadog-instrumentations/src/helpers/hook.js b/packages/datadog-instrumentations/src/helpers/hook.js index 7bec453187a..0177744ea1c 100644 --- a/packages/datadog-instrumentations/src/helpers/hook.js +++ b/packages/datadog-instrumentations/src/helpers/hook.js @@ -11,8 +11,13 @@ const ritm = require('../../../dd-trace/src/ritm') * @param {string[]} modules list of modules to hook into * @param {Function} onrequire callback to be executed upon encountering module */ -function Hook (modules, onrequire) { - if (!(this instanceof Hook)) return new Hook(modules, onrequire) +function Hook (modules, hookOptions, onrequire) { + if (!(this instanceof Hook)) return new Hook(modules, hookOptions, onrequire) + + if (typeof hookOptions === 'function') { + onrequire = hookOptions + hookOptions = {} + } this._patched = Object.create(null) @@ -28,7 +33,7 @@ function Hook (modules, onrequire) { } this._ritmHook = ritm(modules, {}, safeHook) - this._iitmHook = iitm(modules, {}, (moduleExports, moduleName, moduleBaseDir) => { + this._iitmHook = iitm(modules, hookOptions, (moduleExports, moduleName, moduleBaseDir) => { // TODO: Move this logic to import-in-the-middle and only do it for CommonJS // modules and not ESM. In the meantime, all the modules we instrument are // CommonJS modules for which the default export is always moved to diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index 0723ceabd84..94f3318fb62 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -23,6 +23,7 @@ module.exports = { '@opentelemetry/sdk-trace-node': () => require('../otel-sdk-trace'), '@redis/client': () => require('../redis'), '@smithy/smithy-client': () => require('../aws-sdk'), + '@vitest/runner': { esmFirst: true, fn: () => require('../vitest') }, aerospike: () => require('../aerospike'), amqp10: () => require('../amqp10'), amqplib: () => require('../amqplib'), @@ -110,6 +111,7 @@ module.exports = { sharedb: () => require('../sharedb'), tedious: () => require('../tedious'), undici: () => require('../undici'), + vitest: { esmFirst: true, fn: () => require('../vitest') }, when: () => require('../when'), winston: () => require('../winston') } diff --git a/packages/datadog-instrumentations/src/helpers/instrument.js b/packages/datadog-instrumentations/src/helpers/instrument.js index 0889f1e5402..20657335044 100644 --- a/packages/datadog-instrumentations/src/helpers/instrument.js +++ b/packages/datadog-instrumentations/src/helpers/instrument.js @@ -17,10 +17,11 @@ exports.channel = function (name) { /** * @param {string} args.name module name * @param {string[]} args.versions array of semver range strings - * @param {string} args.file path to file within package to instrument? + * @param {string} args.file path to file within package to instrument + * @param {string} args.filePattern pattern to match files within package to instrument * @param Function hook */ -exports.addHook = function addHook ({ name, versions, file }, hook) { +exports.addHook = function addHook ({ name, versions, file, filePattern }, hook) { if (typeof name === 'string') { name = [name] } @@ -29,7 +30,7 @@ exports.addHook = function addHook ({ name, versions, file }, hook) { if (!instrumentations[val]) { instrumentations[val] = [] } - instrumentations[val].push({ name: val, versions, file, hook }) + instrumentations[val].push({ name: val, versions, file, filePattern, hook }) } } diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index eba90d6a980..e45a0c0cd14 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -7,6 +7,7 @@ const Hook = require('./hook') const requirePackageJson = require('../../../dd-trace/src/require-package-json') const log = require('../../../dd-trace/src/log') const checkRequireCache = require('../check_require_cache') +const telemetry = require('../../../dd-trace/src/telemetry/init-telemetry') const { DD_TRACE_DISABLED_INSTRUMENTATIONS = '', @@ -35,22 +36,38 @@ if (DD_TRACE_DEBUG && DD_TRACE_DEBUG.toLowerCase() !== 'false') { setImmediate(checkRequireCache.checkForPotentialConflicts) } +const seenCombo = new Set() + // TODO: make this more efficient for (const packageName of names) { if (disabledInstrumentations.has(packageName)) continue - Hook([packageName], (moduleExports, moduleName, moduleBaseDir, moduleVersion) => { + const hookOptions = {} + + let hook = hooks[packageName] + + if (typeof hook === 'object') { + hookOptions.internals = hook.esmFirst + hook = hook.fn + } + + Hook([packageName], hookOptions, (moduleExports, moduleName, moduleBaseDir, moduleVersion) => { moduleName = moduleName.replace(pathSepExpr, '/') // This executes the integration file thus adding its entries to `instrumentations` - hooks[packageName]() + hook() if (!instrumentations[packageName]) { return moduleExports } - for (const { name, file, versions, hook } of instrumentations[packageName]) { + const namesAndSuccesses = {} + for (const { name, file, versions, hook, filePattern } of instrumentations[packageName]) { + let fullFilePattern = filePattern const fullFilename = filename(name, file) + if (fullFilePattern) { + fullFilePattern = filename(name, fullFilePattern) + } // Create a WeakMap associated with the hook function so that patches on the same moduleExport only happens once // for example by instrumenting both dns and node:dns double the spans would be created @@ -58,13 +75,29 @@ for (const packageName of names) { if (!hook[HOOK_SYMBOL]) { hook[HOOK_SYMBOL] = new WeakMap() } + let matchesFile = false + + matchesFile = moduleName === fullFilename - if (moduleName === fullFilename) { + if (fullFilePattern) { + // Some libraries include a hash in their filenames when installed, + // so our instrumentation has to include a '.*' to match them for more than a single version. + matchesFile = matchesFile || new RegExp(fullFilePattern).test(moduleName) + } + + if (matchesFile) { const version = moduleVersion || getVersion(moduleBaseDir) + if (!Object.hasOwnProperty(namesAndSuccesses, name)) { + namesAndSuccesses[name] = { + success: false, + version + } + } if (matchVersion(version, versions)) { // Check if the hook already has a set moduleExport if (hook[HOOK_SYMBOL].has(moduleExports)) { + namesAndSuccesses[name].success = true return moduleExports } @@ -76,11 +109,29 @@ for (const packageName of names) { // Set the moduleExports in the hooks weakmap hook[HOOK_SYMBOL].set(moduleExports, name) } catch (e) { - log.error(e) + log.info('Error during ddtrace instrumentation of application, aborting.') + log.info(e) + telemetry('error', [ + `error_type:${e.constructor.name}`, + `integration:${name}`, + `integration_version:${version}` + ]) } + namesAndSuccesses[name].success = true } } } + for (const name of Object.keys(namesAndSuccesses)) { + const { success, version } = namesAndSuccesses[name] + if (!success && !seenCombo.has(`${name}@${version}`)) { + telemetry('abort.integration', [ + `integration:${name}`, + `integration_version:${version}` + ]) + log.info(`Found incompatible integration version: ${name}@${version}`) + seenCombo.add(`${name}@${version}`) + } + } return moduleExports }) diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js new file mode 100644 index 00000000000..42f32b1ac42 --- /dev/null +++ b/packages/datadog-instrumentations/src/vitest.js @@ -0,0 +1,282 @@ +const { addHook, channel, AsyncResource } = require('./helpers/instrument') +const shimmer = require('../../datadog-shimmer') + +// test hooks +const testStartCh = channel('ci:vitest:test:start') +const testFinishTimeCh = channel('ci:vitest:test:finish-time') +const testPassCh = channel('ci:vitest:test:pass') +const testErrorCh = channel('ci:vitest:test:error') +const testSkipCh = channel('ci:vitest:test:skip') + +// test suite hooks +const testSuiteStartCh = channel('ci:vitest:test-suite:start') +const testSuiteFinishCh = channel('ci:vitest:test-suite:finish') +const testSuiteErrorCh = channel('ci:vitest:test-suite:error') + +// test session hooks +const testSessionStartCh = channel('ci:vitest:session:start') +const testSessionFinishCh = channel('ci:vitest:session:finish') + +const taskToAsync = new WeakMap() + +const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') + +function isReporterPackage (vitestPackage) { + return vitestPackage.B?.name === 'BaseSequencer' +} + +function getSessionStatus (state) { + if (state.getCountOfFailedTests() > 0) { + return 'fail' + } + if (state.pathsSet.size === 0) { + return 'skip' + } + return 'pass' +} + +// eslint-disable-next-line +// From https://github.com/vitest-dev/vitest/blob/51c04e2f44d91322b334f8ccbcdb368facc3f8ec/packages/runner/src/run.ts#L243-L250 +function getVitestTestStatus (test, retryCount) { + if (test.result.state !== 'fail') { + if (!test.repeats) { + return 'pass' + } else if (test.repeats && (test.retry ?? 0) === retryCount) { + return 'pass' + } + } + return 'fail' +} + +function getTypeTasks (fileTasks, type = 'test') { + const typeTasks = [] + + function getTasks (tasks) { + for (const task of tasks) { + if (task.type === type) { + typeTasks.push(task) + } else if (task.tasks) { + getTasks(task.tasks) + } + } + } + + getTasks(fileTasks) + + return typeTasks +} + +function getTestName (task) { + let testName = task.name + let currentTask = task.suite + + while (currentTask) { + if (currentTask.name) { + testName = `${currentTask.name} ${testName}` + } + currentTask = currentTask.suite + } + + return testName +} + +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + file: 'dist/runners.js' +}, (vitestPackage) => { + const { VitestTestRunner } = vitestPackage + // test start (only tests that are not marked as skip or todo) + shimmer.wrap(VitestTestRunner.prototype, 'onBeforeTryTask', onBeforeTryTask => async function (task) { + if (!testStartCh.hasSubscribers) { + return onBeforeTryTask.apply(this, arguments) + } + const asyncResource = new AsyncResource('bound-anonymous-fn') + taskToAsync.set(task, asyncResource) + + asyncResource.runInAsyncScope(() => { + testStartCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + }) + return onBeforeTryTask.apply(this, arguments) + }) + + // test finish (only passed tests) + shimmer.wrap(VitestTestRunner.prototype, 'onAfterTryTask', onAfterTryTask => + async function (task, { retry: retryCount }) { + if (!testFinishTimeCh.hasSubscribers) { + return onAfterTryTask.apply(this, arguments) + } + const result = await onAfterTryTask.apply(this, arguments) + + const status = getVitestTestStatus(task, retryCount) + const asyncResource = taskToAsync.get(task) + + if (asyncResource) { + // We don't finish here because the test might fail in a later hook + asyncResource.runInAsyncScope(() => { + testFinishTimeCh.publish({ status, task }) + }) + } + + return result + }) + + return vitestPackage +}) + +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + filePattern: 'dist/vendor/index.*' +}, (vitestPackage) => { + // there are multiple index* files so we have to check the exported values + if (!isReporterPackage(vitestPackage)) { + return vitestPackage + } + shimmer.wrap(vitestPackage.B.prototype, 'sort', sort => async function () { + if (!testSessionFinishCh.hasSubscribers) { + return sort.apply(this, arguments) + } + shimmer.wrap(this.ctx, 'exit', exit => async function () { + let onFinish + + const flushPromise = new Promise(resolve => { + onFinish = resolve + }) + const failedSuites = this.state.getFailedFilepaths() + let error + if (failedSuites.length) { + error = new Error(`Test suites failed: ${failedSuites.length}.`) + } + + sessionAsyncResource.runInAsyncScope(() => { + testSessionFinishCh.publish({ + status: getSessionStatus(this.state), + onFinish, + error + }) + }) + + await flushPromise + + return exit.apply(this, arguments) + }) + + return sort.apply(this, arguments) + }) + + return vitestPackage +}) + +// Can't specify file because compiled vitest includes hashes in their files +addHook({ + name: 'vitest', + versions: ['>=1.6.0'], + filePattern: 'dist/vendor/cac.*' +}, (vitestPackage, frameworkVersion) => { + shimmer.wrap(vitestPackage, 'c', oldCreateCli => function () { + if (!testSessionStartCh.hasSubscribers) { + return oldCreateCli.apply(this, arguments) + } + sessionAsyncResource.runInAsyncScope(() => { + const processArgv = process.argv.slice(2).join(' ') + testSessionStartCh.publish({ command: `vitest ${processArgv}`, frameworkVersion }) + }) + return oldCreateCli.apply(this, arguments) + }) + + return vitestPackage +}) + +// test suite start and finish +// only relevant for workers +addHook({ + name: '@vitest/runner', + versions: ['>=1.6.0'], + file: 'dist/index.js' +}, vitestPackage => { + shimmer.wrap(vitestPackage, 'startTests', startTests => async function (testPath) { + let testSuiteError = null + if (!testSuiteStartCh.hasSubscribers) { + return startTests.apply(this, arguments) + } + + const testSuiteAsyncResource = new AsyncResource('bound-anonymous-fn') + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish(testPath[0]) + }) + const startTestsResponse = await startTests.apply(this, arguments) + + let onFinish = null + const onFinishPromise = new Promise(resolve => { + onFinish = resolve + }) + + const testTasks = getTypeTasks(startTestsResponse[0].tasks) + + testTasks.forEach(task => { + const testAsyncResource = taskToAsync.get(task) + const { result } = task + + if (result) { + const { state, duration, errors } = result + if (state === 'skip') { // programmatic skip + testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + } else if (state === 'pass') { + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + testPassCh.publish({ task }) + }) + } + } else if (state === 'fail') { + // If it's failing, we have no accurate finish time, so we have to use `duration` + let testError + + if (errors?.length) { + testError = errors[0] + } + + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + testErrorCh.publish({ duration, error: testError }) + }) + } + if (errors?.length) { + testSuiteError = testError // we store the error to bubble it up to the suite + } + } + } else { // test.skip or test.todo + testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath }) + } + }) + + const testSuiteResult = startTestsResponse[0].result + + if (testSuiteResult.errors?.length) { // Errors from root level hooks + testSuiteError = testSuiteResult.errors[0] + } else if (testSuiteResult.state === 'fail') { // Errors from `describe` level hooks + const suiteTasks = getTypeTasks(startTestsResponse[0].tasks, 'suite') + const failedSuites = suiteTasks.filter(task => task.result?.state === 'fail') + if (failedSuites.length && failedSuites[0].result?.errors?.length) { + testSuiteError = failedSuites[0].result.errors[0] + } + } + + if (testSuiteError) { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish({ error: testSuiteError }) + }) + } + + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish({ status: testSuiteResult.state, onFinish }) + }) + + // TODO: fix too frequent flushes + await onFinishPromise + + return startTestsResponse + }) + + return vitestPackage +}) diff --git a/packages/datadog-instrumentations/test/body-parser.spec.js b/packages/datadog-instrumentations/test/body-parser.spec.js index d502bc00ea6..23c7388f2dd 100644 --- a/packages/datadog-instrumentations/test/body-parser.spec.js +++ b/packages/datadog-instrumentations/test/body-parser.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const dc = require('dc-polyfill') const axios = require('axios') const agent = require('../../dd-trace/test/plugins/agent') @@ -22,11 +21,9 @@ withVersions('body-parser', 'body-parser', version => { middlewareProcessBodyStub() res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) beforeEach(async () => { diff --git a/packages/datadog-instrumentations/test/cookie-parser.spec.js b/packages/datadog-instrumentations/test/cookie-parser.spec.js index 4137ddbef63..14afe44ba90 100644 --- a/packages/datadog-instrumentations/test/cookie-parser.spec.js +++ b/packages/datadog-instrumentations/test/cookie-parser.spec.js @@ -1,7 +1,6 @@ 'use strict' const { assert } = require('chai') -const getPort = require('get-port') const dc = require('dc-polyfill') const axios = require('axios') const agent = require('../../dd-trace/test/plugins/agent') @@ -23,11 +22,9 @@ withVersions('cookie-parser', 'cookie-parser', version => { middlewareProcessCookieStub() res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) beforeEach(async () => { diff --git a/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js b/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js index 7464f83152a..3fcf981e528 100644 --- a/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js +++ b/packages/datadog-instrumentations/test/express-mongo-sanitize.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const { channel } = require('dc-polyfill') const axios = require('axios') describe('express-mongo-sanitize', () => { @@ -25,11 +24,9 @@ describe('express-mongo-sanitize', () => { res.end() }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) diff --git a/packages/datadog-instrumentations/test/express.spec.js b/packages/datadog-instrumentations/test/express.spec.js index 88f75164be6..ff9f577bc8d 100644 --- a/packages/datadog-instrumentations/test/express.spec.js +++ b/packages/datadog-instrumentations/test/express.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const axios = require('axios') const dc = require('dc-polyfill') @@ -20,11 +19,9 @@ withVersions('express', 'express', version => { requestBody() res.end('DONE') }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) beforeEach(async () => { diff --git a/packages/datadog-instrumentations/test/passport-http.spec.js b/packages/datadog-instrumentations/test/passport-http.spec.js index 68b06abbe5c..10b2cd292a0 100644 --- a/packages/datadog-instrumentations/test/passport-http.spec.js +++ b/packages/datadog-instrumentations/test/passport-http.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const axios = require('axios') const dc = require('dc-polyfill') @@ -70,11 +69,9 @@ withVersions('passport-http', 'passport-http', version => { subscriberStub(arguments[0]) }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) beforeEach(() => { diff --git a/packages/datadog-instrumentations/test/passport-local.spec.js b/packages/datadog-instrumentations/test/passport-local.spec.js index f31c7a83230..92ffe9bb1d8 100644 --- a/packages/datadog-instrumentations/test/passport-local.spec.js +++ b/packages/datadog-instrumentations/test/passport-local.spec.js @@ -1,7 +1,6 @@ 'use strict' const agent = require('../../dd-trace/test/plugins/agent') -const getPort = require('get-port') const axios = require('axios') const dc = require('dc-polyfill') @@ -71,11 +70,9 @@ withVersions('passport-local', 'passport-local', version => { subscriberStub(arguments[0]) }) - getPort().then(newPort => { - port = newPort - server = app.listen(port, () => { - done() - }) + server = app.listen(0, () => { + port = server.address().port + done() }) }) beforeEach(() => { diff --git a/packages/datadog-plugin-apollo/test/index.spec.js b/packages/datadog-plugin-apollo/test/index.spec.js index 6718098ef28..5bf25e4e428 100644 --- a/packages/datadog-plugin-apollo/test/index.spec.js +++ b/packages/datadog-plugin-apollo/test/index.spec.js @@ -5,7 +5,6 @@ const agent = require('../../dd-trace/test/plugins/agent.js') const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants.js') const { expectedSchema, rawExpectedSchema } = require('./naming.js') const axios = require('axios') -const getPort = require('get-port') const accounts = require('./fixtures.js') @@ -86,13 +85,15 @@ describe('Plugin', () => { gateway: setupGateway(), subscriptions: false // Disable subscriptions (not supported with Apollo Gateway) }) - getPort().then(newPort => { - port = newPort - startStandaloneServer(server, { - listen: { port } - }).then(() => {}) + + return startStandaloneServer(server, { + listen: { port: 0 } + }).then(({ url }) => { + port = new URL(url).port }) + }) + before(() => { return agent.load('apollo') }) diff --git a/packages/datadog-plugin-aws-sdk/src/base.js b/packages/datadog-plugin-aws-sdk/src/base.js index 7254a0a2fa9..21c48831f92 100644 --- a/packages/datadog-plugin-aws-sdk/src/base.js +++ b/packages/datadog-plugin-aws-sdk/src/base.js @@ -165,6 +165,7 @@ function normalizeConfig (config, serviceIdentifier) { return Object.assign({}, config, specificConfig, { splitByAwsService: config.splitByAwsService !== false, + batchPropagationEnabled: config.batchPropagationEnabled !== false, hooks }) } diff --git a/packages/datadog-plugin-aws-sdk/src/services/kinesis.js b/packages/datadog-plugin-aws-sdk/src/services/kinesis.js index 9e8eb2dcd44..62c9c9b4a6f 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/kinesis.js +++ b/packages/datadog-plugin-aws-sdk/src/services/kinesis.js @@ -152,7 +152,12 @@ class Kinesis extends BaseAwsSdkPlugin { case 'putRecords': stream = params.StreamArn ? params.StreamArn : (params.StreamName ? params.StreamName : '') for (let i = 0; i < params.Records.length; i++) { - this.injectToMessage(span, params.Records[i], stream, i === 0) + this.injectToMessage( + span, + params.Records[i], + stream, + i === 0 || (this.config.kinesis && this.config.kinesis.batchPropagationEnabled) + ) } } } diff --git a/packages/datadog-plugin-aws-sdk/src/services/sns.js b/packages/datadog-plugin-aws-sdk/src/services/sns.js index ee5191ddabc..a88aa8bda46 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/sns.js +++ b/packages/datadog-plugin-aws-sdk/src/services/sns.js @@ -59,7 +59,12 @@ class Sns extends BaseAwsSdkPlugin { break case 'publishBatch': for (let i = 0; i < params.PublishBatchRequestEntries.length; i++) { - this.injectToMessage(span, params.PublishBatchRequestEntries[i], params.TopicArn, i === 0) + this.injectToMessage( + span, + params.PublishBatchRequestEntries[i], + params.TopicArn, + i === 0 || (this.config.sns && this.config.sns.batchPropagationEnabled) + ) } break } diff --git a/packages/datadog-plugin-aws-sdk/src/services/sqs.js b/packages/datadog-plugin-aws-sdk/src/services/sqs.js index 62ede0ae6e4..cce27c18719 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/sqs.js +++ b/packages/datadog-plugin-aws-sdk/src/services/sqs.js @@ -218,7 +218,12 @@ class Sqs extends BaseAwsSdkPlugin { break case 'sendMessageBatch': for (let i = 0; i < params.Entries.length; i++) { - this.injectToMessage(span, params.Entries[i], params.QueueUrl, i === 0) + this.injectToMessage( + span, + params.Entries[i], + params.QueueUrl, + i === 0 || (this.config.sqs && this.config.sqs.batchPropagationEnabled) + ) } break case 'receiveMessage': diff --git a/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js b/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js index 89518c45cdc..cedeb14f000 100644 --- a/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/kinesis.spec.js @@ -52,7 +52,7 @@ describe('Kinesis', function () { describe('no configuration', () => { before(() => { - return agent.load('aws-sdk', { kinesis: { dsmEnabled: false } }, { dsmEnabled: true }) + return agent.load('aws-sdk', { kinesis: { dsmEnabled: false, batchPropagationEnabled: true } }, { dsmEnabled: true }) }) before(done => { @@ -91,6 +91,24 @@ describe('Kinesis', function () { }) }) + it('injects trace context to each message during Kinesis putRecord and batchPropagationEnabled', done => { + helpers.putTestRecords(kinesis, streamName, (err, data) => { + if (err) return done(err) + + helpers.getTestRecord(kinesis, streamName, data.Records[0], (err, data) => { + if (err) return done(err) + + for (const record in data.Records) { + const recordData = JSON.parse(Buffer.from(data.Records[record].Data).toString()) + expect(recordData).to.have.property('_datadog') + expect(recordData._datadog).to.have.property('x-datadog-trace-id') + } + + done() + }) + }) + }) + it('handles already b64 encoded data', done => { helpers.putTestRecord(kinesis, streamName, helpers.dataBuffer.toString('base64'), (err, data) => { if (err) return done(err) diff --git a/packages/datadog-plugin-aws-sdk/test/sns.spec.js b/packages/datadog-plugin-aws-sdk/test/sns.spec.js index 27f194abc7e..293833a6009 100644 --- a/packages/datadog-plugin-aws-sdk/test/sns.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/sns.spec.js @@ -7,7 +7,7 @@ const agent = require('../../dd-trace/test/plugins/agent') const { setup } = require('./spec_helpers') const { rawExpectedSchema } = require('./sns-naming') -describe('Sns', () => { +describe('Sns', function () { setup() withVersions('aws-sdk', ['aws-sdk', '@aws-sdk/smithy-client'], (version, moduleName) => { @@ -25,7 +25,8 @@ describe('Sns', () => { const snsClientName = moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-sns' : 'aws-sdk' const sqsClientName = moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-sqs' : 'aws-sdk' - const assertPropagation = done => { + let childSpansFound = 0 + const assertPropagation = (done, childSpans = 1) => { agent.use(traces => { const span = traces[0][0] @@ -37,7 +38,10 @@ describe('Sns', () => { expect(parentId).to.not.equal('0') expect(parentId).to.equal(spanId) - }).then(done, done) + childSpansFound += 1 + expect(childSpansFound).to.equal(childSpans) + childSpansFound = 0 + }, { timeoutMs: 10000 }).then(done, done) } function createResources (queueName, topicName, cb) { @@ -85,13 +89,13 @@ describe('Sns', () => { parentId = '0' spanId = '0' - return agent.load('aws-sdk', { sns: { dsmEnabled: false } }, { dsmEnabled: true }) + return agent.load('aws-sdk', { sns: { dsmEnabled: false, batchPropagationEnabled: true } }, { dsmEnabled: true }) }) before(done => { process.env.DD_DATA_STREAMS_ENABLED = 'true' tracer = require('../../dd-trace') - tracer.use('aws-sdk', { sns: { dsmEnabled: false } }) + tracer.use('aws-sdk', { sns: { dsmEnabled: false, batchPropagationEnabled: true } }) createResources('TestQueue', 'TestTopic', done) }) @@ -170,6 +174,34 @@ describe('Sns', () => { }, e => e && done(e)) }) }) + + it('injects trace context to each message SNS publishBatch with batch propagation enabled', done => { + assertPropagation(done, 3) + + sns.subscribe(subParams, (err, data) => { + if (err) return done(err) + + sqs.receiveMessage(receiveParams, (err, data) => { + if (err) done(err) + + for (const message in data.Messages) { + const recordData = JSON.parse(data.Messages[message].Body) + expect(recordData.MessageAttributes).to.have.property('_datadog') + + const attributes = JSON.parse(Buffer.from(recordData.MessageAttributes._datadog.Value, 'base64')) + expect(attributes).to.have.property('x-datadog-trace-id') + } + }) + sns.publishBatch({ + TopicArn, + PublishBatchRequestEntries: [ + { Id: '1', Message: 'message 1' }, + { Id: '2', Message: 'message 2' }, + { Id: '3', Message: 'message 3' } + ] + }, e => e && done(e)) + }) + }) } // TODO: Figure out why this fails only in 3.0.0 @@ -261,7 +293,7 @@ describe('Sns', () => { } catch { // pass } - agent.reload('aws-sdk', { kinesis: { dsmEnabled: true } }, { dsmEnabled: true }) + agent.reload('aws-sdk', { sns: { dsmEnabled: true, batchPropagationEnabled: true } }, { dsmEnabled: true }) }) it('injects DSM pathway hash to SNS publish span', done => { diff --git a/packages/datadog-plugin-aws-sdk/test/sqs.spec.js b/packages/datadog-plugin-aws-sdk/test/sqs.spec.js index a32f8a51a86..9c0c3686f9b 100644 --- a/packages/datadog-plugin-aws-sdk/test/sqs.spec.js +++ b/packages/datadog-plugin-aws-sdk/test/sqs.spec.js @@ -38,8 +38,11 @@ describe('Plugin', () => { before(() => { process.env.DD_DATA_STREAMS_ENABLED = 'true' tracer = require('../../dd-trace') + tracer.use('aws-sdk', { sqs: { batchPropagationEnabled: true } }) - return agent.load('aws-sdk', { sqs: { dsmEnabled: false } }, { dsmEnabled: true }) + return agent.load( + 'aws-sdk', { sqs: { dsmEnabled: false, batchPropagationEnabled: true } }, { dsmEnabled: true } + ) }) before(done => { @@ -148,6 +151,74 @@ describe('Plugin', () => { }) }) + it('should propagate the tracing context from the producer to the consumer in batch operations', (done) => { + let parentId + let traceId + + agent.use(traces => { + const span = traces[0][0] + + expect(span.resource.startsWith('sendMessageBatch')).to.equal(true) + expect(span.meta).to.include({ + queuename: 'SQS_QUEUE_NAME' + }) + + parentId = span.span_id.toString() + traceId = span.trace_id.toString() + }) + + let batchChildSpans = 0 + agent.use(traces => { + const span = traces[0][0] + + expect(parentId).to.be.a('string') + expect(span.parent_id.toString()).to.equal(parentId) + expect(span.trace_id.toString()).to.equal(traceId) + batchChildSpans += 1 + expect(batchChildSpans).to.equal(3) + }, { timeoutMs: 2000 }).then(done, done) + + sqs.sendMessageBatch( + { + Entries: [ + { + Id: '1', + MessageBody: 'test batch propagation 1' + }, + { + Id: '2', + MessageBody: 'test batch propagation 2' + }, + { + Id: '3', + MessageBody: 'test batch propagation 3' + } + ], + QueueUrl + }, (err) => { + if (err) return done(err) + + function receiveMessage () { + sqs.receiveMessage({ + QueueUrl, + MaxNumberOfMessages: 1 + }, (err, data) => { + if (err) return done(err) + + for (const message in data.Messages) { + const recordData = data.Messages[message].MessageAttributes + expect(recordData).to.have.property('_datadog') + const traceContext = JSON.parse(recordData._datadog.StringValue) + expect(traceContext).to.have.property('x-datadog-trace-id') + } + }) + } + receiveMessage() + receiveMessage() + receiveMessage() + }) + }) + it('should run the consumer in the context of its span', (done) => { sqs.sendMessage({ MessageBody: 'test body', diff --git a/packages/datadog-plugin-connect/test/index.spec.js b/packages/datadog-plugin-connect/test/index.spec.js index f30b4967d44..62b64bcc8a7 100644 --- a/packages/datadog-plugin-connect/test/index.spec.js +++ b/packages/datadog-plugin-connect/test/index.spec.js @@ -2,7 +2,6 @@ const axios = require('axios') const http = require('http') -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const { AsyncLocalStorage } = require('async_hooks') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') @@ -45,7 +44,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -62,11 +63,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -76,7 +75,9 @@ describe('Plugin', () => { app.use(function named (req, res, next) { next() }) app.use('/app/user', (req, res) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -94,11 +95,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -109,7 +108,9 @@ describe('Plugin', () => { app.use('/foo/bar', (req, res, next) => next()) app.use('/foo', (req, res) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -119,11 +120,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -137,7 +136,9 @@ describe('Plugin', () => { app.use('/parent', childApp) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -148,11 +149,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/parent/child`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/parent/child`) + .catch(done) }) }) @@ -175,12 +174,12 @@ describe('Plugin', () => { done() }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -196,7 +195,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -206,11 +207,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -239,12 +238,12 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -253,7 +252,9 @@ describe('Plugin', () => { app.use((req, res, next) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -263,11 +264,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -293,11 +292,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -308,7 +307,9 @@ describe('Plugin', () => { app.use('/app', (req, res) => res.end()) app.use('/bar', (req, res, next) => next()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -318,10 +319,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios.get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -332,7 +331,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -342,17 +343,15 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { - 'x-datadog-trace-id': '1234', - 'x-datadog-parent-id': '5678', - 'ot-baggage-foo': 'bar' - } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { + 'x-datadog-trace-id': '1234', + 'x-datadog-parent-id': '5678', + 'ot-baggage-foo': 'bar' + } + }) + .catch(done) }) }) @@ -368,7 +367,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -381,13 +382,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -403,7 +402,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -416,13 +417,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -432,7 +431,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -447,13 +448,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -477,12 +476,12 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -497,7 +496,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -516,13 +517,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) @@ -551,7 +550,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -561,11 +562,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -577,7 +576,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -587,13 +588,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -604,7 +603,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -614,13 +615,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { 'User-Agent': 'test' } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { 'User-Agent': 'test' } + }) + .catch(done) }) }) @@ -632,7 +631,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -649,11 +650,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -668,7 +667,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -687,13 +688,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -703,7 +702,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -718,13 +719,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) @@ -750,7 +749,9 @@ describe('Plugin', () => { app.use(function named (req, res, next) { next() }) app.use('/app/user', (req, res) => res.end()) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -762,11 +763,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -791,11 +790,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -810,7 +809,9 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -825,13 +826,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -841,7 +840,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = http.createServer(app).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -856,13 +857,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = http.createServer(app).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) diff --git a/packages/datadog-plugin-cypress/test/index.spec.js b/packages/datadog-plugin-cypress/test/index.spec.js index 6be933cf88b..67ca6387ac4 100644 --- a/packages/datadog-plugin-cypress/test/index.spec.js +++ b/packages/datadog-plugin-cypress/test/index.spec.js @@ -1,5 +1,4 @@ 'use strict' -const getPort = require('get-port') const { expect } = require('chai') const semver = require('semver') @@ -31,15 +30,16 @@ describe('Plugin', function () { let agentListenPort this.retries(2) withVersions('cypress', 'cypress', (version, moduleName) => { - beforeEach(function () { + beforeEach(() => { + return agent.load() + }) + beforeEach(function (done) { this.timeout(10000) - return agent.load().then(() => { - agentListenPort = agent.server.address().port - cypressExecutable = require(`../../../versions/cypress@${version}`).get() - return getPort().then(port => { - appPort = port - appServer.listen(appPort) - }) + agentListenPort = agent.server.address().port + cypressExecutable = require(`../../../versions/cypress@${version}`).get() + appServer.listen(0, () => { + appPort = appServer.address().port + done() }) }) afterEach(() => agent.close({ ritmReset: false })) diff --git a/packages/datadog-plugin-express/test/index.spec.js b/packages/datadog-plugin-express/test/index.spec.js index c96722e1246..55a608f4adf 100644 --- a/packages/datadog-plugin-express/test/index.spec.js +++ b/packages/datadog-plugin-express/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') const agent = require('../../dd-trace/test/plugins/agent') const plugin = require('../src') @@ -45,7 +44,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port const timer = setTimeout(done, 100) agent.use(() => { @@ -53,11 +53,9 @@ describe('Plugin', () => { done(new Error('Agent received an unexpected trace.')) }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -70,13 +68,13 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .then(() => done()) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .then(() => done()) + .catch(done) }) }) }) @@ -101,7 +99,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -119,11 +119,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -137,7 +135,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -154,11 +154,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -174,7 +172,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -191,11 +191,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -210,7 +208,9 @@ describe('Plugin', () => { app.use(function named (req, res, next) { next() }) app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -246,11 +246,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -272,7 +270,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -287,11 +287,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/1`) + .catch(done) }) }) @@ -317,7 +315,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -331,11 +331,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/1`) + .catch(done) }) }) @@ -349,7 +347,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -359,11 +359,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -377,7 +375,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -387,11 +387,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -411,7 +409,9 @@ describe('Plugin', () => { app.use('/foo/bar', (req, res, next) => next()) app.use('/foo', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -421,11 +421,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -439,7 +437,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -449,11 +449,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -468,7 +466,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -478,11 +478,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -496,7 +494,9 @@ describe('Plugin', () => { app.use('/parent', childApp) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -508,11 +508,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/parent/child`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/parent/child`) + .catch(done) }) }) @@ -535,12 +533,12 @@ describe('Plugin', () => { done() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -564,7 +562,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -574,11 +574,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -596,7 +594,9 @@ describe('Plugin', () => { app.use('/app', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -606,11 +606,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -625,7 +623,9 @@ describe('Plugin', () => { res.status(200).send(error.message) }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -635,11 +635,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -660,7 +658,9 @@ describe('Plugin', () => { app.use('/v1', routerA) app.use('/v1', routerB) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -670,11 +670,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/v1/a`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/v1/a`) + .catch(() => {}) }) }) @@ -690,7 +688,9 @@ describe('Plugin', () => { res.status(200).send(req.body) }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -700,11 +700,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -726,7 +724,9 @@ describe('Plugin', () => { res.status(200).send('') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -736,11 +736,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -762,7 +760,9 @@ describe('Plugin', () => { res.status(200).send('') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -772,11 +772,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -800,7 +798,9 @@ describe('Plugin', () => { res.status(200).send('') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -810,11 +810,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/foo/bar`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/foo/bar`) + .catch(done) }) }) @@ -831,7 +829,9 @@ describe('Plugin', () => { app.use('/v1', router) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -841,11 +841,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/v1/a`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/v1/a`) + .catch(() => {}) }) }) @@ -874,12 +872,12 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -888,7 +886,9 @@ describe('Plugin', () => { app.use((req, res, next) => res.status(200).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -898,11 +898,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -928,11 +926,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -958,11 +956,11 @@ describe('Plugin', () => { } ) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -980,7 +978,9 @@ describe('Plugin', () => { app.use('/app', router) app.use('/bar', (req, res, next) => next()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -990,10 +990,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios.get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -1004,7 +1002,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1014,17 +1014,15 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { - 'x-datadog-trace-id': '1234', - 'x-datadog-parent-id': '5678', - 'ot-baggage-foo': 'bar' - } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { + 'x-datadog-trace-id': '1234', + 'x-datadog-parent-id': '5678', + 'ot-baggage-foo': 'bar' + } + }) + .catch(done) }) }) @@ -1039,7 +1037,9 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1051,13 +1051,11 @@ describe('Plugin', () => { done() }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1073,7 +1071,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1085,13 +1085,11 @@ describe('Plugin', () => { done() }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -1101,7 +1099,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1116,13 +1116,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1134,7 +1132,9 @@ describe('Plugin', () => { // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => res.status(500).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1153,13 +1153,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1171,7 +1169,9 @@ describe('Plugin', () => { // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => res.status(500).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1190,13 +1190,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1207,7 +1205,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1218,11 +1218,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1261,12 +1259,12 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1281,7 +1279,9 @@ describe('Plugin', () => { res.status(200).send('hi') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1294,13 +1294,11 @@ describe('Plugin', () => { done() }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/does-not-exist`, { - validateStatus: status => status === 404 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/does-not-exist`, { + validateStatus: status => status === 404 + }) + .catch(done) }) }) @@ -1320,7 +1318,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1337,10 +1337,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/dd`) - .catch(done) - }) + axios.get(`http://localhost:${port}/dd`) + .catch(done) }) }) @@ -1355,7 +1353,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1367,10 +1367,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/dd`) - .catch(done) - }) + axios.get(`http://localhost:${port}/dd`) + .catch(done) }) }) }) @@ -1401,7 +1399,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1411,11 +1411,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1426,7 +1424,9 @@ describe('Plugin', () => { res.status(400).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1436,13 +1436,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -1453,7 +1451,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1463,13 +1463,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { 'User-Agent': 'test' } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { 'User-Agent': 'test' } + }) + .catch(done) }) }) @@ -1480,7 +1478,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port const spy = sinon.spy() agent @@ -1496,11 +1495,9 @@ describe('Plugin', () => { } }, 100) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/health`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/health`) + .catch(done) }) }) }) @@ -1540,11 +1537,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1559,7 +1556,9 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1570,10 +1569,8 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -1588,7 +1585,9 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -1600,13 +1599,11 @@ describe('Plugin', () => { done() }) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1622,7 +1619,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1635,13 +1634,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -1653,7 +1650,9 @@ describe('Plugin', () => { // eslint-disable-next-line n/handle-callback-err app.use((error, req, res, next) => res.status(500).send()) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1667,13 +1666,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -1683,7 +1680,9 @@ describe('Plugin', () => { app.use(() => { throw error }) - getPort().then(port => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -1698,13 +1697,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) }) diff --git a/packages/datadog-plugin-express/test/leak.js b/packages/datadog-plugin-express/test/leak.js index 3c8b97b4d1c..9d13fc8f978 100644 --- a/packages/datadog-plugin-express/test/leak.js +++ b/packages/datadog-plugin-express/test/leak.js @@ -7,23 +7,22 @@ require('../../dd-trace') const test = require('tape') const express = require('../../../versions/express').get() const axios = require('axios') -const getPort = require('get-port') const profile = require('../../dd-trace/test/profile') test('express plugin should not leak', t => { - getPort().then(port => { - const app = express() + const app = express() - app.use((req, res) => { - res.status(200).send() - }) + app.use((req, res) => { + res.status(200).send() + }) + + const listener = app.listen(0, '127.0.0.1', () => { + const port = listener.address().port - const listener = app.listen(port, '127.0.0.1', () => { - profile(t, operation).then(() => listener.close()) + profile(t, operation).then(() => listener.close()) - function operation (done) { - axios.get(`http://localhost:${port}`).then(done) - } - }) + function operation (done) { + axios.get(`http://localhost:${port}`).then(done) + } }) }) diff --git a/packages/datadog-plugin-fastify/test/index.spec.js b/packages/datadog-plugin-fastify/test/index.spec.js index 920807fbd29..33b1430f98c 100644 --- a/packages/datadog-plugin-fastify/test/index.spec.js +++ b/packages/datadog-plugin-fastify/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') const agent = require('../../dd-trace/test/plugins/agent') @@ -48,7 +47,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -66,11 +67,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -83,7 +82,9 @@ describe('Plugin', () => { } }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -101,11 +102,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -117,7 +116,9 @@ describe('Plugin', () => { } }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -135,11 +136,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) } @@ -155,12 +154,12 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.get(`http://localhost:${port}/user`) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.get(`http://localhost:${port}/user`) + .then(() => done()) + .catch(done) }) }) @@ -172,12 +171,12 @@ describe('Plugin', () => { app.get('/user', (request, reply) => reply.send()) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.get(`http://localhost:${port}/user`) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.get(`http://localhost:${port}/user`) + .then(() => done()) + .catch(done) }) }) @@ -187,12 +186,12 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) + .then(() => done()) + .catch(done) }) }) @@ -211,12 +210,12 @@ describe('Plugin', () => { } }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) + .then(() => done()) + .catch(done) }) }) @@ -248,12 +247,12 @@ describe('Plugin', () => { } }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) - .then(() => done()) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.post(`http://localhost:${port}/user`, { foo: 'bar' }) + .then(() => done()) + .catch(done) }) }) @@ -264,7 +263,9 @@ describe('Plugin', () => { reply.send(error = new Error('boom')) }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -279,11 +280,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -295,11 +294,11 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, async () => { - await axios.get(`http://localhost:${port}/user`) - done() - }) + app.listen({ host, port: 0 }, async () => { + const port = app.server.address().port + + await axios.get(`http://localhost:${port}/user`) + done() }) }) @@ -324,11 +323,11 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { - app.listen({ host, port }, () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -343,7 +342,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -359,11 +360,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -376,7 +375,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -389,11 +390,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) } @@ -407,7 +406,9 @@ describe('Plugin', () => { return Promise.reject(error = new Error('boom')) }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -422,11 +423,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -442,7 +441,9 @@ describe('Plugin', () => { throw (error = new Error('boom')) }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -458,11 +459,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -476,7 +475,9 @@ describe('Plugin', () => { throw new Error('boom') }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -492,11 +493,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) } @@ -515,7 +514,9 @@ describe('Plugin', () => { reply.send() }) - getPort().then(port => { + app.listen({ host, port: 0 }, () => { + const port = app.server.address().port + agent .use(traces => { const spans = traces[0] @@ -531,11 +532,9 @@ describe('Plugin', () => { .then(done) .catch(done) - app.listen({ host, port }, () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) } diff --git a/packages/datadog-plugin-fetch/test/index.spec.js b/packages/datadog-plugin-fetch/test/index.spec.js index f7bd7b85889..1d322de04a4 100644 --- a/packages/datadog-plugin-fetch/test/index.spec.js +++ b/packages/datadog-plugin-fetch/test/index.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const tags = require('../../../ext/tags') const { expect } = require('chai') @@ -21,9 +20,9 @@ describe('Plugin', () => { let appListener describe('fetch', () => { - function server (app, port, listener) { + function server (app, listener) { const server = require('http').createServer(app) - server.listen(port, 'localhost', listener) + server.listen(0, 'localhost', () => listener(server.address().port)) return server } @@ -54,10 +53,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/user`) }) }, rawExpectedSchema.client @@ -68,7 +65,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -84,9 +81,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + fetch(`http://localhost:${port}/user`) }) }) @@ -95,7 +90,7 @@ describe('Plugin', () => { app.post('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -111,9 +106,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) - }) + fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) }) }) @@ -122,7 +115,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -138,9 +131,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(new globalThis.Request(`http://localhost:${port}/user`)) - }) + fetch(new globalThis.Request(`http://localhost:${port}/user`)) }) }) @@ -149,15 +140,13 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(new globalThis.Request(`http://localhost:${port}/user`)) - .then(res => { - expect(res).to.have.property('status', 200) - done() - }) - .catch(done) - }) + appListener = server(app, port => { + fetch(new globalThis.Request(`http://localhost:${port}/user`)) + .then(res => { + expect(res).to.have.property('status', 200) + done() + }) + .catch(done) }) }) @@ -168,7 +157,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -177,9 +166,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user?foo=bar`) - }) + fetch(`http://localhost:${port}/user?foo=bar`) }) }) @@ -193,7 +180,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -201,9 +188,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user?foo=bar`) - }) + fetch(`http://localhost:${port}/user?foo=bar`) }) }) @@ -218,7 +203,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -226,9 +211,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user?foo=bar`, { headers: { foo: 'bar' } }) - }) + fetch(`http://localhost:${port}/user?foo=bar`, { headers: { foo: 'bar' } }) }) }) @@ -248,13 +231,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/`, { - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/`, { + headers: { + Authorization: 'AWS4-HMAC-SHA256 ...' + } }) }) }) @@ -275,13 +256,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/`, { - headers: { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/`, { + headers: { + Authorization: ['AWS4-HMAC-SHA256 ...'] + } }) }) }) @@ -302,13 +281,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/`, { - headers: { - 'X-Amz-Signature': 'abc123' - } - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/`, { + headers: { + 'X-Amz-Signature': 'abc123' + } }) }) }) @@ -329,30 +306,26 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/?X-Amz-Signature=abc123`) - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/?X-Amz-Signature=abc123`) }) }) it('should handle connection errors', done => { - getPort().then(port => { - let error - - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.have.property('component', 'fetch') - }) - .then(done) - .catch(done) + let error - fetch(`http://localhost:${port}/user`).catch(err => { - error = err + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'fetch') }) + .then(done) + .catch(done) + + fetch('http://localhost:7357/user').catch(err => { + error = err }) }) @@ -363,7 +336,7 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -371,9 +344,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + fetch(`http://localhost:${port}/user`) }) }) @@ -384,7 +355,7 @@ describe('Plugin', () => { res.status(400).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -392,9 +363,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`) - }) + fetch(`http://localhost:${port}/user`) }) }) @@ -403,7 +372,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -412,15 +381,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const controller = new AbortController() + const controller = new AbortController() - fetch(`http://localhost:${port}/user`, { - signal: controller.signal - }).catch(e => {}) + fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(e => {}) - controller.abort() - }) + controller.abort() }) }) @@ -431,7 +398,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -439,15 +406,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const controller = new AbortController() + const controller = new AbortController() - fetch(`http://localhost:${port}/user`, { - signal: controller.signal - }).catch(e => {}) + fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(e => {}) - controller.abort() - }) + controller.abort() }) }) @@ -458,7 +423,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -467,15 +432,13 @@ describe('Plugin', () => { clearTimeout(timer) }) - appListener = server(app, port, () => { - const store = storage.getStore() + const store = storage.getStore() - storage.enterWith({ noop: true }) + storage.enterWith({ noop: true }) - fetch(`http://localhost:${port}/user`).catch(() => {}) + fetch(`http://localhost:${port}/user`).catch(() => {}) - storage.enterWith(store) - }) + storage.enterWith(store) }) }) }) @@ -502,7 +465,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'custom') @@ -510,9 +473,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -539,7 +500,7 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -547,9 +508,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -576,7 +535,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', `localhost:${port}`) @@ -584,9 +543,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -614,7 +571,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const meta = traces[0][0].meta @@ -625,13 +582,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`, { - headers: { - 'x-baz': 'qux' - } - }).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`, { + headers: { + 'x-baz': 'qux' + } + }).catch(() => {}) }) }) }) @@ -662,7 +617,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('foo', '/foo') @@ -670,9 +625,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -708,10 +661,8 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/users`).catch(() => {}) - }) + appListener = server(app, port => { + fetch(`http://localhost:${port}/users`).catch(() => {}) }) }) }) @@ -738,7 +689,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -748,9 +699,7 @@ describe('Plugin', () => { }) .catch(done) - appListener = server(app, port, () => { - fetch(`http://localhost:${port}/users`).catch(() => {}) - }) + fetch(`http://localhost:${port}/users`).catch(() => {}) }) }) }) diff --git a/packages/datadog-plugin-graphql/test/index.spec.js b/packages/datadog-plugin-graphql/test/index.spec.js index 7d7aae7fb71..aa8c754f28a 100644 --- a/packages/datadog-plugin-graphql/test/index.spec.js +++ b/packages/datadog-plugin-graphql/test/index.spec.js @@ -7,7 +7,6 @@ const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/c const { expectedSchema, rawExpectedSchema } = require('./naming') const axios = require('axios') const http = require('http') -const getPort = require('get-port') const dc = require('dc-polyfill') const plugin = require('../src') @@ -231,14 +230,16 @@ describe('Plugin', () => { const yoga = graphqlYoga.createYoga({ schema }) server = http.createServer(yoga) - - getPort().then(newPort => { - port = newPort - server.listen(port) - }) }) }) + before(done => { + server.listen(0, () => { + port = server.address().port + done() + }) + }) + after(() => { server.close() return agent.close({ ritmReset: false }) diff --git a/packages/datadog-plugin-hapi/test/index.spec.js b/packages/datadog-plugin-hapi/test/index.spec.js index d3bb3a5d68a..2e67022f494 100644 --- a/packages/datadog-plugin-hapi/test/index.spec.js +++ b/packages/datadog-plugin-hapi/test/index.spec.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') const { ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') @@ -47,15 +46,13 @@ describe('Plugin', () => { if (semver.intersects(version, '>=17')) { beforeEach(() => { - return getPort() - .then(_port => { - port = _port - server = Hapi.server({ - address: 'localhost', - port - }) - return server.start() - }) + server = Hapi.server({ + address: 'localhost', + port: 0 + }) + return server.start().then(() => { + port = server.listener.address().port + }) }) afterEach(() => { @@ -63,19 +60,19 @@ describe('Plugin', () => { }) } else { beforeEach(done => { - getPort() - .then(_port => { - port = _port - - if (Hapi.Server.prototype.connection) { - server = new Hapi.Server() - server.connection({ address: 'localhost', port }) - } else { - server = new Hapi.Server('localhost', port) - } + if (Hapi.Server.prototype.connection) { + server = new Hapi.Server() + server.connection({ address: 'localhost', port }) + } else { + server = new Hapi.Server('localhost', port) + } - server.start(done) - }) + server.start(err => { + if (!err) { + port = server.listener.address().port + } + done(err) + }) }) afterEach(done => { diff --git a/packages/datadog-plugin-http2/test/client.spec.js b/packages/datadog-plugin-http2/test/client.spec.js index 970569c12a5..f8d44f3ac0b 100644 --- a/packages/datadog-plugin-http2/test/client.spec.js +++ b/packages/datadog-plugin-http2/test/client.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const fs = require('fs') const path = require('path') @@ -24,7 +23,7 @@ describe('Plugin', () => { const protocol = pluginToBeLoaded.split(':')[1] || pluginToBeLoaded const loadPlugin = pluginToBeLoaded.includes('node:') ? 'node:http2' : 'http2' describe(`http2/client, protocol ${pluginToBeLoaded}`, () => { - function server (app, port, listener) { + function server (app, listener) { let server if (pluginToBeLoaded === 'https' || pluginToBeLoaded === 'node:https') { process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' @@ -33,7 +32,7 @@ describe('Plugin', () => { server = require(loadPlugin).createServer() } server.on('stream', app) - server.listen(port, 'localhost', listener) + server.listen(0, 'localhost', () => listener(server.address().port)) return server } @@ -58,23 +57,22 @@ describe('Plugin', () => { }) const spanProducerFn = (done) => { - getPort().then(port => { - const app = (stream, headers) => { - stream.respond({ - ':status': 200 - }) - stream.end() - } - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const app = (stream, headers) => { + stream.respond({ + ':status': 200 + }) + stream.end() + } - const req = client.request({ ':path': '/user', ':method': 'GET' }) - req.on('error', done) + appListener = server(app, port => { + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - req.end() - }) + const req = client.request({ ':path': '/user', ':method': 'GET' }) + req.on('error', done) + + req.end() }) } @@ -99,7 +97,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -116,16 +114,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user', ':method': 'GET' }) - req.on('error', done) + const req = client.request({ ':path': '/user', ':method': 'GET' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -137,7 +133,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('span.kind', 'client') @@ -146,16 +142,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({}) - .on('error', done) + const req = client.request({}) + .on('error', done) - req.end() - }) + req.end() }) }) @@ -167,7 +161,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -181,16 +175,14 @@ describe('Plugin', () => { port } - appListener = server(app, port, () => { - const client = http2 - .connect(uri) - .on('error', done) + const client = http2 + .connect(uri) + .on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -202,7 +194,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -210,16 +202,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user?foo=bar' }) - req.on('error', done) + const req = client.request({ ':path': '/user?foo=bar' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -232,7 +222,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -252,21 +242,19 @@ describe('Plugin', () => { port: 1337 } - appListener = server(app, port, () => { - let client - if (protocol === 'https') { - client = http2.connect(incorrectConfig, correctConfig) - } else { - client = http2.connect(correctConfig, incorrectConfig) - } + let client + if (protocol === 'https') { + client = http2.connect(incorrectConfig, correctConfig) + } else { + client = http2.connect(correctConfig, incorrectConfig) + } - client.on('error', done) + client.on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -279,7 +267,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/user`) @@ -299,21 +287,19 @@ describe('Plugin', () => { port: 1337 } - appListener = server(app, port, () => { - let client - if (protocol === 'https') { - client = http2.connect(`${protocol}://remotehost:1337`, correctConfig) - } else { - client = http2.connect(`${protocol}://localhost:${port}`, incorrectConfig) - } + let client + if (protocol === 'https') { + client = http2.connect(`${protocol}://remotehost:1337`, correctConfig) + } else { + client = http2.connect(`${protocol}://localhost:${port}`, incorrectConfig) + } - client.on('error', done) + client.on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -325,7 +311,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.url', `${protocol}://localhost:${port}/`) @@ -338,16 +324,14 @@ describe('Plugin', () => { port } - appListener = server(app, port, () => { - const client = http2 - .connect(uri) - .on('error', done) + const client = http2 + .connect(uri) + .on('error', done) - const req = client.request({}) - req.on('error', done) + const req = client.request({}) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -362,7 +346,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -370,16 +354,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({}) - req.on('error', done) + const req = client.request({}) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -400,20 +382,18 @@ describe('Plugin', () => { } } - getPort().then(port => { - appListener = server(app, port, () => { - const headers = { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const headers = { + Authorization: 'AWS4-HMAC-SHA256 ...' + } + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request(headers) - req.on('error', done) + const req = client.request(headers) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -434,20 +414,18 @@ describe('Plugin', () => { } } - getPort().then(port => { - appListener = server(app, port, () => { - const headers = { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const headers = { + Authorization: ['AWS4-HMAC-SHA256 ...'] + } + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request(headers) - req.on('error', done) + const req = client.request(headers) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -468,20 +446,18 @@ describe('Plugin', () => { } } - getPort().then(port => { - appListener = server(app, port, () => { - const headers = { - 'X-Amz-Signature': 'abc123' - } - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const headers = { + 'X-Amz-Signature': 'abc123' + } + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request(headers) - req.on('error', done) + const req = client.request(headers) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -502,17 +478,15 @@ describe('Plugin', () => { } } - getPort().then(port => { - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/?X-Amz-Signature=abc123' }) - req.on('error', done) + const req = client.request({ ':path': '/?X-Amz-Signature=abc123' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -524,53 +498,49 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + appListener = server(app, port => { + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const span = {} + const span = {} - tracer.scope().activate(span, () => { - const req = client.request({ ':path': '/user' }) - req.on('response', (headers, flags) => { - expect(tracer.scope().active()).to.equal(span) - done() - }) + tracer.scope().activate(span, () => { + const req = client.request({ ':path': '/user' }) + req.on('response', (headers, flags) => { + expect(tracer.scope().active()).to.equal(span) + done() + }) - req.on('error', done) + req.on('error', done) - req.end() - }) + req.end() }) }) }) it('should handle connection errors', done => { - getPort().then(port => { - let error + let error - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.have.property('component', 'http2') - expect(traces[0][0].metrics).to.have.property('network.destination.port', port) - }) - .then(done) - .catch(done) + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'http2') + expect(traces[0][0].metrics).to.have.property('network.destination.port', 7357) + }) + .then(done) + .catch(done) - const client = http2.connect(`${protocol}://localhost:${port}`) - // eslint-disable-next-line n/handle-callback-err - .on('error', (err) => {}) + const client = http2.connect(`${protocol}://localhost:7357`) + // eslint-disable-next-line n/handle-callback-err + .on('error', (err) => {}) - const req = client.request({ ':path': '/user' }) - .on('error', (err) => { error = err }) + const req = client.request({ ':path': '/user' }) + .on('error', (err) => { error = err }) - req.end() - }) + req.end() }) it('should not record HTTP 5XX responses as errors by default', done => { @@ -581,7 +551,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -589,16 +559,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/' }) - req.on('error', done) + const req = client.request({ ':path': '/' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -610,7 +578,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -618,16 +586,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/' }) - req.on('error', done) + const req = client.request({ ':path': '/' }) + req.on('error', done) - req.end() - }) + req.end() }) }) @@ -640,7 +606,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const spans = traces[0] @@ -649,24 +615,22 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - // Activate a new parent span so we capture any double counting that may happen, otherwise double-counts - // would be siblings and our test would only capture 1 as a false positive. - const span = tracer.startSpan('http-test') - tracer.scope().activate(span, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + // Activate a new parent span so we capture any double counting that may happen, otherwise double-counts + // would be siblings and our test would only capture 1 as a false positive. + const span = tracer.startSpan('http-test') + tracer.scope().activate(span, () => { + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/test-1' }) - .on('error', done) - .end() + client.request({ ':path': '/test-1' }) + .on('error', done) + .end() - client.request({ ':path': '/user?test=2' }) - .on('error', done) - .end() + client.request({ ':path': '/user?test=2' }) + .on('error', done) + .end() - span.finish() - }) + span.finish() }) }) }) @@ -697,7 +661,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'custom') @@ -705,16 +669,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) }) @@ -744,7 +706,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -752,16 +714,14 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) - const req = client.request({ ':path': '/user' }) - req.on('error', done) + const req = client.request({ ':path': '/user' }) + req.on('error', done) - req.end() - }) + req.end() }) }) }) @@ -786,24 +746,23 @@ describe('Plugin', () => { withNamingSchema( (done) => { - getPort().then(port => { - serverPort = port - const app = (stream, headers) => { - stream.respond({ - ':status': 200 - }) - stream.end() - } - appListener = server(app, port, () => { - const client = http2 - .connect(`${protocol}://localhost:${port}`) - .on('error', done) - - const req = client.request({ ':path': '/user', ':method': 'GET' }) - req.on('error', done) - - req.end() + const app = (stream, headers) => { + stream.respond({ + ':status': 200 }) + stream.end() + } + appListener = server(app, port => { + serverPort = port + + const client = http2 + .connect(`${protocol}://localhost:${port}`) + .on('error', done) + + const req = client.request({ ':path': '/user', ':method': 'GET' }) + req.on('error', done) + + req.end() }) }, { @@ -826,7 +785,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', `localhost:${port}`) @@ -834,14 +793,12 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/user' }) - .on('error', done) - .end() - }) + client.request({ ':path': '/user' }) + .on('error', done) + .end() }) }) }) @@ -872,7 +829,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const meta = traces[0][0].meta @@ -883,14 +840,12 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/user' }) - .on('error', done) - .end() - }) + client.request({ ':path': '/user' }) + .on('error', done) + .end() }) }) }) @@ -920,7 +875,7 @@ describe('Plugin', () => { stream.end() } - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -930,14 +885,12 @@ describe('Plugin', () => { }) .catch(done) - appListener = server(app, port, () => { - const client = http2.connect(`${protocol}://localhost:${port}`) - .on('error', done) + const client = http2.connect(`${protocol}://localhost:${port}`) + .on('error', done) - client.request({ ':path': '/user' }) - .on('error', done) - .end() - }) + client.request({ ':path': '/user' }) + .on('error', done) + .end() }) }) }) diff --git a/packages/datadog-plugin-http2/test/server.spec.js b/packages/datadog-plugin-http2/test/server.spec.js index c412c9ccbea..d86817b2860 100644 --- a/packages/datadog-plugin-http2/test/server.spec.js +++ b/packages/datadog-plugin-http2/test/server.spec.js @@ -1,7 +1,6 @@ 'use strict' const { EventEmitter } = require('events') -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const { rawExpectedSchema } = require('./naming') @@ -63,12 +62,6 @@ describe('Plugin', () => { } }) - beforeEach(() => { - return getPort().then(newPort => { - port = newPort - }) - }) - afterEach(() => { appListener && appListener.close() app = null @@ -96,7 +89,10 @@ describe('Plugin', () => { beforeEach(done => { const server = http2.createServer(listener) appListener = server - .listen(port, 'localhost', () => done()) + .listen(0, 'localhost', () => { + port = appListener.address().port + done() + }) }) it('should send traces to agent', (done) => { diff --git a/packages/datadog-plugin-koa/test/index.spec.js b/packages/datadog-plugin-koa/test/index.spec.js index f37db4acf32..2a123f18f3c 100644 --- a/packages/datadog-plugin-koa/test/index.spec.js +++ b/packages/datadog-plugin-koa/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const { ERROR_TYPE } = require('../../dd-trace/src/constants') const agent = require('../../dd-trace/test/plugins/agent') @@ -16,14 +15,9 @@ describe('Plugin', () => { describe('koa', () => { withVersions('koa', 'koa', version => { - let port - beforeEach(() => { tracer = require('../../dd-trace') Koa = require(`../../../versions/koa@${version}`).get() - return getPort().then(newPort => { - port = newPort - }) }) afterEach(done => { @@ -41,29 +35,31 @@ describe('Plugin', () => { ctx.body = '' }) - agent - .use(traces => { - const spans = sort(traces[0]) - - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') - - expect(spans[1]).to.have.property('name', 'koa.middleware') - expect(spans[1]).to.have.property('service', 'test') - expect(spans[1]).to.have.property('resource', 'handle') - expect(spans[1].meta).to.have.property('component', 'koa') - }) - .then(done) - .catch(done) - - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans[1]).to.have.property('name', 'koa.middleware') + expect(spans[1]).to.have.property('service', 'test') + expect(spans[1]).to.have.property('resource', 'handle') + expect(spans[1].meta).to.have.property('component', 'koa') + }) + .then(done) + .catch(done) + axios .get(`http://localhost:${port}/user`) .catch(done) @@ -78,29 +74,31 @@ describe('Plugin', () => { yield next }) - agent - .use(traces => { - const spans = sort(traces[0]) - - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') - - expect(spans[1]).to.have.property('name', 'koa.middleware') - expect(spans[1]).to.have.property('service', 'test') - expect(spans[1]).to.have.property('resource', 'converted') - expect(spans[1].meta).to.have.property('component', 'koa') - }) - .then(done) - .catch(done) - - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans[1]).to.have.property('name', 'koa.middleware') + expect(spans[1]).to.have.property('service', 'test') + expect(spans[1]).to.have.property('resource', 'converted') + expect(spans[1].meta).to.have.property('component', 'koa') + }) + .then(done) + .catch(done) + axios .get(`http://localhost:${port}/user`) .catch(done) @@ -123,7 +121,9 @@ describe('Plugin', () => { .catch(done) }) - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + axios .get(`http://localhost:${port}/app/user/123`) .catch(done) @@ -151,11 +151,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -206,12 +206,12 @@ describe('Plugin', () => { return next() }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app/user/1`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/app/user/1`) + .catch(done) }) }) @@ -232,17 +232,19 @@ describe('Plugin', () => { app .use(koaRouter.get('/user/:id', getUser)) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -269,21 +271,23 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[1]).to.have.property('resource') - expect(spans[1].resource).to.match(/^dispatch/) + agent + .use(traces => { + const spans = sort(traces[0]) + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - expect(spans[2]).to.have.property('resource', 'handle') - }) - .then(done) - .catch(done) + expect(spans[1]).to.have.property('resource') + expect(spans[1].resource).to.match(/^dispatch/) + + expect(spans[2]).to.have.property('resource', 'handle') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -303,16 +307,18 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -332,16 +338,18 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -360,16 +368,18 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -390,16 +400,18 @@ describe('Plugin', () => { app.use(router1.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /public/plop') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /public/plop') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', (e) => { axios .get(`http://localhost:${port}/public/plop`) .catch(done) @@ -422,18 +434,20 @@ describe('Plugin', () => { app.use(forums.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/discussions/:did/posts/:pid') - expect(spans[0].meta) - .to.have.property('http.url', `http://localhost:${port}/forums/123/discussions/456/posts/789`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/discussions/:did/posts/:pid') + expect(spans[0].meta) + .to.have.property('http.url', `http://localhost:${port}/forums/123/discussions/456/posts/789`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/forums/123/discussions/456/posts/789`) .catch(done) @@ -457,18 +471,20 @@ describe('Plugin', () => { app.use(first.routes()) app.use(second.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /first/child') - expect(spans[0].meta) - .to.have.property('http.url', `http://localhost:${port}/first/child`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /first/child') + expect(spans[0].meta) + .to.have.property('http.url', `http://localhost:${port}/first/child`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/first/child`) .catch(done) @@ -492,17 +508,19 @@ describe('Plugin', () => { app.use(forums.routes()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/posts/:pid') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/forums/123/posts/456`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /forums/:fid/posts/:pid') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/forums/123/posts/456`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/forums/123/posts/456`) .catch(done) @@ -523,17 +541,19 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user/123`) .catch(done) @@ -556,26 +576,28 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - expect(spans[0].error).to.equal(1) + agent + .use(traces => { + const spans = sort(traces[0]) - expect(spans[1]).to.have.property('resource') - expect(spans[1].resource).to.match(/^dispatch/) - expect(spans[1].meta).to.include({ - [ERROR_TYPE]: error.name, - component: 'koa' + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + expect(spans[0].error).to.equal(1) + + expect(spans[1]).to.have.property('resource') + expect(spans[1].resource).to.match(/^dispatch/) + expect(spans[1].meta).to.include({ + [ERROR_TYPE]: error.name, + component: 'koa' + }) + expect(spans[1].error).to.equal(1) }) - expect(spans[1].error).to.equal(1) - }) - .then(done) - .catch(done) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user/123`) .catch(() => {}) @@ -609,7 +631,9 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + ws = new WebSocket(`ws://localhost:${port}/message`) ws.on('error', done) ws.on('open', () => { @@ -638,26 +662,28 @@ describe('Plugin', () => { ctx.body = '' }) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') + agent + .use(traces => { + const spans = sort(traces[0]) - expect(spans).to.have.length(1) - }) - .then(done) - .catch(done) + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans).to.have.length(1) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user`) .catch(done) @@ -672,26 +698,28 @@ describe('Plugin', () => { yield next }) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('name', 'koa.request') - expect(spans[0]).to.have.property('service', 'test') - expect(spans[0]).to.have.property('type', 'web') - expect(spans[0]).to.have.property('resource', 'GET') - expect(spans[0].meta).to.have.property('span.kind', 'server') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) - expect(spans[0].meta).to.have.property('http.method', 'GET') - expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('component', 'koa') + agent + .use(traces => { + const spans = sort(traces[0]) - expect(spans).to.have.length(1) - }) - .then(done) - .catch(done) + expect(spans[0]).to.have.property('name', 'koa.request') + expect(spans[0]).to.have.property('service', 'test') + expect(spans[0]).to.have.property('type', 'web') + expect(spans[0]).to.have.property('resource', 'GET') + expect(spans[0].meta).to.have.property('span.kind', 'server') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`) + expect(spans[0].meta).to.have.property('http.method', 'GET') + expect(spans[0].meta).to.have.property('http.status_code', '200') + expect(spans[0].meta).to.have.property('component', 'koa') + + expect(spans).to.have.length(1) + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user`) .catch(done) @@ -714,7 +742,9 @@ describe('Plugin', () => { .catch(done) }) - appListener = app.listen(port, 'localhost', () => { + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + axios .get(`http://localhost:${port}/app/user/123`) .catch(done) @@ -742,11 +772,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -770,11 +800,11 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = app.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios.get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -801,19 +831,21 @@ describe('Plugin', () => { .use(router.routes()) .use(router.allowedMethods()) - agent - .use(traces => { - const spans = sort(traces[0]) + appListener = app.listen(0, 'localhost', () => { + const port = appListener.address().port - expect(spans[0]).to.have.property('resource', 'GET /user/:id') - expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) - expect(spans[0].error).to.equal(1) - expect(spans[0].meta).to.have.property('component', 'koa') - }) - .then(done) - .catch(done) + agent + .use(traces => { + const spans = sort(traces[0]) + + expect(spans[0]).to.have.property('resource', 'GET /user/:id') + expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user/123`) + expect(spans[0].error).to.equal(1) + expect(spans[0].meta).to.have.property('component', 'koa') + }) + .then(done) + .catch(done) - appListener = app.listen(port, 'localhost', () => { axios .get(`http://localhost:${port}/user/123`) .catch(() => {}) diff --git a/packages/datadog-plugin-microgateway-core/test/index.spec.js b/packages/datadog-plugin-microgateway-core/test/index.spec.js index c6beea05ec6..1b76c947122 100644 --- a/packages/datadog-plugin-microgateway-core/test/index.spec.js +++ b/packages/datadog-plugin-microgateway-core/test/index.spec.js @@ -2,7 +2,6 @@ const axios = require('axios') const http = require('http') -const getPort = require('get-port') const os = require('os') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') @@ -22,7 +21,11 @@ describe('Plugin', () => { const api = http.createServer((req, res) => res.end('OK')) api.listen(apiPort, function () { + const apiPort = api.address().port + proxy.listen(proxyPort, function () { + const proxyPort = proxy.address().port + gateway = Gateway({ edgemicro: { port: gatewayPort, @@ -34,7 +37,10 @@ describe('Plugin', () => { ] }) - gateway.start(cb) + gateway.start((err, server) => { + gatewayPort = server.address().port + cb(err) + }) }) }) } @@ -47,12 +53,6 @@ describe('Plugin', () => { describe('microgateway-core', () => { withVersions('microgateway-core', 'microgateway-core', (version) => { - beforeEach(async () => { - gatewayPort = await getPort() - proxyPort = await getPort() - apiPort = await getPort() - }) - afterEach(() => { stopGateway() }) diff --git a/packages/datadog-plugin-net/test/index.spec.js b/packages/datadog-plugin-net/test/index.spec.js index ee91149d9ba..adcf175e405 100644 --- a/packages/datadog-plugin-net/test/index.spec.js +++ b/packages/datadog-plugin-net/test/index.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const dns = require('dns') const agent = require('../../dd-trace/test/plugins/agent') const { expectSomeSpan } = require('../../dd-trace/test/plugins/helpers') @@ -36,11 +35,7 @@ describe('Plugin', () => { tracer = require('../../dd-trace') parent = tracer.startSpan('parent') parent.finish() - - return getPort() }).then(_port => { - port = _port - return new Promise(resolve => setImmediate(resolve)) }) }) @@ -49,7 +44,10 @@ describe('Plugin', () => { tcp = new net.Server(socket => { socket.write('') }) - tcp.listen(port, () => done()) + tcp.listen(0, () => { + port = tcp.address().port + done() + }) }) beforeEach(done => { diff --git a/packages/datadog-plugin-openai/src/index.js b/packages/datadog-plugin-openai/src/index.js index aa00beb0a44..739a80e8e7d 100644 --- a/packages/datadog-plugin-openai/src/index.js +++ b/packages/datadog-plugin-openai/src/index.js @@ -7,6 +7,7 @@ const { storage } = require('../../datadog-core') const services = require('./services') const Sampler = require('../../dd-trace/src/sampler') const { MEASURED } = require('../../../ext/tags') +const { estimateTokens } = require('./token-estimator') // String#replaceAll unavailable on Node.js@v14 (dd-trace@<=v3) const RE_NEWLINE = /\n/g @@ -15,14 +16,17 @@ const RE_TAB = /\t/g // TODO: In the future we should refactor config.js to make it requirable let MAX_TEXT_LEN = 128 -let encodingForModel -try { - // eslint-disable-next-line import/no-extraneous-dependencies - encodingForModel = require('tiktoken').encoding_for_model -} catch { - // we will use token count estimations in this case +function safeRequire (path) { + try { + // eslint-disable-next-line import/no-extraneous-dependencies + return require(path) + } catch { + return null + } } +const encodingForModel = safeRequire('tiktoken')?.encoding_for_model + class OpenApiPlugin extends TracingPlugin { static get id () { return 'openai' } static get operation () { return 'request' } @@ -305,6 +309,7 @@ class OpenApiPlugin extends TracingPlugin { } sendLog (methodName, span, tags, store, error) { + if (!store) return if (!Object.keys(store).length) return if (!this.sampler.isSampled()) return @@ -325,9 +330,22 @@ function countPromptTokens (methodName, payload, model) { const messages = payload.messages for (const message of messages) { const content = message.content - const { tokens, estimated } = countTokens(content, model) - promptTokens += tokens - promptEstimated = estimated + if (typeof content === 'string') { + const { tokens, estimated } = countTokens(content, model) + promptTokens += tokens + promptEstimated = estimated + } else if (Array.isArray(content)) { + for (const c of content) { + if (c.type === 'text') { + const { tokens, estimated } = countTokens(c.text, model) + promptTokens += tokens + promptEstimated = estimated + } + // unsupported token computation for image_url + // as even though URL is a string, its true token count + // is based on the image itself, something onerous to do client-side + } + } } } else if (methodName === 'completions.create') { let prompt = payload.prompt @@ -382,25 +400,6 @@ function countTokens (content, model) { } } -// If model is unavailable or tiktoken is not imported, then provide a very rough estimate of the number of tokens -// Approximate using the following assumptions: -// * English text -// * 1 token ~= 4 chars -// * 1 token ~= ¾ words -function estimateTokens (content) { - let estimatedTokens = 0 - if (typeof content === 'string') { - const estimation1 = content.length / 4 - - const matches = content.match(/[\w']+|[.,!?;~@#$%^&*()+/-]/g) - const estimation2 = matches ? matches.length * 0.75 : 0 // in the case of an empty string - estimatedTokens = Math.round((1.5 * estimation1 + 0.5 * estimation2) / 2) - } else if (Array.isArray(content) && typeof content[0] === 'number') { - estimatedTokens = content.length - } - return estimatedTokens -} - function createEditRequestExtraction (tags, payload, store) { const instruction = payload.instruction tags['openai.request.instruction'] = instruction @@ -418,7 +417,7 @@ function createChatCompletionRequestExtraction (tags, payload, store) { store.messages = payload.messages for (let i = 0; i < payload.messages.length; i++) { const message = payload.messages[i] - tags[`openai.request.messages.${i}.content`] = truncateText(message.content) + tagChatCompletionRequestContent(message.content, i, tags) tags[`openai.request.messages.${i}.role`] = message.role tags[`openai.request.messages.${i}.name`] = message.name tags[`openai.request.messages.${i}.finish_reason`] = message.finish_reason @@ -707,7 +706,7 @@ function commonCreateResponseExtraction (tags, body, store, methodName) { for (let choiceIdx = 0; choiceIdx < body.choices.length; choiceIdx++) { const choice = body.choices[choiceIdx] - // logprobs can be nullm and we still want to tag it as 'returned' even when set to 'null' + // logprobs can be null and we still want to tag it as 'returned' even when set to 'null' const specifiesLogProb = Object.keys(choice).indexOf('logprobs') !== -1 tags[`openai.response.choices.${choiceIdx}.finish_reason`] = choice.finish_reason @@ -781,6 +780,7 @@ function truncateApiKey (apiKey) { */ function truncateText (text) { if (!text) return + if (typeof text !== 'string' || !text || (typeof text === 'string' && text.length === 0)) return text = text .replace(RE_NEWLINE, '\\n') @@ -793,6 +793,28 @@ function truncateText (text) { return text } +function tagChatCompletionRequestContent (contents, messageIdx, tags) { + if (typeof contents === 'string') { + tags[`openai.request.messages.${messageIdx}.content`] = contents + } else if (Array.isArray(contents)) { + // content can also be an array of objects + // which represent text input or image url + for (const contentIdx in contents) { + const content = contents[contentIdx] + const type = content.type + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.type`] = content.type + if (type === 'text') { + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.text`] = truncateText(content.text) + } else if (type === 'image_url') { + tags[`openai.request.messages.${messageIdx}.content.${contentIdx}.image_url.url`] = + truncateText(content.image_url.url) + } + // unsupported type otherwise, won't be tagged + } + } + // unsupported type otherwise, won't be tagged +} + // The server almost always responds with JSON function coerceResponseBody (body, methodName) { switch (methodName) { diff --git a/packages/datadog-plugin-openai/src/token-estimator.js b/packages/datadog-plugin-openai/src/token-estimator.js new file mode 100644 index 00000000000..46595f0c2a5 --- /dev/null +++ b/packages/datadog-plugin-openai/src/token-estimator.js @@ -0,0 +1,20 @@ +'use strict' + +// If model is unavailable or tiktoken is not imported, then provide a very rough estimate of the number of tokens +// Approximate using the following assumptions: +// * English text +// * 1 token ~= 4 chars +// * 1 token ~= ¾ words +module.exports.estimateTokens = function (content) { + let estimatedTokens = 0 + if (typeof content === 'string') { + const estimation1 = content.length / 4 + + const matches = content.match(/[\w']+|[.,!?;~@#$%^&*()+/-]/g) + const estimation2 = matches ? matches.length * 0.75 : 0 // in the case of an empty string + estimatedTokens = Math.round((1.5 * estimation1 + 0.5 * estimation2) / 2) + } else if (Array.isArray(content) && typeof content[0] === 'number') { + estimatedTokens = content.length + } + return estimatedTokens +} diff --git a/packages/datadog-plugin-openai/test/index.spec.js b/packages/datadog-plugin-openai/test/index.spec.js index cdbcb72b969..b9db0e27c0a 100644 --- a/packages/datadog-plugin-openai/test/index.spec.js +++ b/packages/datadog-plugin-openai/test/index.spec.js @@ -715,7 +715,7 @@ describe('Plugin', () => { }) if (semver.satisfies(realVersion, '<4.0.0')) { - // `edits.create` was deprecated and removed after 4.0.0 + // `edits.create` was deprecated and removed after 4.0.0 it('makes a successful call', async () => { const checkTraces = agent .use(traces => { @@ -1124,11 +1124,11 @@ describe('Plugin', () => { const result = await openai.downloadFile('file-t3k1gVSQDHrfZnPckzftlZ4A') /** - * TODO: Seems like an OpenAI library bug? - * downloading single line JSONL file results in the JSON being converted into an object. - * downloading multi-line JSONL file then provides a basic string. - * This suggests the library is doing `try { return JSON.parse(x) } catch { return x }` - */ + * TODO: Seems like an OpenAI library bug? + * downloading single line JSONL file results in the JSON being converted into an object. + * downloading multi-line JSONL file then provides a basic string. + * This suggests the library is doing `try { return JSON.parse(x) } catch { return x }` + */ expect(result.data[0]).to.eql('{') // raw JSONL file } @@ -2655,9 +2655,9 @@ describe('Plugin', () => { expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', message: - semver.satisfies(realVersion, '>=4.0.0') - ? 'sampled chat.completions.create' - : 'sampled createChatCompletion', + semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled chat.completions.create' + : 'sampled createChatCompletion', messages: [ { role: 'user', @@ -2703,6 +2703,62 @@ describe('Plugin', () => { await checkTraces }) + + it('should tag image_url', async () => { + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + // image_url is only relevant on request/input, output has the same shape as a normal chat completion + expect(span.meta).to.have.property('openai.request.messages.0.content.0.type', 'text') + expect(span.meta).to.have.property( + 'openai.request.messages.0.content.0.text', 'I\'m allergic to peanuts. Should I avoid this food?' + ) + expect(span.meta).to.have.property('openai.request.messages.0.content.1.type', 'image_url') + expect(span.meta).to.have.property( + 'openai.request.messages.0.content.1.image_url.url', 'dummy/url/peanut_food.png' + ) + }) + + const params = { + model: 'gpt-4-visual-preview', + messages: [ + { + role: 'user', + name: 'hunter2', + content: [ + { + type: 'text', + text: 'I\'m allergic to peanuts. Should I avoid this food?' + }, + { + type: 'image_url', + image_url: { + url: 'dummy/url/peanut_food.png' + } + } + ] + } + ] + } + + if (semver.satisfies(realVersion, '>=4.0.0')) { + const result = await openai.chat.completions.create(params) + + expect(result.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') + expect(result.choices[0].message.role).to.eql('assistant') + expect(result.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') + expect(result.choices[0].finish_reason).to.eql('length') + } else { + const result = await openai.createChatCompletion(params) + + expect(result.data.id).to.eql('chatcmpl-7GaWqyMTD9BLmkmy8SxyjUGX3KSRN') + expect(result.data.choices[0].message.role).to.eql('assistant') + expect(result.data.choices[0].message.content).to.eql('In that case, it\'s best to avoid peanut') + expect(result.data.choices[0].finish_reason).to.eql('length') + } + + await checkTraces + }) }) describe('create chat completion with tools', () => { @@ -2809,9 +2865,9 @@ describe('Plugin', () => { expect(externalLoggerStub).to.have.been.calledWith({ status: 'info', message: - semver.satisfies(realVersion, '>=4.0.0') - ? 'sampled chat.completions.create' - : 'sampled createChatCompletion', + semver.satisfies(realVersion, '>=4.0.0') + ? 'sampled chat.completions.create' + : 'sampled createChatCompletion', messages: [ { role: 'user', @@ -3088,13 +3144,11 @@ describe('Plugin', () => { expect(span.meta).to.have.property('openai.response.choices.0.message.content', 'Hello! How can I assist you today?') - // token metrics - these should be estimated counts expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) + expect(span.metrics).to.not.have.property('openai.response.usage.prompt_tokens_estimated') expect(span.metrics).to.have.property('openai.response.usage.completion_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) + expect(span.metrics).to.not.have.property('openai.response.usage.completion_tokens_estimated') expect(span.metrics).to.have.property('openai.response.usage.total_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) }) const stream = await openai.chat.completions.create({ @@ -3193,7 +3247,7 @@ describe('Plugin', () => { expect(span.meta).to.have.property('openai.response.choices.1.message.role', 'assistant') expect(span.meta).to.have.property('openai.response.choices.1.message.content', 'I\'m just a computer program so I don\'t have feelings, ' + - 'but I\'m here and ready to help you with anything you need. How can I assis...' + 'but I\'m here and ready to help you with anything you need. How can I assis...' ) // message 2 @@ -3202,7 +3256,7 @@ describe('Plugin', () => { expect(span.meta).to.have.property('openai.response.choices.2.message.role', 'assistant') expect(span.meta).to.have.property('openai.response.choices.2.message.content', 'I\'m just a computer program, so I don\'t have feelings like humans do. ' + - 'I\'m here and ready to assist you with any questions or tas...' + 'I\'m here and ready to assist you with any questions or tas...' ) }) @@ -3269,6 +3323,54 @@ describe('Plugin', () => { expect(metricStub).to.have.been.calledWith('openai.tokens.total', 16, 'd', expectedTags) }) + it('makes a successful chat completion call without image_url usage computed', async () => { + nock('https://api.openai.com:443') + .post('/v1/chat/completions') + .reply(200, function () { + return fs.createReadStream(Path.join(__dirname, 'streamed-responses/chat.completions.simple.txt')) + }, { + 'Content-Type': 'text/plain', + 'openai-organization': 'kill-9' + }) + + const checkTraces = agent + .use(traces => { + const span = traces[0][0] + + // we shouldn't be trying to capture the image_url tokens + expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens', 1) + }) + + const stream = await openai.chat.completions.create({ + stream: 1, + model: 'gpt-4o', + messages: [ + { + role: 'user', + name: 'hunter2', + content: [ + { + type: 'text', + text: 'One' // one token, for ease of testing + }, + { + type: 'image_url', + image_url: { + url: 'dummy/url/peanut_food.png' + } + } + ] + } + ] + }) + + for await (const part of stream) { + expect(part).to.have.property('choices') + } + + await checkTraces + }) + it('makes a successful completion call', async () => { nock('https://api.openai.com:443') .post('/v1/completions') @@ -3289,23 +3391,21 @@ describe('Plugin', () => { expect(span.meta).to.have.property('openai.organization.name', 'kill-9') expect(span.meta).to.have.property('openai.request.method', 'POST') expect(span.meta).to.have.property('openai.request.endpoint', '/v1/completions') - expect(span.meta).to.have.property('openai.request.model', 'gpt-4o') + expect(span.meta).to.have.property('openai.request.model', 'text-davinci-002') expect(span.meta).to.have.property('openai.request.prompt', 'Hello, OpenAI!') expect(span.meta).to.have.property('openai.response.choices.0.finish_reason', 'stop') expect(span.meta).to.have.property('openai.response.choices.0.logprobs', 'returned') expect(span.meta).to.have.property('openai.response.choices.0.text', ' this is a test.') - // token metrics - these should be estimated counts expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) + expect(span.metrics).to.not.have.property('openai.response.usage.prompt_tokens_estimated') expect(span.metrics).to.have.property('openai.response.usage.completion_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) + expect(span.metrics).to.not.have.property('openai.response.usage.completion_tokens_estimated') expect(span.metrics).to.have.property('openai.response.usage.total_tokens') - expect(span.metrics).to.have.property('openai.response.usage.prompt_tokens_estimated', 1) }) const stream = await openai.completions.create({ - model: 'gpt-4o', + model: 'text-davinci-002', prompt: 'Hello, OpenAI!', temperature: 0.5, stream: true diff --git a/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt b/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt index 38b54feeac2..b0f9045ae9e 100644 --- a/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt +++ b/packages/datadog-plugin-openai/test/streamed-responses/completions.simple.txt @@ -1,15 +1,15 @@ -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" ","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" ","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":"this","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":"this","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" is","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" is","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" a","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" a","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" test","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":" test","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":".","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":".","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-002"} -data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":"","index":0,"logprobs":null,"finish_reason":"stop"}],"model":"gpt-3.5-turbo-instruct"} +data: {"id":"cmpl-9SBFwkdjMAXO6n7Z0cz7ScN1SKJSr","object":"text_completion","created":1716503480,"choices":[{"text":"","index":0,"logprobs":null,"finish_reason":"stop"}],"model":"text-davinci-002"} data: [DONE] \ No newline at end of file diff --git a/packages/datadog-plugin-openai/test/token-estimator.spec.js b/packages/datadog-plugin-openai/test/token-estimator.spec.js new file mode 100644 index 00000000000..375c655738a --- /dev/null +++ b/packages/datadog-plugin-openai/test/token-estimator.spec.js @@ -0,0 +1,28 @@ +'use strict' + +const { estimateTokens } = require('../src/token-estimator') + +describe('Plugin', () => { + describe('openai token estimation', () => { + function testEstimation (input, expected) { + const tokens = estimateTokens(input) + expect(tokens).to.equal(expected) + } + + it('should compute the number of tokens in a string', () => { + testEstimation('hello world', 2) + }) + + it('should not throw for an empty string', () => { + testEstimation('', 0) + }) + + it('should compute the number of tokens in an array of integer inputs', () => { + testEstimation([1, 2, 3], 3) + }) + + it('should compute no tokens for invalid content', () => { + testEstimation({}, 0) + }) + }) +}) diff --git a/packages/datadog-plugin-paperplane/test/index.spec.js b/packages/datadog-plugin-paperplane/test/index.spec.js index 167fb3841a4..5499e9cc98b 100644 --- a/packages/datadog-plugin-paperplane/test/index.spec.js +++ b/packages/datadog-plugin-paperplane/test/index.spec.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') @@ -76,7 +75,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -94,11 +95,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -113,7 +112,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -123,11 +124,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/1`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/1`) + .catch(done) }) }) @@ -155,7 +154,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -165,11 +166,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -191,7 +190,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -201,11 +202,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -220,7 +219,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -230,13 +231,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -247,7 +246,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -257,11 +258,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/app`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/app`) + .catch(done) }) }) @@ -280,7 +279,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -290,10 +291,8 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios.get(`http://localhost:${port}/app/user/123`) - .catch(done) - }) + axios.get(`http://localhost:${port}/app/user/123`) + .catch(done) }) }) @@ -308,7 +307,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -318,17 +319,15 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { - 'x-datadog-trace-id': '1234', - 'x-datadog-parent-id': '5678', - 'ot-baggage-foo': 'bar' - } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { + 'x-datadog-trace-id': '1234', + 'x-datadog-parent-id': '5678', + 'ot-baggage-foo': 'bar' + } + }) + .catch(done) }) }) @@ -343,7 +342,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -355,13 +356,11 @@ describe('Plugin', () => { done() }) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -376,7 +375,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent.use(traces => { const spans = sort(traces[0]) @@ -388,13 +389,11 @@ describe('Plugin', () => { done() }) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -405,7 +404,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -417,13 +418,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 500 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 500 + }) + .catch(done) }) }) @@ -477,7 +476,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -487,11 +488,9 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -506,7 +505,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -516,13 +517,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - validateStatus: status => status === 400 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + validateStatus: status => status === 400 + }) + .catch(done) }) }) @@ -537,7 +536,9 @@ describe('Plugin', () => { server = http.createServer(mount({ app, cry, logger })) - getPort().then(port => { + server.listen(0, 'localhost', () => { + const port = server.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -547,13 +548,11 @@ describe('Plugin', () => { .then(done) .catch(done) - server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`, { - headers: { 'User-Agent': 'test' } - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user`, { + headers: { 'User-Agent': 'test' } + }) + .catch(done) }) }) diff --git a/packages/datadog-plugin-restify/test/index.spec.js b/packages/datadog-plugin-restify/test/index.spec.js index bb96b34a132..71dc94d44a4 100644 --- a/packages/datadog-plugin-restify/test/index.spec.js +++ b/packages/datadog-plugin-restify/test/index.spec.js @@ -2,7 +2,6 @@ const { AsyncLocalStorage } = require('async_hooks') const axios = require('axios') -const getPort = require('get-port') const semver = require('semver') const agent = require('../../dd-trace/test/plugins/agent') const { ERROR_MESSAGE } = require('../../dd-trace/src/constants') @@ -35,7 +34,9 @@ describe('Plugin', () => { it('should do automatic instrumentation', done => { const server = restify.createServer() - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('name', 'restify.request') @@ -51,11 +52,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(() => {}) - }) + axios + .get(`http://localhost:${port}/user`) + .catch(() => {}) }) }) @@ -67,7 +66,9 @@ describe('Plugin', () => { return next() }) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /user/:id') @@ -77,11 +78,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -96,7 +95,9 @@ describe('Plugin', () => { } ) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /user/:id') @@ -106,11 +107,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -135,7 +134,9 @@ describe('Plugin', () => { } ) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(warningSpy).to.not.have.been.called @@ -143,11 +144,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -184,12 +183,12 @@ describe('Plugin', () => { next() }) - getPort().then(port => { - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -201,7 +200,9 @@ describe('Plugin', () => { return next() }]) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /user/:id') @@ -211,11 +212,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/user/123`) + .catch(done) }) }) @@ -239,12 +238,12 @@ describe('Plugin', () => { res.end() }) - getPort().then(port => { - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/user`) - .catch(done) - }) + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + + axios + .get(`http://localhost:${port}/user`) + .catch(done) }) }) @@ -264,7 +263,9 @@ describe('Plugin', () => { throw new Error('uncaught') }]) - getPort().then(port => { + appListener = server.listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { expect(traces[0][0]).to.have.property('resource', 'GET /error') @@ -276,13 +277,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server.listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/error`, { - validateStatus: status => status === 599 - }) - .catch(done) - }) + axios + .get(`http://localhost:${port}/error`, { + validateStatus: status => status === 599 + }) + .catch(done) }) }) }) diff --git a/packages/datadog-plugin-router/test/index.spec.js b/packages/datadog-plugin-router/test/index.spec.js index 3c8e8ee68f3..ac208f0e2a1 100644 --- a/packages/datadog-plugin-router/test/index.spec.js +++ b/packages/datadog-plugin-router/test/index.spec.js @@ -5,7 +5,6 @@ const axios = require('axios') const http = require('http') const { once } = require('events') -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const web = require('../../dd-trace/src/plugins/util/web') @@ -87,7 +86,9 @@ describe('Plugin', () => { router.use('/parent', childRouter) - getPort().then(port => { + appListener = server(router).listen(0, 'localhost', () => { + const port = appListener.address().port + agent .use(traces => { const spans = sort(traces[0]) @@ -97,11 +98,9 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(router).listen(port, 'localhost', () => { - axios - .get(`http://localhost:${port}/parent/child/123`) - .catch(done) - }) + axios + .get(`http://localhost:${port}/parent/child/123`) + .catch(done) }) }) @@ -115,15 +114,15 @@ describe('Plugin', () => { res.end() }) - const port = await getPort() const agentPromise = agent.use(traces => { for (const span of traces[0]) { expect(span.error).to.equal(0) } }, { rejectFirst: true }) - const httpd = server(router).listen(port, 'localhost') + const httpd = server(router).listen(0, 'localhost') await once(httpd, 'listening') + const port = httpd.address().port const reqPromise = axios.get(`http://localhost:${port}/foo`) return Promise.all([agentPromise, reqPromise]) @@ -139,7 +138,6 @@ describe('Plugin', () => { res.end() }) - const port = await getPort() const agentPromise = agent.use(traces => { for (const span of traces[0]) { expect(span.error).to.equal(0) @@ -147,8 +145,9 @@ describe('Plugin', () => { }, { rejectFirst: true }) // eslint-disable-next-line n/handle-callback-err - const httpd = server(router, (req, res) => err => res.end()).listen(port, 'localhost') + const httpd = server(router, (req, res) => err => res.end()).listen(0, 'localhost') await once(httpd, 'listening') + const port = httpd.address().port const reqPromise = axios.get(`http://localhost:${port}/foo`) return Promise.all([agentPromise, reqPromise]) diff --git a/packages/datadog-plugin-undici/test/index.spec.js b/packages/datadog-plugin-undici/test/index.spec.js index 734e8f6c9a9..70f4ea02f0c 100644 --- a/packages/datadog-plugin-undici/test/index.spec.js +++ b/packages/datadog-plugin-undici/test/index.spec.js @@ -1,6 +1,5 @@ 'use strict' -const getPort = require('get-port') const agent = require('../../dd-trace/test/plugins/agent') const tags = require('../../../ext/tags') const { expect } = require('chai') @@ -20,9 +19,9 @@ describe('Plugin', () => { describe('undici-fetch', () => { withVersions('undici', 'undici', version => { - function server (app, port, listener) { + function server (app, listener) { const server = require('http').createServer(app) - server.listen(port, 'localhost', listener) + server.listen(0, 'localhost', () => listener(server.address().port)) return server } @@ -59,10 +58,8 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`, { method: 'GET' }) - }) + appListener = server(app, port => { + fetch.fetch(`http://localhost:${port}/user`, { method: 'GET' }) }) }, rawExpectedSchema.client @@ -73,7 +70,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'test') @@ -89,9 +86,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`, { method: 'GET' }) - }) + fetch.fetch(`http://localhost:${port}/user`, { method: 'GET' }) }) }) @@ -100,7 +95,7 @@ describe('Plugin', () => { app.post('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -116,9 +111,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) - }) + fetch.fetch(new URL(`http://localhost:${port}/user`), { method: 'POST' }) }) }) @@ -127,15 +120,13 @@ describe('Plugin', () => { app.get('/user', (req, res) => { res.status(200).send() }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch.fetch((`http://localhost:${port}/user`)) - .then(res => { - expect(res).to.have.property('status', 200) - done() - }) - .catch(done) - }) + appListener = server(app, port => { + fetch.fetch((`http://localhost:${port}/user`)) + .then(res => { + expect(res).to.have.property('status', 200) + done() + }) + .catch(done) }) }) @@ -146,7 +137,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -155,9 +146,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user?foo=bar`) - }) + fetch.fetch(`http://localhost:${port}/user?foo=bar`) }) }) @@ -171,7 +160,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -179,9 +168,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user?foo=bar`) - }) + fetch.fetch(`http://localhost:${port}/user?foo=bar`) }) }) @@ -196,7 +183,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('http.status_code', '200') @@ -204,28 +191,24 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user?foo=bar`, { headers: { foo: 'bar' } }) - }) + fetch.fetch(`http://localhost:${port}/user?foo=bar`, { headers: { foo: 'bar' } }) }) }) it('should handle connection errors', done => { - getPort().then(port => { - let error - - agent - .use(traces => { - expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) - expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) - expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) - expect(traces[0][0].meta).to.have.property('component', 'undici') - }) - .then(done) - .catch(done) - - fetch.fetch(`http://localhost:${port}/user`).catch(err => { - error = err + let error + + agent + .use(traces => { + expect(traces[0][0].meta).to.have.property(ERROR_TYPE, error.name) + expect(traces[0][0].meta).to.have.property(ERROR_MESSAGE, error.message || error.code) + expect(traces[0][0].meta).to.have.property(ERROR_STACK, error.stack) + expect(traces[0][0].meta).to.have.property('component', 'undici') }) + .then(done) + .catch(done) + + fetch.fetch('http://localhost:7357/user').catch(err => { + error = err }) }) it('should not record HTTP 5XX responses as errors by default', done => { @@ -235,7 +218,7 @@ describe('Plugin', () => { res.status(500).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -243,9 +226,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`) - }) + fetch.fetch(`http://localhost:${port}/user`) }) }) @@ -256,7 +237,7 @@ describe('Plugin', () => { res.status(400).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 1) @@ -264,9 +245,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`) - }) + fetch.fetch(`http://localhost:${port}/user`) }) }) @@ -275,7 +254,7 @@ describe('Plugin', () => { app.get('/user', (req, res) => {}) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('error', 0) @@ -284,15 +263,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const controller = new AbortController() + const controller = new AbortController() - fetch.fetch(`http://localhost:${port}/user`, { - signal: controller.signal - }).catch(() => {}) + fetch.fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(() => {}) - controller.abort() - }) + controller.abort() }) }) @@ -303,7 +280,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', SERVICE_NAME) @@ -311,15 +288,13 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - const controller = new AbortController() + const controller = new AbortController() - fetch.fetch(`http://localhost:${port}/user`, { - signal: controller.signal - }).catch(() => {}) + fetch.fetch(`http://localhost:${port}/user`, { + signal: controller.signal + }).catch(() => {}) - controller.abort() - }) + controller.abort() }) }) }) @@ -345,7 +320,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0]).to.have.property('service', 'custom') @@ -353,9 +328,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch.fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -382,7 +355,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { const meta = traces[0][0].meta @@ -392,13 +365,11 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`, { - headers: { - 'x-baz': 'qux' - } - }).catch(() => {}) - }) + fetch.fetch(`http://localhost:${port}/user`, { + headers: { + 'x-baz': 'qux' + } + }).catch(() => {}) }) }) }) @@ -428,7 +399,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { agent .use(traces => { expect(traces[0][0].meta).to.have.property('foo', '/foo') @@ -436,9 +407,7 @@ describe('Plugin', () => { .then(done) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/user`).catch(() => {}) - }) + fetch.fetch(`http://localhost:${port}/user`).catch(() => {}) }) }) }) @@ -474,10 +443,8 @@ describe('Plugin', () => { } }) - getPort().then(port => { - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/users`).catch(() => {}) - }) + appListener = server(app, port => { + fetch.fetch(`http://localhost:${port}/users`).catch(() => {}) }) }) }) @@ -504,7 +471,7 @@ describe('Plugin', () => { res.status(200).send() }) - getPort().then(port => { + appListener = server(app, port => { const timer = setTimeout(done, 100) agent @@ -514,9 +481,7 @@ describe('Plugin', () => { }) .catch(done) - appListener = server(app, port, () => { - fetch.fetch(`http://localhost:${port}/users`).catch(() => {}) - }) + fetch.fetch(`http://localhost:${port}/users`).catch(() => {}) }) }) }) diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js new file mode 100644 index 00000000000..c47467528e6 --- /dev/null +++ b/packages/datadog-plugin-vitest/src/index.js @@ -0,0 +1,156 @@ +const CiPlugin = require('../../dd-trace/src/plugins/ci_plugin') +const { storage } = require('../../datadog-core') + +const { + TEST_STATUS, + finishAllTraceSpans, + getTestSuitePath, + getTestSuiteCommonTags, + TEST_SOURCE_FILE +} = require('../../dd-trace/src/plugins/util/test') +const { COMPONENT } = require('../../dd-trace/src/constants') + +// Milliseconds that we subtract from the error test duration +// so that they do not overlap with the following test +// This is because there's some loss of resolution. +const MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION = 5 + +class VitestPlugin extends CiPlugin { + static get id () { + return 'vitest' + } + + constructor (...args) { + super(...args) + + this.taskToFinishTime = new WeakMap() + + this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const store = storage.getStore() + const span = this.startTestSpan( + testName, + testSuite, + this.testSuiteSpan, + { + [TEST_SOURCE_FILE]: testSuite + } + ) + + this.enter(span, store) + }) + + this.addSub('ci:vitest:test:finish-time', ({ status, task }) => { + const store = storage.getStore() + const span = store?.span + + // we store the finish time to finish at a later hook + // this is because the test might fail at a `afterEach` hook + if (span) { + span.setTag(TEST_STATUS, status) + this.taskToFinishTime.set(task, span._getTime()) + } + }) + + this.addSub('ci:vitest:test:pass', ({ task }) => { + const store = storage.getStore() + const span = store?.span + + if (span) { + span.setTag(TEST_STATUS, 'pass') + span.finish(this.taskToFinishTime.get(task)) + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:vitest:test:error', ({ duration, error }) => { + const store = storage.getStore() + const span = store?.span + + if (span) { + span.setTag(TEST_STATUS, 'fail') + + if (error) { + span.setTag('error', error) + } + span.finish(span._startTime + duration - MILLISECONDS_TO_SUBTRACT_FROM_FAILED_TEST_DURATION) // milliseconds + finishAllTraceSpans(span) + } + }) + + this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + this.startTestSpan( + testName, + testSuite, + this.testSuiteSpan, + { + [TEST_SOURCE_FILE]: testSuite, + [TEST_STATUS]: 'skip' + } + ).finish() + }) + + this.addSub('ci:vitest:test-suite:start', (testSuiteAbsolutePath) => { + const testSessionSpanContext = this.tracer.extract('text_map', { + 'x-datadog-trace-id': process.env.DD_CIVISIBILITY_TEST_SESSION_ID, + 'x-datadog-parent-id': process.env.DD_CIVISIBILITY_TEST_MODULE_ID + }) + + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const testSuiteMetadata = getTestSuiteCommonTags( + this.command, + this.frameworkVersion, + testSuite, + 'vitest' + ) + const testSuiteSpan = this.tracer.startSpan('vitest.test_suite', { + childOf: testSessionSpanContext, + tags: { + [COMPONENT]: this.constructor.id, + ...this.testEnvironmentMetadata, + ...testSuiteMetadata + } + }) + const store = storage.getStore() + this.enter(testSuiteSpan, store) + this.testSuiteSpan = testSuiteSpan + }) + + this.addSub('ci:vitest:test-suite:finish', ({ status, onFinish }) => { + const store = storage.getStore() + const span = store?.span + if (span) { + span.setTag(TEST_STATUS, status) + span.finish() + finishAllTraceSpans(span) + } + // TODO: too frequent flush - find for method in worker to decrease frequency + this.tracer._exporter.flush(onFinish) + }) + + this.addSub('ci:vitest:test-suite:error', ({ error }) => { + const store = storage.getStore() + const span = store?.span + if (span && error) { + span.setTag('error', error) + span.setTag(TEST_STATUS, 'fail') + } + }) + + this.addSub('ci:vitest:session:finish', ({ status, onFinish, error }) => { + this.testSessionSpan.setTag(TEST_STATUS, status) + this.testModuleSpan.setTag(TEST_STATUS, status) + if (error) { + this.testModuleSpan.setTag('error', error) + this.testSessionSpan.setTag('error', error) + } + this.testModuleSpan.finish() + this.testSessionSpan.finish() + finishAllTraceSpans(this.testSessionSpan) + this.tracer._exporter.flush(onFinish) + }) + } +} + +module.exports = VitestPlugin diff --git a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js index 8bd00c40596..cc25d51b1e9 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js +++ b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js @@ -1,14 +1,14 @@ 'use strict' const { MANUAL_KEEP } = require('../../../../../ext/tags') -const { LRUCache } = require('lru-cache') +const LRU = require('lru-cache') const vulnerabilitiesFormatter = require('./vulnerabilities-formatter') const { IAST_ENABLED_TAG_KEY, IAST_JSON_TAG_KEY } = require('./tags') const standalone = require('../standalone') const VULNERABILITIES_KEY = 'vulnerabilities' const VULNERABILITY_HASHES_MAX_SIZE = 1000 -const VULNERABILITY_HASHES = new LRUCache({ max: VULNERABILITY_HASHES_MAX_SIZE }) +const VULNERABILITY_HASHES = new LRU({ max: VULNERABILITY_HASHES_MAX_SIZE }) const RESET_VULNERABILITY_CACHE_INTERVAL = 60 * 60 * 1000 // 1 hour let tracer diff --git a/packages/dd-trace/src/appsec/reporter.js b/packages/dd-trace/src/appsec/reporter.js index a75e1c92984..e652eaa2099 100644 --- a/packages/dd-trace/src/appsec/reporter.js +++ b/packages/dd-trace/src/appsec/reporter.js @@ -29,19 +29,17 @@ const contentHeaderList = [ 'content-language' ] -const REQUEST_HEADERS_MAP = mapHeaderAndTags([ +const EVENT_HEADERS_MAP = mapHeaderAndTags([ ...ipHeaderList, 'forwarded', 'via', ...contentHeaderList, 'host', - 'user-agent', - 'accept', 'accept-encoding', 'accept-language' ], 'http.request.headers.') -const IDENTIFICATION_HEADERS_MAP = mapHeaderAndTags([ +const identificationHeaders = [ 'x-amzn-trace-id', 'cloudfront-viewer-ja3-fingerprint', 'cf-ray', @@ -50,6 +48,14 @@ const IDENTIFICATION_HEADERS_MAP = mapHeaderAndTags([ 'x-sigsci-requestid', 'x-sigsci-tags', 'akamai-user-risk' +] + +// these request headers are always collected - it breaks the expected spec orders +const REQUEST_HEADERS_MAP = mapHeaderAndTags([ + 'content-type', + 'user-agent', + 'accept', + ...identificationHeaders ], 'http.request.headers.') const RESPONSE_HEADERS_MAP = mapHeaderAndTags(contentHeaderList, 'http.response.headers.') @@ -118,9 +124,9 @@ function reportAttack (attackData) { const currentTags = rootSpan.context()._tags - const newTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP) - - newTags['appsec.event'] = 'true' + const newTags = { + 'appsec.event': 'true' + } if (limiter.isAllowed()) { newTags[MANUAL_KEEP] = 'true' @@ -142,11 +148,6 @@ function reportAttack (attackData) { newTags['_dd.appsec.json'] = '{"triggers":' + attackData + '}' } - const ua = newTags['http.request.headers.user-agent'] - if (ua) { - newTags['http.useragent'] = ua - } - newTags['network.client.ip'] = req.socket.remoteAddress rootSpan.addTags(newTags) @@ -205,19 +206,40 @@ function finishRequest (req, res) { incrementWafRequestsMetric(req) // collect some headers even when no attack is detected - rootSpan.addTags(filterHeaders(req.headers, IDENTIFICATION_HEADERS_MAP)) + const mandatoryTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP) + const ua = mandatoryTags['http.request.headers.user-agent'] + if (ua) { + mandatoryTags['http.useragent'] = ua + } + rootSpan.addTags(mandatoryTags) - if (!rootSpan.context()._tags['appsec.event']) return + const tags = rootSpan.context()._tags + if (!shouldCollectEventHeaders(tags)) return const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_MAP) + Object.assign(newTags, filterHeaders(req.headers, EVENT_HEADERS_MAP)) - if (req.route && typeof req.route.path === 'string') { + if (tags['appsec.event'] === 'true' && typeof req.route?.path === 'string') { newTags['http.endpoint'] = req.route.path } rootSpan.addTags(newTags) } +function shouldCollectEventHeaders (tags = {}) { + if (tags['appsec.event'] === 'true') { + return true + } + + for (const tagName of Object.keys(tags)) { + if (tagName.startsWith('appsec.events.')) { + return true + } + } + + return false +} + function setRateLimit (rateLimit) { limiter = new Limiter(rateLimit) } diff --git a/packages/dd-trace/src/datastreams/pathway.js b/packages/dd-trace/src/datastreams/pathway.js index dc954a9428f..b813b622ca6 100644 --- a/packages/dd-trace/src/datastreams/pathway.js +++ b/packages/dd-trace/src/datastreams/pathway.js @@ -3,7 +3,7 @@ // this inconsistency is ok because hashes do not need to be consistent across services const crypto = require('crypto') const { encodeVarint, decodeVarint } = require('./encoding') -const { LRUCache } = require('lru-cache') +const LRUCache = require('lru-cache') const options = { max: 500 } const cache = new LRUCache(options) diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 310cb2dc940..4daeb02a4bf 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -91,6 +91,13 @@ module.exports = class CiPlugin extends Plugin { ...testModuleSpanMetadata } }) + // only for vitest + // These are added for the worker threads to use + if (this.constructor.id === 'vitest') { + process.env.DD_CIVISIBILITY_TEST_SESSION_ID = this.testSessionSpan.context().toTraceId() + process.env.DD_CIVISIBILITY_TEST_MODULE_ID = this.testModuleSpan.context().toSpanId() + } + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'module') }) diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index 0b98cd9c076..fd9288afcc4 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -18,6 +18,7 @@ module.exports = { get '@opensearch-project/opensearch' () { return require('../../../datadog-plugin-opensearch/src') }, get '@redis/client' () { return require('../../../datadog-plugin-redis/src') }, get '@smithy/smithy-client' () { return require('../../../datadog-plugin-aws-sdk/src') }, + get '@vitest/runner' () { return require('../../../datadog-plugin-vitest/src') }, get aerospike () { return require('../../../datadog-plugin-aerospike/src') }, get amqp10 () { return require('../../../datadog-plugin-amqp10/src') }, get amqplib () { return require('../../../datadog-plugin-amqplib/src') }, @@ -54,6 +55,7 @@ module.exports = { get 'microgateway-core' () { return require('../../../datadog-plugin-microgateway-core/src') }, get mocha () { return require('../../../datadog-plugin-mocha/src') }, get 'mocha-each' () { return require('../../../datadog-plugin-mocha/src') }, + get vitest () { return require('../../../datadog-plugin-vitest/src') }, get workerpool () { return require('../../../datadog-plugin-mocha/src') }, get moleculer () { return require('../../../datadog-plugin-moleculer/src') }, get mongodb () { return require('../../../datadog-plugin-mongodb-core/src') }, diff --git a/packages/dd-trace/src/telemetry/init-telemetry.js b/packages/dd-trace/src/telemetry/init-telemetry.js new file mode 100644 index 00000000000..a126ecc6238 --- /dev/null +++ b/packages/dd-trace/src/telemetry/init-telemetry.js @@ -0,0 +1,75 @@ +'use strict' + +const fs = require('fs') +const { spawn } = require('child_process') +const tracerVersion = require('../../../../package.json').version +const log = require('../log') + +module.exports = sendTelemetry + +if (!process.env.DD_INJECTION_ENABLED) { + module.exports = () => {} +} + +if (!process.env.DD_TELEMETRY_FORWARDER_PATH) { + module.exports = () => {} +} + +if (!fs.existsSync(process.env.DD_TELEMETRY_FORWARDER_PATH)) { + module.exports = () => {} +} + +const metadata = { + language_name: 'nodejs', + language_version: process.versions.node, + runtime_name: 'nodejs', + runtime_version: process.versions.node, + tracer_version: tracerVersion, + pid: process.pid +} + +const seen = [] +function hasSeen (point) { + if (point.name === 'abort') { + // This one can only be sent once, regardless of tags + return seen.includes('abort') + } + if (point.name === 'abort.integration') { + // For now, this is the only other one we want to dedupe + const compiledPoint = point.name + point.tags.join('') + return seen.includes(compiledPoint) + } + return false +} + +function sendTelemetry (name, tags = []) { + let points = name + if (typeof name === 'string') { + points = [{ name, tags }] + } + if (['1', 'true', 'True'].includes(process.env.DD_INJECT_FORCE)) { + points = points.filter(p => ['error', 'complete'].includes(p.name)) + } + points = points.filter(p => !hasSeen(p)) + points.forEach(p => { + p.name = `library_entrypoint.${p.name}` + }) + if (points.length === 0) { + return + } + const proc = spawn(process.env.DD_TELEMETRY_FORWARDER_PATH, ['library_entrypoint'], { + stdio: 'pipe' + }) + proc.on('error', () => { + log.error('Failed to spawn telemetry forwarder') + }) + proc.on('exit', (code) => { + if (code !== 0) { + log.error(`Telemetry forwarder exited with code ${code}`) + } + }) + proc.stdin.on('error', () => { + log.error('Failed to write telemetry data to telemetry forwarder') + }) + proc.stdin.end(JSON.stringify({ metadata, points })) +} diff --git a/packages/dd-trace/test/appsec/rasp.express.plugin.spec.js b/packages/dd-trace/test/appsec/rasp.express.plugin.spec.js index 2aefdb89693..0229984b6fa 100644 --- a/packages/dd-trace/test/appsec/rasp.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/rasp.express.plugin.spec.js @@ -85,7 +85,7 @@ withVersions('express', 'express', expressVersion => { res.end('end') } - axios.get('/?host=ifconfig.pro') + axios.get('/?host=localhost/ifconfig.pro') await agent.use((traces) => { const span = getWebSpan(traces) @@ -108,7 +108,7 @@ withVersions('express', 'express', expressVersion => { res.end('end') } - axios.get('/?host=ifconfig.pro') + axios.get('/?host=localhost/ifconfig.pro') await agent.use((traces) => { const span = getWebSpan(traces) diff --git a/packages/dd-trace/test/appsec/reporter.spec.js b/packages/dd-trace/test/appsec/reporter.spec.js index 721534f6d5b..b8ce6d94fb6 100644 --- a/packages/dd-trace/test/appsec/reporter.spec.js +++ b/packages/dd-trace/test/appsec/reporter.spec.js @@ -225,9 +225,6 @@ describe('reporter', () => { 'manual.keep': 'true', '_dd.origin': 'appsec', '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]}]}', - 'http.request.headers.host': 'localhost', - 'http.request.headers.user-agent': 'arachni', - 'http.useragent': 'arachni', 'network.client.ip': '8.8.8.8' }) }) @@ -267,12 +264,9 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'http.request.headers.host': 'localhost', - 'http.request.headers.user-agent': 'arachni', 'appsec.event': 'true', 'manual.keep': 'true', '_dd.appsec.json': '{"triggers":[]}', - 'http.useragent': 'arachni', 'network.client.ip': '8.8.8.8' }) }) @@ -285,13 +279,10 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'http.request.headers.host': 'localhost', - 'http.request.headers.user-agent': 'arachni', 'appsec.event': 'true', 'manual.keep': 'true', '_dd.origin': 'appsec', '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]},{"rule":{}},{"rule":{},"rule_matches":[{}]}]}', - 'http.useragent': 'arachni', 'network.client.ip': '8.8.8.8' }) }) @@ -304,13 +295,10 @@ describe('reporter', () => { expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ - 'http.request.headers.host': 'localhost', - 'http.request.headers.user-agent': 'arachni', 'appsec.event': 'true', 'manual.keep': 'true', '_dd.origin': 'appsec', '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]},{"rule":{}},{"rule":{},"rule_matches":[{}]}]}', - 'http.useragent': 'arachni', 'network.client.ip': '8.8.8.8' }) @@ -365,6 +353,33 @@ describe('reporter', () => { describe('finishRequest', () => { let wafContext + const requestHeadersToTrackOnEvent = [ + 'x-forwarded-for', + 'x-real-ip', + 'true-client-ip', + 'x-client-ip', + 'x-forwarded', + 'forwarded-for', + 'x-cluster-client-ip', + 'fastly-client-ip', + 'cf-connecting-ip', + 'cf-connecting-ipv6', + 'forwarded', + 'via', + 'content-length', + 'content-encoding', + 'content-language', + 'host', + 'accept-encoding', + 'accept-language' + ] + const requestHeadersAndValuesToTrackOnEvent = {} + const expectedRequestTagsToTrackOnEvent = {} + requestHeadersToTrackOnEvent.forEach((header, index) => { + requestHeadersAndValuesToTrackOnEvent[header] = `val-${index}` + expectedRequestTagsToTrackOnEvent[`http.request.headers.${header}`] = `val-${index}` + }) + beforeEach(() => { wafContext = { dispose: sinon.stub() @@ -398,7 +413,7 @@ describe('reporter', () => { expect(Reporter.metricsQueue).to.be.empty }) - it('should only add identification headers when no attack was previously found', () => { + it('should only add mandatory headers when no attack or event was previously found', () => { const req = { headers: { 'not-included': 'hello', @@ -409,7 +424,10 @@ describe('reporter', () => { 'x-appgw-trace-id': 'e', 'x-sigsci-requestid': 'f', 'x-sigsci-tags': 'g', - 'akamai-user-risk': 'h' + 'akamai-user-risk': 'h', + 'content-type': 'i', + accept: 'j', + 'user-agent': 'k' } } @@ -423,7 +441,11 @@ describe('reporter', () => { 'http.request.headers.x-appgw-trace-id': 'e', 'http.request.headers.x-sigsci-requestid': 'f', 'http.request.headers.x-sigsci-tags': 'g', - 'http.request.headers.akamai-user-risk': 'h' + 'http.request.headers.akamai-user-risk': 'h', + 'http.request.headers.content-type': 'i', + 'http.request.headers.accept': 'j', + 'http.request.headers.user-agent': 'k', + 'http.useragent': 'k' }) }) @@ -484,6 +506,108 @@ describe('reporter', () => { }) }) + it('should add http request data inside request span when appsec.event is true', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + span.context()._tags['appsec.event'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni', + 'http.useragent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + + it('should add http request data inside request span when user login success is tracked', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + + span.context() + ._tags['appsec.events.users.login.success.track'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni', + 'http.useragent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + + it('should add http request data inside request span when user login failure is tracked', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + + span.context() + ._tags['appsec.events.users.login.failure.track'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni', + 'http.useragent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + + it('should add http request data inside request span when user custom event is tracked', () => { + const req = { + headers: { + 'user-agent': 'arachni', + ...requestHeadersAndValuesToTrackOnEvent + } + } + const res = { + getHeaders: () => { + return {} + } + } + + span.context() + ._tags['appsec.events.custon.event.track'] = 'true' + + Reporter.finishRequest(req, res) + + expect(span.addTags).to.have.been.calledWithExactly({ + 'http.request.headers.user-agent': 'arachni', + 'http.useragent': 'arachni' + }) + + expect(span.addTags).to.have.been.calledWithExactly(expectedRequestTagsToTrackOnEvent) + }) + it('should call incrementWafRequestsMetric', () => { const req = {} const res = {} diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index f61774f5619..24b9a0a9c96 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -80,7 +80,7 @@ async function assertVersions () { } async function assertInstrumentation (instrumentation, external) { - const versions = process.env.PACKAGE_VERSION_RANGE + const versions = process.env.PACKAGE_VERSION_RANGE && !external ? [process.env.PACKAGE_VERSION_RANGE] : [].concat(instrumentation.versions || []) diff --git a/yarn.lock b/yarn.lock index 0e8f6319d62..5726f2126ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3455,11 +3455,6 @@ loupe@^2.3.1: dependencies: get-func-name "^2.0.0" -lru-cache@^10.2.2: - version "10.2.2" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.2.2.tgz#48206bc114c1252940c41b25b41af5b545aca878" - integrity sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ== - lru-cache@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz" @@ -4879,6 +4874,11 @@ through@^2.3.8, through@~2.3.4: resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz" integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== +tiktoken@^1.0.15: + version "1.0.15" + resolved "https://registry.yarnpkg.com/tiktoken/-/tiktoken-1.0.15.tgz#a1e11681fa51b50c81bb7eaaee53b7a66e844a23" + integrity sha512-sCsrq/vMWUSEW29CJLNmPvWxlVp7yh2tlkAjpJltIKqp5CKf98ZNpdeHRmAlPVFlGEbswDc6SmI8vz64W/qErw== + tildify@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz"