add support for skipped tests

DataDog · Jun 24, 2024 · ba5088c · ba5088c
1 parent cb580df
commit ba5088c
Show file tree

Hide file tree

Showing 4 changed files with 91 additions and 45 deletions.
diff --git a/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs b/integration-tests/ci-visibility/vitest-tests/test-visibility-passed-suite.mjs
@@ -17,4 +17,16 @@ describe('other context', () => {
   test('can report more', () => {
     expect(sum(1, 2)).to.equal(3)
   })
+  test.skip('can skip', () => {
+    expect(sum(1, 2)).to.equal(3)
+  })
+  test.todo('can todo', () => {
+    expect(sum(1, 2)).to.equal(3)
+  })
+  // eslint-disable-next-line
+  test('can programmatic skip', (context) => {
+    // eslint-disable-next-line
+    context.skip()
+    expect(sum(1, 2)).to.equal(3)
+  })
 })
diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js
@@ -75,21 +75,23 @@ versions.forEach((version) => {
         assert.includeMembers(testEvents.map(test => test.content.resource),
           [
             'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
-            '.test-visibility-failed-suite-first-describe can report failed test ',
+            '.test-visibility-failed-suite-first-describe can report failed test',
             'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
-            '.test-visibility-failed-suite-first-describe can report more ',
+            '.test-visibility-failed-suite-first-describe can report more',
             'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
-            '.test-visibility-failed-suite-second-describe can report passed test ',
+            '.test-visibility-failed-suite-second-describe can report passed test',
             'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
-            '.test-visibility-failed-suite-second-describe can report more ',
-            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test ',
-            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more ',
-            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test ',
-            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more ',
-            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test ',
-            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more ',
-            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test ',
-            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more '
+            '.test-visibility-failed-suite-second-describe can report more',
+            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test',
+            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more',
+            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test',
+            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more',
+            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip',
+            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo',
+            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test',
+            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more',
+            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test',
+            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more'
           ]
         )
 
@@ -99,11 +101,22 @@ versions.forEach((version) => {
           failedTests.map(test => test.content.resource),
           [
             'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
-            '.test-visibility-failed-suite-first-describe can report failed test ',
-            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test ',
-            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more ',
-            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test ',
-            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more '
+            '.test-visibility-failed-suite-first-describe can report failed test',
+            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test',
+            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more',
+            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test',
+            'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more'
+          ]
+        )
+
+        const skippedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'skip')
+
+        assert.includeMembers(
+          skippedTests.map(test => test.content.resource),
+          [
+            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip',
+            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo',
+            'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip'
           ]
         )
         // TODO: check error messages
@@ -116,11 +129,14 @@ versions.forEach((version) => {
           env: {
             ...getCiVisAgentlessConfig(receiver.port),
             // maybe only in node@20
-            NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' // ESM requires more stuff
+            NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' // ESM requires more flags
           },
           stdio: 'pipe'
         }
       )
+
+      childProcess.stderr.pipe(process.stderr)
+      childProcess.stdout.pipe(process.stdout)
     })
   })
 })
diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js
@@ -10,6 +10,7 @@ const testStartCh = channel('ci:vitest:test:start')
 const testFinishTimeCh = channel('ci:vitest:test:finish-time')
 const testPassCh = channel('ci:vitest:test:pass')
 const testErrorCh = channel('ci:vitest:test:error')
+const testSkipCh = channel('ci:vitest:test:skip')
 
 const testSuiteStartCh = channel('ci:vitest:test-suite:start')
 const testSuiteFinishCh = channel('ci:vitest:test-suite:finish')
@@ -76,10 +77,10 @@ function getTypeTasks (fileTasks, type = 'test') {
 }
 
 function getTestName (task) {
-  let testName = ''
-  let currentTask = task
+  let testName = task.name
+  let currentTask = task.suite
 
-  while (currentTask.suite) {
+  while (currentTask) {
     if (currentTask.name) {
       testName = `${currentTask.name} ${testName}`
     }
@@ -136,8 +137,7 @@ addHook({
 
   if (isVitestTestRunner(vitestPackage)) {
     const { VitestTestRunner } = vitestPackage
-    // test start
-    // TODO: add test for beforeEach / afterEach and before/after hooks: they're likely tasks too
+    // test start (only tests that are not marked as skip or todo)
     shimmer.wrap(VitestTestRunner.prototype, 'onBeforeTryTask', onBeforeTryTask => async function (task) {
       const asyncResource = new AsyncResource('bound-anonymous-fn')
       taskToAsync.set(task, asyncResource)
@@ -148,8 +148,7 @@ addHook({
       return onBeforeTryTask.apply(this, arguments)
     })
 
-    // test finish
-    // TODO: add test for beforeEach / afterEach and before/after hooks: they're likely tasks too
+    // test finish (only passed tests)
     shimmer.wrap(VitestTestRunner.prototype, 'onAfterTryTask', onAfterTryTask =>
       async function (task, { retry: retryCount }) {
         const result = await onAfterTryTask.apply(this, arguments)
@@ -192,27 +191,33 @@ addHook({
 
       testTasks.forEach(task => {
         const testAsyncResource = taskToAsync.get(task)
-        const { result: { state, duration, errors } } = task
-
-        // what about 'skip'??
-        if (state === 'pass') {
-          testAsyncResource.runInAsyncScope(() => {
-            testPassCh.publish({ task })
-          })
-        } else if (state === 'fail') {
-          // If it's failing, we have no accurate finish time, so we have to use `duration`
-          let testError
-
-          if (errors?.length) {
-            testError = errors[0]
-          }
-
-          testAsyncResource.runInAsyncScope(() => {
-            testErrorCh.publish({ duration, error: testError })
-          })
-          if (errors?.length) {
-            testSuiteError = testError // we store the error to bubble it up to the suite
+        const { result } = task
+
+        if (result) {
+          const { state, duration, errors } = result
+          if (state === 'skip') { // programmatic skip
+            testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath })
+          } else if (state === 'pass') {
+            testAsyncResource.runInAsyncScope(() => {
+              testPassCh.publish({ task })
+            })
+          } else if (state === 'fail') {
+            // If it's failing, we have no accurate finish time, so we have to use `duration`
+            let testError
+
+            if (errors?.length) {
+              testError = errors[0]
+            }
+
+            testAsyncResource.runInAsyncScope(() => {
+              testErrorCh.publish({ duration, error: testError })
+            })
+            if (errors?.length) {
+              testSuiteError = testError // we store the error to bubble it up to the suite
+            }
           }
+        } else { // test.skip or test.todo
+          testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.suite.file.filepath })
         }
       })
 

diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js
@@ -78,6 +78,19 @@ class VitestPlugin extends CiPlugin {
       }
     })
 
+    this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath }) => {
+      const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
+      this.startTestSpan(
+        testName,
+        testSuite,
+        this.testSuiteSpan,
+        {
+          [TEST_SOURCE_FILE]: testSuite,
+          [TEST_STATUS]: 'skip'
+        }
+      ).finish()
+    })
+
     this.addSub('ci:vitest:test-suite:start', (testSuiteAbsolutePath) => {
       const testSessionSpanContext = this.tracer.extract('text_map', {
         'x-datadog-trace-id': process.env.DD_CIVISIBILITY_TEST_SESSION_ID,