[test visibility] Speed up cypress tests (#4457)

DataDog · Jul 2, 2024 · 123b365 · 123b365
1 parent b0acc4f
commit 123b365
Show file tree

Hide file tree

Showing 2 changed files with 58 additions and 16 deletions.
diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js
@@ -87,8 +87,7 @@ moduleType.forEach(({
     })
 
     beforeEach(async function () {
-      const port = await getPort()
-      receiver = await new FakeCiVisIntake(port).start()
+      receiver = await new FakeCiVisIntake().start()
     })
 
     afterEach(async () => {
@@ -117,7 +116,8 @@ moduleType.forEach(({
           env: {
             ...restEnvVars,
             CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
-            DD_SITE: '= invalid = url'
+            DD_SITE: '= invalid = url',
+            SPEC_PATTERN: 'cypress/e2e/spec.cy.js'
           },
           stdio: 'pipe'
         }
@@ -130,7 +130,7 @@ moduleType.forEach(({
       })
       childProcess.on('exit', () => {
         assert.notInclude(testOutput, 'TypeError')
-        assert.include(testOutput, '3 of 4 failed')
+        assert.include(testOutput, '1 of 1 failed')
         done()
       })
     })
@@ -351,7 +351,8 @@ moduleType.forEach(({
           cwd,
           env: {
             ...restEnvVars,
-            CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+            CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+            SPEC_PATTERN: 'cypress/e2e/spec.cy.js'
           },
           stdio: 'pipe'
         }
@@ -383,7 +384,8 @@ moduleType.forEach(({
             cwd,
             env: {
               ...restEnvVars,
-              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+              SPEC_PATTERN: 'cypress/e2e/spec.cy.js'
             },
             stdio: 'pipe'
           }
@@ -399,6 +401,7 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+
       it('does not report code coverage if disabled by the API', (done) => {
         receiver.setSettings({
           code_coverage: false,
@@ -428,7 +431,8 @@ moduleType.forEach(({
             cwd,
             env: {
               ...restEnvVars,
-              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+              SPEC_PATTERN: 'cypress/e2e/spec.cy.js'
             },
             stdio: 'pipe'
           }
@@ -440,6 +444,7 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+
       it('can skip tests received by the intelligent test runner API and still reports code coverage', (done) => {
         receiver.setSuitesToSkip([{
           type: 'test',
@@ -498,7 +503,8 @@ moduleType.forEach(({
             cwd,
             env: {
               ...restEnvVars,
-              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+              SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js'
             },
             stdio: 'pipe'
           }
@@ -509,6 +515,7 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+
       it('does not skip tests if test skipping is disabled by the API', (done) => {
         receiver.setSettings({
           code_coverage: true,
@@ -535,6 +542,7 @@ moduleType.forEach(({
               event.content.resource === 'cypress/e2e/other.cy.js.context passes'
             )
             assert.exists(notSkippedTest)
+            assert.equal(notSkippedTest.content.meta[TEST_STATUS], 'pass')
           }, 25000)
 
         const {
@@ -548,7 +556,8 @@ moduleType.forEach(({
             cwd,
             env: {
               ...restEnvVars,
-              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+              SPEC_PATTERN: 'cypress/e2e/other.cy.js'
             },
             stdio: 'pipe'
           }
@@ -560,6 +569,7 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+
       it('does not skip tests if suite is marked as unskippable', (done) => {
         receiver.setSettings({
           code_coverage: true,
@@ -621,7 +631,8 @@ moduleType.forEach(({
             cwd,
             env: {
               ...restEnvVars,
-              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+              SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js'
             },
             stdio: 'pipe'
           }
@@ -633,6 +644,7 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+
       it('only sets forced to run if test was going to be skipped by ITR', (done) => {
         receiver.setSettings({
           code_coverage: true,
@@ -689,7 +701,8 @@ moduleType.forEach(({
             cwd,
             env: {
               ...restEnvVars,
-              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+              SPEC_PATTERN: 'cypress/e2e/{other,spec}.cy.js'
             },
             stdio: 'pipe'
           }
@@ -701,6 +714,7 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+
       it('sets _dd.ci.itr.tests_skipped to false if the received test is not skipped', (done) => {
         receiver.setSuitesToSkip([{
           type: 'test',
@@ -741,7 +755,8 @@ moduleType.forEach(({
             cwd,
             env: {
               ...restEnvVars,
-              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+              SPEC_PATTERN: 'cypress/e2e/spec.cy.js'
             },
             stdio: 'pipe'
           }
@@ -752,6 +767,7 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+
       it('reports itr_correlation_id in tests', (done) => {
         const itrCorrelationId = '4321'
         receiver.setItrCorrelationId(itrCorrelationId)
@@ -775,7 +791,8 @@ moduleType.forEach(({
             cwd,
             env: {
               ...restEnvVars,
-              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
+              CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
+              SPEC_PATTERN: 'cypress/e2e/spec.cy.js'
             },
             stdio: 'pipe'
           }
@@ -816,7 +833,8 @@ moduleType.forEach(({
           env: {
             ...restEnvVars,
             CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
-            CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN: '1'
+            CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN: '1',
+            SPEC_PATTERN: 'cypress/e2e/spec.cy.js'
           },
           stdio: 'pipe'
         }
@@ -841,7 +859,7 @@ moduleType.forEach(({
           assert.equal(testSuiteEvents.length, 4)
           const testEvents = events.filter(event => event.type === 'test')
           assert.equal(testEvents.length, 9)
-        })
+        }, 30000)
 
       const {
         NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress
@@ -880,7 +898,7 @@ moduleType.forEach(({
           assert.equal(testSuiteEvents.length, 4)
           const testEvents = events.filter(event => event.type === 'test')
           assert.equal(testEvents.length, 9)
-        })
+        }, 30000)
 
       const {
         NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress
@@ -980,6 +998,7 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+
       it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => {
         receiver.setSettings({
           itr_enabled: false,
@@ -1041,6 +1060,7 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+
       it('does not retry tests that are skipped', (done) => {
         receiver.setSettings({
           itr_enabled: false,
@@ -1097,6 +1117,7 @@ moduleType.forEach(({
           }).catch(done)
         })
       })
+
       it('does not run EFD if the known tests request fails', (done) => {
         receiver.setSettings({
           itr_enabled: false,

diff --git a/integration-tests/cypress/plugins-old/index.js b/integration-tests/cypress/plugins-old/index.js
@@ -1,6 +1,27 @@
+const ddAfterRun = require('dd-trace/ci/cypress/after-run')
+const ddAfterSpec = require('dd-trace/ci/cypress/after-spec')
+
 module.exports = (on, config) => {
   if (process.env.CYPRESS_ENABLE_INCOMPATIBLE_PLUGIN) {
     require('cypress-fail-fast/plugin')(on, config)
   }
+  if (process.env.SPEC_PATTERN) {
+    config.testFiles = process.env.SPEC_PATTERN.replace('cypress/e2e/', '')
+  }
+  if (process.env.CYPRESS_ENABLE_AFTER_RUN_CUSTOM) {
+    on('after:run', (...args) => {
+      // do custom stuff
+      // and call after-run at the end
+      return ddAfterRun(...args)
+    })
+  }
+  if (process.env.CYPRESS_ENABLE_AFTER_SPEC_CUSTOM) {
+    on('after:spec', (...args) => {
+      // do custom stuff
+      // and call after-spec at the end
+      return ddAfterSpec(...args)
+    })
+  }
   require('dd-trace/ci/cypress/plugin')(on, config)
+  return config
 }